@forzalabs/remora 1.0.21 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/actions/automap.js +26 -42
- package/actions/compile.js +27 -43
- package/actions/create_consumer.js +24 -40
- package/actions/create_producer.js +16 -32
- package/actions/debug.js +18 -34
- package/actions/deploy.js +30 -46
- package/actions/discover.js +13 -29
- package/actions/init.js +29 -45
- package/actions/mock.js +16 -32
- package/actions/run.js +34 -52
- package/actions/sample.js +42 -58
- package/index.js +38 -43
- package/package.json +4 -4
- package/workers/ExecutorWorker.js +18 -32
- package/Constants.js +0 -34
- package/core/Affirm.js +0 -42
- package/core/Algo.js +0 -160
- package/core/dste/DSTE.js +0 -113
- package/core/logger/DebugLogService.js +0 -48
- package/core/logger/DevelopmentLogService.js +0 -70
- package/core/logger/LocalLogService.js +0 -70
- package/core/logger/Logger.js +0 -54
- package/database/DatabaseEngine.js +0 -149
- package/database/DatabaseStructure.js +0 -27
- package/definitions/DatasetDefinitions.js +0 -2
- package/definitions/ExecutorDefinitions.js +0 -2
- package/definitions/ProcessENV.js +0 -2
- package/definitions/agents/DestinationDriver.js +0 -2
- package/definitions/agents/SourceDriver.js +0 -2
- package/definitions/cli.js +0 -2
- package/definitions/database/ApiKeys.js +0 -2
- package/definitions/database/Stored.js +0 -7
- package/definitions/database/UsageStat.js +0 -2
- package/definitions/database/User.js +0 -2
- package/definitions/json_schemas/consumer-schema.json +0 -1226
- package/definitions/json_schemas/producer-schema.json +0 -308
- package/definitions/json_schemas/project-schema.json +0 -100
- package/definitions/json_schemas/source-schema.json +0 -249
- package/definitions/requests/ConsumerRequest.js +0 -2
- package/definitions/requests/Developer.js +0 -2
- package/definitions/requests/Mapping.js +0 -2
- package/definitions/requests/ProducerRequest.js +0 -2
- package/definitions/requests/Request.js +0 -2
- package/definitions/resources/Compiled.js +0 -2
- package/definitions/resources/Consumer.js +0 -2
- package/definitions/resources/Environment.js +0 -2
- package/definitions/resources/Library.js +0 -2
- package/definitions/resources/Producer.js +0 -2
- package/definitions/resources/Project.js +0 -2
- package/definitions/resources/Schema.js +0 -2
- package/definitions/resources/Source.js +0 -2
- package/definitions/temp.js +0 -2
- package/definitions/transform/Transformations.js +0 -2
- package/drivers/DeltaShareDriver.js +0 -186
- package/drivers/DriverFactory.js +0 -72
- package/drivers/DriverHelper.js +0 -248
- package/drivers/HttpApiDriver.js +0 -208
- package/drivers/RedshiftDriver.js +0 -184
- package/drivers/files/LocalDestinationDriver.js +0 -146
- package/drivers/files/LocalSourceDriver.js +0 -405
- package/drivers/s3/S3DestinationDriver.js +0 -197
- package/drivers/s3/S3SourceDriver.js +0 -495
- package/engines/CryptoEngine.js +0 -75
- package/engines/Environment.js +0 -170
- package/engines/ProcessENVManager.js +0 -83
- package/engines/RandomEngine.js +0 -47
- package/engines/SecretManager.js +0 -23
- package/engines/UserManager.js +0 -66
- package/engines/ai/AutoMapperEngine.js +0 -37
- package/engines/ai/DeveloperEngine.js +0 -497
- package/engines/ai/LLM.js +0 -255
- package/engines/consumer/ConsumerManager.js +0 -218
- package/engines/consumer/ConsumerOnFinishManager.js +0 -202
- package/engines/dataset/Dataset.js +0 -824
- package/engines/dataset/DatasetManager.js +0 -211
- package/engines/dataset/DatasetRecord.js +0 -120
- package/engines/dataset/DatasetRecordPool.js +0 -77
- package/engines/execution/RequestExecutor.js +0 -67
- package/engines/parsing/CSVParser.js +0 -60
- package/engines/parsing/LineParser.js +0 -71
- package/engines/parsing/ParseCompression.js +0 -101
- package/engines/parsing/ParseHelper.js +0 -18
- package/engines/parsing/ParseManager.js +0 -54
- package/engines/parsing/XLSParser.js +0 -87
- package/engines/parsing/XMLParser.js +0 -115
- package/engines/producer/ProducerEngine.js +0 -127
- package/engines/producer/ProducerManager.js +0 -43
- package/engines/scheduler/CronScheduler.js +0 -222
- package/engines/scheduler/QueueManager.js +0 -314
- package/engines/schema/SchemaValidator.js +0 -67
- package/engines/transform/JoinEngine.js +0 -232
- package/engines/transform/TransformationEngine.js +0 -277
- package/engines/transform/TypeCaster.js +0 -59
- package/engines/usage/DataframeManager.js +0 -55
- package/engines/usage/UsageDataManager.js +0 -151
- package/engines/usage/UsageManager.js +0 -65
- package/engines/validation/Validator.js +0 -216
- package/executors/ConsumerExecutor.js +0 -280
- package/executors/Executor.js +0 -177
- package/executors/ExecutorOrchestrator.js +0 -331
- package/executors/ExecutorPerformance.js +0 -17
- package/executors/ExecutorProgress.js +0 -54
- package/executors/ExecutorScope.js +0 -52
- package/executors/OutputExecutor.js +0 -118
- package/executors/ProducerExecutor.js +0 -108
- package/helper/Helper.js +0 -149
- package/helper/Logger.js +0 -84
- package/helper/Runtime.js +0 -20
- package/helper/Settings.js +0 -13
- package/licencing/LicenceManager.js +0 -64
- package/settings.js +0 -12
package/index.js
CHANGED
|
@@ -1,44 +1,39 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
};
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
const
|
|
21
|
-
const sample_1 = require("./actions/sample");
|
|
22
|
-
const mock_1 = require("./actions/mock");
|
|
23
|
-
const ProcessENVManager_1 = __importDefault(require("./engines/ProcessENVManager"));
|
|
24
|
-
dotenv_1.default.configDotenv();
|
|
25
|
-
const program = new commander_1.Command();
|
|
2
|
+
import { Command } from 'commander';
|
|
3
|
+
import dotenv from 'dotenv';
|
|
4
|
+
import { compile } from './actions/compile';
|
|
5
|
+
import { debug } from './actions/debug';
|
|
6
|
+
import { deploy } from './actions/deploy';
|
|
7
|
+
import { init } from './actions/init';
|
|
8
|
+
import { run } from './actions/run';
|
|
9
|
+
import { discover } from './actions/discover';
|
|
10
|
+
import { create_producer } from './actions/create_producer';
|
|
11
|
+
import { create_consumer } from './actions/create_consumer';
|
|
12
|
+
import { CONSTANTS } from '@remora/app-constants';
|
|
13
|
+
import { LicenceManager } from '@remora/app-auth';
|
|
14
|
+
import { Runtime } from '@remora/app-helper';
|
|
15
|
+
import { automap } from './actions/automap';
|
|
16
|
+
import { sample } from './actions/sample';
|
|
17
|
+
import { mock } from './actions/mock';
|
|
18
|
+
import { ProcessENVManager } from '@remora/app-common';
|
|
19
|
+
dotenv.configDotenv();
|
|
20
|
+
const program = new Command();
|
|
26
21
|
// Validate the remora licence
|
|
27
|
-
const remoraLicenceKey =
|
|
28
|
-
const check =
|
|
22
|
+
const remoraLicenceKey = ProcessENVManager.getEnvVariable('REMORA_LICENCE_KEY');
|
|
23
|
+
const check = LicenceManager.validate(remoraLicenceKey);
|
|
29
24
|
if (!check.valid) {
|
|
30
25
|
console.error(`Invalid Remora licence key, the product is not active: remember to set "REMORA_LICENCE_KEY" environment variable.`);
|
|
31
26
|
process.exit(1);
|
|
32
27
|
}
|
|
33
28
|
// Runtime check on heap size to warn user of insufficent runtime resources
|
|
34
|
-
const { heapSizeMB } =
|
|
35
|
-
if (heapSizeMB <
|
|
36
|
-
console.warn(`Remora is running with ${heapSizeMB}MB of runtime heap, which is below the bare minimum of ${
|
|
37
|
-
else if (heapSizeMB <
|
|
38
|
-
console.warn(`Remora is running with ${heapSizeMB} MB of runtime heap, which is below the recommended of ${
|
|
29
|
+
const { heapSizeMB } = Runtime.getHeap();
|
|
30
|
+
if (heapSizeMB < CONSTANTS.defaults.MIN_RUNTIME_HEAP_MB)
|
|
31
|
+
console.warn(`Remora is running with ${heapSizeMB}MB of runtime heap, which is below the bare minimum of ${CONSTANTS.defaults.MIN_RUNTIME_HEAP_MB}MB (Recommended: ${CONSTANTS.defaults.RECOMMENDED_RUNTIME_HEAP_MB}MB).`);
|
|
32
|
+
else if (heapSizeMB < CONSTANTS.defaults.RECOMMENDED_RUNTIME_HEAP_MB)
|
|
33
|
+
console.warn(`Remora is running with ${heapSizeMB} MB of runtime heap, which is below the recommended of ${CONSTANTS.defaults.RECOMMENDED_RUNTIME_HEAP_MB} MB.`);
|
|
39
34
|
// Initialize all commands
|
|
40
35
|
program
|
|
41
|
-
.version(
|
|
36
|
+
.version(CONSTANTS.cliVersion + '', '-v, --version', 'Display the version of the CLI')
|
|
42
37
|
.description('CLI tool for setting up and managing Data-Remora');
|
|
43
38
|
program
|
|
44
39
|
.command('version')
|
|
@@ -50,47 +45,47 @@ program
|
|
|
50
45
|
program
|
|
51
46
|
.command('init')
|
|
52
47
|
.description('Initialize the application and set up configuration')
|
|
53
|
-
.action(
|
|
48
|
+
.action(init);
|
|
54
49
|
program
|
|
55
50
|
.command('compile')
|
|
56
51
|
.description('Compile the config and check for errors')
|
|
57
|
-
.action(
|
|
52
|
+
.action(compile);
|
|
58
53
|
program
|
|
59
54
|
.command('debug')
|
|
60
55
|
.description('Check the connection with sources and producers')
|
|
61
|
-
.action(
|
|
56
|
+
.action(debug);
|
|
62
57
|
program
|
|
63
58
|
.command('deploy')
|
|
64
59
|
.description('Deploy the application to the specified environment')
|
|
65
60
|
.option('-e, --env <environment>', 'Target environment (staging, production)', 'staging')
|
|
66
61
|
.option('--skip-tests', 'Skip running tests before deployment')
|
|
67
62
|
.option('--build-only', 'Build without deploying')
|
|
68
|
-
.action(
|
|
63
|
+
.action(deploy);
|
|
69
64
|
program
|
|
70
65
|
.command('run [name]')
|
|
71
66
|
.description('Execute consumers. Optionally specify a single consumer name or use --project to run all consumers in a project.')
|
|
72
67
|
.option('-p, --project <name>', 'Run all consumers belonging to the specified project')
|
|
73
|
-
.action((name, options) =>
|
|
68
|
+
.action((name, options) => run(name, options));
|
|
74
69
|
program
|
|
75
70
|
.command('discover')
|
|
76
71
|
.description('Discover the data shape of a producer and automatically create the resource for it.')
|
|
77
72
|
.argument('<producer>', 'The producer to discover')
|
|
78
|
-
.action(
|
|
73
|
+
.action(discover);
|
|
79
74
|
program
|
|
80
75
|
.command('create-producer <name>')
|
|
81
76
|
.description('Create a new producer configuration with default settings')
|
|
82
|
-
.action(
|
|
77
|
+
.action(create_producer);
|
|
83
78
|
program
|
|
84
79
|
.command('automap')
|
|
85
80
|
.description('Automatically map a producer to consumers using specified schemas.')
|
|
86
81
|
.argument('<producer>', 'The producer to analyze')
|
|
87
82
|
.argument('<schemas...>', 'One or more schema names to map against')
|
|
88
|
-
.action(
|
|
83
|
+
.action(automap);
|
|
89
84
|
program
|
|
90
85
|
.command('create-consumer <name>')
|
|
91
86
|
.description('Create a new consumer configuration with default settings')
|
|
92
87
|
.option('-p, --producer <name>', 'Producer to create a one-to-one mapping from')
|
|
93
|
-
.action((name, options) =>
|
|
88
|
+
.action((name, options) => create_consumer(name, options.producer));
|
|
94
89
|
program
|
|
95
90
|
.command('sample <name>')
|
|
96
91
|
.description('Sample data from a producer or consumer and display it in a formatted table')
|
|
@@ -101,7 +96,7 @@ program
|
|
|
101
96
|
console.error('Sample size must be a positive number');
|
|
102
97
|
process.exit(1);
|
|
103
98
|
}
|
|
104
|
-
|
|
99
|
+
sample(name, sampleSize);
|
|
105
100
|
});
|
|
106
101
|
program
|
|
107
102
|
.command('mock <producer>')
|
|
@@ -113,6 +108,6 @@ program
|
|
|
113
108
|
console.error('Record count must be a positive number');
|
|
114
109
|
process.exit(1);
|
|
115
110
|
}
|
|
116
|
-
|
|
111
|
+
mock(producer, recordCount);
|
|
117
112
|
});
|
|
118
113
|
program.parse(process.argv);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@forzalabs/remora",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.1.1",
|
|
4
4
|
"description": "A powerful CLI tool for seamless data translation.",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"private": false,
|
|
@@ -8,7 +8,6 @@
|
|
|
8
8
|
"remora": "index.js"
|
|
9
9
|
},
|
|
10
10
|
"scripts": {
|
|
11
|
-
"sync": "cd ../dev_ops && npm run sync",
|
|
12
11
|
"dev": "clear && npm run fast-build && clear && npx tsx scripts/dev.ts",
|
|
13
12
|
"dev:w": "clear && npm run fast-build && clear && npx tsx scripts/dev.ts",
|
|
14
13
|
"ts-check": "npx tsc --noemit",
|
|
@@ -20,8 +19,8 @@
|
|
|
20
19
|
"deploy": "npx tsx ./src/index.ts deploy",
|
|
21
20
|
"debug": "npx tsx ./src/index.ts debug",
|
|
22
21
|
"create-producer": "npx tsx ./src/index.ts create-producer",
|
|
23
|
-
"copy-static-file": "npx tsx ./scripts/CopyStaticFile.
|
|
24
|
-
"build": "npm i &&
|
|
22
|
+
"copy-static-file": "npx tsx ./scripts/CopyStaticFile.ts",
|
|
23
|
+
"build": "npm i && rm -rf .build && tsc --outDir .build && npm run copy-static-file",
|
|
25
24
|
"fast-build": "tsc --outDir .build",
|
|
26
25
|
"upload": "npm run build && cd .build && npm publish --access=public"
|
|
27
26
|
},
|
|
@@ -34,6 +33,7 @@
|
|
|
34
33
|
"author": "",
|
|
35
34
|
"license": "BSL",
|
|
36
35
|
"dependencies": {
|
|
36
|
+
"@remora/app-definitions": "1.0.0",
|
|
37
37
|
"@aws-sdk/client-redshift-data": "^3.699.0",
|
|
38
38
|
"@aws-sdk/client-s3": "^3.701.0",
|
|
39
39
|
"@aws-sdk/client-sqs": "^3.886.0",
|
|
@@ -1,34 +1,20 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
-
};
|
|
14
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
-
const workerpool_1 = __importDefault(require("workerpool"));
|
|
16
|
-
const dotenv_1 = __importDefault(require("dotenv"));
|
|
17
|
-
const Affirm_1 = __importDefault(require("../core/Affirm"));
|
|
18
|
-
const Environment_1 = __importDefault(require("../engines/Environment"));
|
|
19
|
-
const Executor_1 = __importDefault(require("../executors/Executor"));
|
|
20
|
-
dotenv_1.default.configDotenv();
|
|
21
|
-
const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
|
|
22
|
-
Environment_1.default.load('./');
|
|
1
|
+
import workerpool from 'workerpool';
|
|
2
|
+
import dotenv from 'dotenv';
|
|
3
|
+
import { affirm } from '@remora/app-core';
|
|
4
|
+
import { Environment } from '@remora/app-common';
|
|
5
|
+
import { Executor } from '@remora/app-executors';
|
|
6
|
+
dotenv.configDotenv();
|
|
7
|
+
const run = async (workerData) => {
|
|
8
|
+
Environment.load('./');
|
|
23
9
|
try {
|
|
24
10
|
const { workerId, chunk, consumer, producer, prodDimensions, scope, options } = workerData;
|
|
25
|
-
(
|
|
26
|
-
(
|
|
27
|
-
(
|
|
28
|
-
(
|
|
29
|
-
(
|
|
30
|
-
const executor = new
|
|
31
|
-
const res =
|
|
11
|
+
affirm(workerId, `Invalid worker id`);
|
|
12
|
+
affirm(consumer, `Invalid consumer`);
|
|
13
|
+
affirm(producer, `Invalid producer`);
|
|
14
|
+
affirm(chunk, `Invalid chunk`);
|
|
15
|
+
affirm(scope, `Invalid executor scope`);
|
|
16
|
+
const executor = new Executor();
|
|
17
|
+
const res = await executor.run({
|
|
32
18
|
consumer,
|
|
33
19
|
producer,
|
|
34
20
|
prodDimensions,
|
|
@@ -36,7 +22,7 @@ const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
|
|
|
36
22
|
chunk,
|
|
37
23
|
options,
|
|
38
24
|
scope,
|
|
39
|
-
reportWork: packet =>
|
|
25
|
+
reportWork: packet => workerpool.workerEmit(packet)
|
|
40
26
|
});
|
|
41
27
|
return res;
|
|
42
28
|
}
|
|
@@ -44,7 +30,7 @@ const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
|
|
|
44
30
|
console.error(error);
|
|
45
31
|
return null;
|
|
46
32
|
}
|
|
47
|
-
}
|
|
48
|
-
|
|
33
|
+
};
|
|
34
|
+
workerpool.worker({
|
|
49
35
|
executor: run
|
|
50
36
|
});
|
package/Constants.js
DELETED
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
const CONSTANTS = {
|
|
4
|
-
cliVersion: '1.0.21',
|
|
5
|
-
backendVersion: 1,
|
|
6
|
-
backendPort: 5088,
|
|
7
|
-
workerVersion: 2,
|
|
8
|
-
workerPort: 5069,
|
|
9
|
-
/**
|
|
10
|
-
* Column name for the dynamically injected source filename dimension.
|
|
11
|
-
* Prefixed with $ to indicate it's a system-generated dynamic value.
|
|
12
|
-
*/
|
|
13
|
-
SOURCE_FILENAME_COLUMN: '$source_filename',
|
|
14
|
-
defaults: {
|
|
15
|
-
REMORA_PATH: './remora',
|
|
16
|
-
PRODUCER_TEMP_FOLDER: '.temp',
|
|
17
|
-
SQL_MAX_QUERY_ROWS: 10000,
|
|
18
|
-
STRING_MAX_CHARACTERS_LENGTH: 10000000,
|
|
19
|
-
MAX_ITEMS_IN_MEMORY: 200000,
|
|
20
|
-
MIN_RUNTIME_HEAP_MB: 4000,
|
|
21
|
-
RECOMMENDED_RUNTIME_HEAP_MB: 8000,
|
|
22
|
-
INDICATIVE_THREAD_LINE_COUNT: 750000,
|
|
23
|
-
MAX_THREAD_COUNT: 8,
|
|
24
|
-
/**
|
|
25
|
-
* Minimum file size to consider parallel processing (10 MB)
|
|
26
|
-
*/
|
|
27
|
-
MIN_FILE_SIZE_FOR_PARALLEL: 10 * 1024 * 1024,
|
|
28
|
-
/**
|
|
29
|
-
* Minimum chunk size per worker to justify overhead (2 MB)
|
|
30
|
-
*/
|
|
31
|
-
MIN_CHUNK_SIZE: 2 * 1024 * 1024
|
|
32
|
-
}
|
|
33
|
-
};
|
|
34
|
-
exports.default = CONSTANTS;
|
package/core/Affirm.js
DELETED
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
/**
|
|
4
|
-
* Checks if 'value' is truthy. If not, throws an exception with custom 'msg' (if present).
|
|
5
|
-
*/
|
|
6
|
-
const affirm = (value, msg) => {
|
|
7
|
-
if (!value)
|
|
8
|
-
throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
|
|
9
|
-
};
|
|
10
|
-
affirm.hasValue = (value, msg) => {
|
|
11
|
-
if (value === null || value === undefined)
|
|
12
|
-
throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
|
|
13
|
-
};
|
|
14
|
-
affirm.hasItems = (value, msg) => {
|
|
15
|
-
if (!value || !Array.isArray(value) || !(value.length > 0))
|
|
16
|
-
throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
|
|
17
|
-
};
|
|
18
|
-
affirm.doesntInclude = (value, array, msg) => {
|
|
19
|
-
affirm(value, msg);
|
|
20
|
-
if (!array || !Array.isArray(array) || array.includes(value))
|
|
21
|
-
throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
|
|
22
|
-
};
|
|
23
|
-
affirm.doesntExist = (item, array, key, msg) => {
|
|
24
|
-
affirm(item, msg);
|
|
25
|
-
if (!array || !Array.isArray(array) || !array.some(arrayItem => arrayItem[key] === item[key]))
|
|
26
|
-
throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
|
|
27
|
-
};
|
|
28
|
-
affirm.noDupes = (array, msg) => {
|
|
29
|
-
if (!array || !Array.isArray(array) || new Set(array).size !== array.length)
|
|
30
|
-
throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
|
|
31
|
-
};
|
|
32
|
-
affirm.noDupedItems = (array, key, msg) => {
|
|
33
|
-
if (!array || !Array.isArray(array))
|
|
34
|
-
throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
|
|
35
|
-
const seen = new Set();
|
|
36
|
-
for (const item of array) {
|
|
37
|
-
if (seen.has(item[key]))
|
|
38
|
-
throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
|
|
39
|
-
seen.add(item[key]);
|
|
40
|
-
}
|
|
41
|
-
};
|
|
42
|
-
exports.default = affirm;
|
package/core/Algo.js
DELETED
|
@@ -1,160 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
const Affirm_1 = __importDefault(require("./Affirm"));
|
|
7
|
-
/**
|
|
8
|
-
* Core Algorithms that are shared between different project/problems.
|
|
9
|
-
* This file needs to be as lean as possible and must not include any external dependencies.
|
|
10
|
-
*/
|
|
11
|
-
const algo = {
|
|
12
|
-
hasVal: (value) => value !== null && value !== undefined,
|
|
13
|
-
sleep: (milliseconds) => {
|
|
14
|
-
(0, Affirm_1.default)(milliseconds > 0, 'Invalid @milliseconds parameter. Must be a positive number.');
|
|
15
|
-
return new Promise(resolve => setTimeout(resolve, milliseconds));
|
|
16
|
-
},
|
|
17
|
-
rng: (min, max) => {
|
|
18
|
-
(0, Affirm_1.default)(min, 'Invalid @min parameter. Must be a valid number.');
|
|
19
|
-
(0, Affirm_1.default)(max, 'Invalid @max parameter. Must be a valid number.');
|
|
20
|
-
(0, Affirm_1.default)(min < max, 'The minimum value must be less than the maximum value.');
|
|
21
|
-
return Math.floor(Math.random() * (max - min + 1)) + min;
|
|
22
|
-
},
|
|
23
|
-
uniqBy: (array, pk) => {
|
|
24
|
-
if (!array || array.length === 0)
|
|
25
|
-
return [];
|
|
26
|
-
return Array.from(new Set(array.map(x => x[pk])));
|
|
27
|
-
},
|
|
28
|
-
uniq: (array) => {
|
|
29
|
-
if (!array || array.length === 0)
|
|
30
|
-
return [];
|
|
31
|
-
return Array.from(new Set(array));
|
|
32
|
-
},
|
|
33
|
-
duplicatesObject: (array, pk) => {
|
|
34
|
-
if (!array || array.length === 0)
|
|
35
|
-
return [];
|
|
36
|
-
return array.filter(x => array.filter(k => k[pk] === x[pk]).length > 1);
|
|
37
|
-
},
|
|
38
|
-
duplicates: (array) => {
|
|
39
|
-
if (!array || array.length === 0)
|
|
40
|
-
return [];
|
|
41
|
-
return array.filter(x => array.filter(k => k === x).length > 1);
|
|
42
|
-
},
|
|
43
|
-
locations: (string, substring) => {
|
|
44
|
-
(0, Affirm_1.default)(string, 'Invalid @string parameter.');
|
|
45
|
-
(0, Affirm_1.default)(substring, 'Invalid @substring parameter.');
|
|
46
|
-
const a = [];
|
|
47
|
-
let i = -1;
|
|
48
|
-
while ((i = string.indexOf(substring, i + 1)) >= 0)
|
|
49
|
-
a.push(i);
|
|
50
|
-
return a;
|
|
51
|
-
},
|
|
52
|
-
isNumber: (value) => typeof value === 'number',
|
|
53
|
-
orderBy: (list, key, direction = 'asc') => {
|
|
54
|
-
(0, Affirm_1.default)(list, 'Invalid array.');
|
|
55
|
-
Affirm_1.default.hasItems(list, 'Array must be non-empty.');
|
|
56
|
-
(0, Affirm_1.default)(direction, 'Invalid direction parameter. Must be "asc" or "desc".');
|
|
57
|
-
const sortedList = [...list];
|
|
58
|
-
if (key) {
|
|
59
|
-
sortedList.sort((a, b) => {
|
|
60
|
-
const aValue = a[key];
|
|
61
|
-
const bValue = b[key];
|
|
62
|
-
if (aValue < bValue)
|
|
63
|
-
return direction === 'asc' ? -1 : 1;
|
|
64
|
-
if (aValue > bValue)
|
|
65
|
-
return direction === 'asc' ? 1 : -1;
|
|
66
|
-
return 0;
|
|
67
|
-
});
|
|
68
|
-
return sortedList;
|
|
69
|
-
}
|
|
70
|
-
else {
|
|
71
|
-
sortedList.sort((a, b) => {
|
|
72
|
-
if (a < b)
|
|
73
|
-
return direction === 'asc' ? -1 : 1;
|
|
74
|
-
if (a > b)
|
|
75
|
-
return direction === 'asc' ? 1 : -1;
|
|
76
|
-
return 0;
|
|
77
|
-
});
|
|
78
|
-
return sortedList;
|
|
79
|
-
}
|
|
80
|
-
},
|
|
81
|
-
groupBy: (array, key) => {
|
|
82
|
-
Affirm_1.default.hasItems(array, 'Array must be non-empty');
|
|
83
|
-
(0, Affirm_1.default)(key, 'key is invalid');
|
|
84
|
-
const result = new Map();
|
|
85
|
-
for (let i = 0; i < array.length; i++) {
|
|
86
|
-
const item = array[i];
|
|
87
|
-
const keyVal = item[key];
|
|
88
|
-
if (result.has(keyVal)) {
|
|
89
|
-
const current = result.get(keyVal);
|
|
90
|
-
current.push(item);
|
|
91
|
-
result.set(keyVal, current);
|
|
92
|
-
}
|
|
93
|
-
else {
|
|
94
|
-
result.set(keyVal, [item]);
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
return result;
|
|
98
|
-
},
|
|
99
|
-
chunkString: (text, chunkSize) => {
|
|
100
|
-
(0, Affirm_1.default)(text, 'Invalid text');
|
|
101
|
-
(0, Affirm_1.default)(chunkSize, 'Invalid chunk size');
|
|
102
|
-
(0, Affirm_1.default)(text.length > 0, 'Text length must be greater than 0');
|
|
103
|
-
(0, Affirm_1.default)(chunkSize > 0, 'Chunk size must be greater than 0');
|
|
104
|
-
if (text.length <= chunkSize)
|
|
105
|
-
return [text];
|
|
106
|
-
const numChunks = Math.ceil(text.length / chunkSize);
|
|
107
|
-
const chunks = new Array(numChunks);
|
|
108
|
-
for (let i = 0; i < chunks.length; i++) {
|
|
109
|
-
chunks[i] = text.substring(i * chunkSize, chunkSize);
|
|
110
|
-
}
|
|
111
|
-
return chunks;
|
|
112
|
-
},
|
|
113
|
-
first: (arr) => {
|
|
114
|
-
(0, Affirm_1.default)(algo.hasVal(arr), 'Array must not be null or undefined');
|
|
115
|
-
Affirm_1.default.hasItems(arr, 'Array must be non-empty');
|
|
116
|
-
return arr[0];
|
|
117
|
-
},
|
|
118
|
-
last: (arr) => {
|
|
119
|
-
(0, Affirm_1.default)(algo.hasVal(arr), 'Array must not be null or undefined');
|
|
120
|
-
(0, Affirm_1.default)(arr.length > 0, 'Array must be non-empty');
|
|
121
|
-
return arr[arr.length - 1];
|
|
122
|
-
},
|
|
123
|
-
mean: (numbers) => {
|
|
124
|
-
(0, Affirm_1.default)(algo.hasVal(numbers), 'Array must not be null or undefined');
|
|
125
|
-
if (numbers.length === 0)
|
|
126
|
-
return 0;
|
|
127
|
-
const total = algo.sum(numbers);
|
|
128
|
-
return total / numbers.length;
|
|
129
|
-
},
|
|
130
|
-
sum: (numbers) => {
|
|
131
|
-
(0, Affirm_1.default)(algo.hasVal(numbers), 'Array must not be null or undefined');
|
|
132
|
-
if (numbers.length === 0)
|
|
133
|
-
return 0;
|
|
134
|
-
let total = 0;
|
|
135
|
-
for (let i = 0; i < numbers.length; i++) {
|
|
136
|
-
total += numbers[i];
|
|
137
|
-
}
|
|
138
|
-
return total;
|
|
139
|
-
},
|
|
140
|
-
round: (number, precision) => {
|
|
141
|
-
(0, Affirm_1.default)(algo.hasVal(number), 'Number must not be null or undefined');
|
|
142
|
-
const factor = Math.pow(10, precision !== null && precision !== void 0 ? precision : 2);
|
|
143
|
-
return Math.round(number * factor) / factor;
|
|
144
|
-
},
|
|
145
|
-
min: (arr) => {
|
|
146
|
-
(0, Affirm_1.default)(algo.hasVal(arr), 'Array must not be null or undefined');
|
|
147
|
-
if (arr.length === 0)
|
|
148
|
-
return 0;
|
|
149
|
-
return Math.min(...arr);
|
|
150
|
-
},
|
|
151
|
-
max: (arr) => {
|
|
152
|
-
(0, Affirm_1.default)(algo.hasVal(arr), 'Array must not be null or undefined');
|
|
153
|
-
if (arr.length === 0)
|
|
154
|
-
return 0;
|
|
155
|
-
return Math.max(...arr);
|
|
156
|
-
},
|
|
157
|
-
replaceAll: (text, search, replace) => text.replace(new RegExp(search, 'g'), replace),
|
|
158
|
-
deepClone: (data) => JSON.parse(JSON.stringify(data))
|
|
159
|
-
};
|
|
160
|
-
exports.default = algo;
|
package/core/dste/DSTE.js
DELETED
|
@@ -1,113 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
-
var ownKeys = function(o) {
|
|
20
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
-
var ar = [];
|
|
22
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
-
return ar;
|
|
24
|
-
};
|
|
25
|
-
return ownKeys(o);
|
|
26
|
-
};
|
|
27
|
-
return function (mod) {
|
|
28
|
-
if (mod && mod.__esModule) return mod;
|
|
29
|
-
var result = {};
|
|
30
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
-
__setModuleDefault(result, mod);
|
|
32
|
-
return result;
|
|
33
|
-
};
|
|
34
|
-
})();
|
|
35
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
36
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
37
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
38
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
39
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
40
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
41
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
42
|
-
});
|
|
43
|
-
};
|
|
44
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
45
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
46
|
-
};
|
|
47
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
48
|
-
const child_process_1 = require("child_process");
|
|
49
|
-
const Affirm_1 = __importDefault(require("../Affirm"));
|
|
50
|
-
const fs = __importStar(require("fs"));
|
|
51
|
-
const path_1 = __importDefault(require("path"));
|
|
52
|
-
/**
|
|
53
|
-
* This class is used to abstract away all calls that are non-deterministic.
|
|
54
|
-
* As of now they just return non-deterministic values, but in the future the implementation can be easily replaced with deterministic code
|
|
55
|
-
*/
|
|
56
|
-
class DeterministicSimulationTestingEngine {
|
|
57
|
-
constructor() {
|
|
58
|
-
this.init = (options) => {
|
|
59
|
-
(0, Affirm_1.default)(options, 'Invalid options');
|
|
60
|
-
this._options = options;
|
|
61
|
-
};
|
|
62
|
-
/**
|
|
63
|
-
* returns a random number between 0 and 1
|
|
64
|
-
*/
|
|
65
|
-
this.random = () => Math.random();
|
|
66
|
-
this.now = () => new Date();
|
|
67
|
-
this.simulate = () => {
|
|
68
|
-
const dir = __dirname;
|
|
69
|
-
const root = path_1.default.join(dir, '..', '..', '..');
|
|
70
|
-
const folders = fs.readdirSync(root);
|
|
71
|
-
const testFolder = folders.find(x => x === '_tests');
|
|
72
|
-
if (testFolder) {
|
|
73
|
-
const baseCommand = `npx tsx ./${testFolder}`;
|
|
74
|
-
const testFiles = fs.readdirSync(testFolder);
|
|
75
|
-
for (let i = 0; i < testFiles.length; i++) {
|
|
76
|
-
const testFile = testFiles[i];
|
|
77
|
-
const command = `${baseCommand}/${testFile}`;
|
|
78
|
-
console.log('starting command', command);
|
|
79
|
-
const tt = (0, child_process_1.execSync)(command);
|
|
80
|
-
console.log('completed command', tt);
|
|
81
|
-
// TODO: I can't get the output back from this execution and not even the console gets logged
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
};
|
|
85
|
-
this.tests = (tests) => __awaiter(this, void 0, void 0, function* () {
|
|
86
|
-
const results = yield Promise.all(tests);
|
|
87
|
-
console.log(results.map(x => `- ${x.message}`).join('\n'));
|
|
88
|
-
const count = results.length;
|
|
89
|
-
const successCount = results.filter(x => x.success).length;
|
|
90
|
-
const failCount = results.filter(x => !x.success).length;
|
|
91
|
-
console.log(`\nTests summary:\n- test count: ${count}\n- successes: \x1b[42m${successCount}\x1b[0m\n- failures: \x1b[41m${failCount}\x1b[0m`);
|
|
92
|
-
});
|
|
93
|
-
this.test = (name, fn) => __awaiter(this, void 0, void 0, function* () {
|
|
94
|
-
try {
|
|
95
|
-
const res = yield Promise.resolve(fn());
|
|
96
|
-
if (!res)
|
|
97
|
-
return { message: `test "${name}"\x1b[41m failed.\x1b[0m`, success: false };
|
|
98
|
-
else
|
|
99
|
-
return { message: `test "${name}"\x1b[42m succeded.\x1b[0m`, success: true };
|
|
100
|
-
}
|
|
101
|
-
catch (error) {
|
|
102
|
-
const myerror = error;
|
|
103
|
-
return { message: `test "${name}"\x1b[41m crashed\x1b[0m with message: ${myerror.message}; trace: ${myerror.stack}`, success: false };
|
|
104
|
-
}
|
|
105
|
-
});
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
/**
|
|
109
|
-
* Deterministic Simulation Testing Engine
|
|
110
|
-
*/
|
|
111
|
-
const DSTE = new DeterministicSimulationTestingEngine();
|
|
112
|
-
DSTE.init({ seed: 0 }); // this is going to bite me in the butt...
|
|
113
|
-
exports.default = DSTE;
|
|
@@ -1,48 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
class DebugLogService {
|
|
4
|
-
constructor() {
|
|
5
|
-
this.id = 'DevelopmentLogService';
|
|
6
|
-
this._folder = 'local_logs';
|
|
7
|
-
this._file = 'logs.txt';
|
|
8
|
-
this._path = this._folder + '/' + this._file;
|
|
9
|
-
this.init = () => {
|
|
10
|
-
return null;
|
|
11
|
-
};
|
|
12
|
-
this.log = (text) => {
|
|
13
|
-
try {
|
|
14
|
-
console.log(text);
|
|
15
|
-
}
|
|
16
|
-
catch (error) {
|
|
17
|
-
console.error('ERROR on log', error);
|
|
18
|
-
}
|
|
19
|
-
finally {
|
|
20
|
-
return null;
|
|
21
|
-
}
|
|
22
|
-
};
|
|
23
|
-
this.info = (text) => {
|
|
24
|
-
try {
|
|
25
|
-
console.info(text);
|
|
26
|
-
}
|
|
27
|
-
catch (error) {
|
|
28
|
-
console.error('ERROR on info', error);
|
|
29
|
-
}
|
|
30
|
-
finally {
|
|
31
|
-
return null;
|
|
32
|
-
}
|
|
33
|
-
};
|
|
34
|
-
this.error = (error) => {
|
|
35
|
-
try {
|
|
36
|
-
console.error(error);
|
|
37
|
-
}
|
|
38
|
-
catch (error) {
|
|
39
|
-
console.error('ERROR on error', error);
|
|
40
|
-
}
|
|
41
|
-
finally {
|
|
42
|
-
return null;
|
|
43
|
-
}
|
|
44
|
-
};
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
const DebugLogger = new DebugLogService();
|
|
48
|
-
exports.default = DebugLogger;
|