@forzalabs/remora 1.0.22 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/actions/automap.js +26 -42
- package/actions/compile.js +27 -43
- package/actions/create_consumer.js +24 -40
- package/actions/create_producer.js +16 -32
- package/actions/debug.js +18 -34
- package/actions/deploy.js +30 -46
- package/actions/discover.js +13 -29
- package/actions/init.js +29 -45
- package/actions/mock.js +16 -32
- package/actions/run.js +34 -52
- package/actions/sample.js +42 -58
- package/index.js +109 -119
- package/package.json +4 -4
- package/workers/ExecutorWorker.js +18 -34
- package/Constants.js +0 -34
- package/core/Affirm.js +0 -42
- package/core/Algo.js +0 -160
- package/core/dste/DSTE.js +0 -113
- package/core/logger/DebugLogService.js +0 -48
- package/core/logger/DevelopmentLogService.js +0 -70
- package/core/logger/LocalLogService.js +0 -70
- package/core/logger/Logger.js +0 -54
- package/database/DatabaseEngine.js +0 -149
- package/database/DatabaseStructure.js +0 -27
- package/definitions/DatasetDefinitions.js +0 -2
- package/definitions/ExecutorDefinitions.js +0 -2
- package/definitions/ProcessENV.js +0 -169
- package/definitions/agents/DestinationDriver.js +0 -2
- package/definitions/agents/SourceDriver.js +0 -2
- package/definitions/cli.js +0 -2
- package/definitions/database/ApiKeys.js +0 -2
- package/definitions/database/Stored.js +0 -7
- package/definitions/database/UsageStat.js +0 -2
- package/definitions/database/User.js +0 -2
- package/definitions/json_schemas/consumer-schema.json +0 -1226
- package/definitions/json_schemas/producer-schema.json +0 -308
- package/definitions/json_schemas/project-schema.json +0 -100
- package/definitions/json_schemas/source-schema.json +0 -249
- package/definitions/requests/ConsumerRequest.js +0 -2
- package/definitions/requests/Developer.js +0 -2
- package/definitions/requests/Mapping.js +0 -2
- package/definitions/requests/ProducerRequest.js +0 -2
- package/definitions/requests/Request.js +0 -2
- package/definitions/resources/Compiled.js +0 -2
- package/definitions/resources/Consumer.js +0 -2
- package/definitions/resources/Environment.js +0 -2
- package/definitions/resources/Library.js +0 -2
- package/definitions/resources/Producer.js +0 -2
- package/definitions/resources/Project.js +0 -2
- package/definitions/resources/Schema.js +0 -2
- package/definitions/resources/Source.js +0 -2
- package/definitions/temp.js +0 -2
- package/definitions/transform/Transformations.js +0 -2
- package/drivers/DeltaShareDriver.js +0 -186
- package/drivers/DriverFactory.js +0 -72
- package/drivers/DriverHelper.js +0 -248
- package/drivers/HttpApiDriver.js +0 -208
- package/drivers/RedshiftDriver.js +0 -184
- package/drivers/files/LocalDestinationDriver.js +0 -146
- package/drivers/files/LocalSourceDriver.js +0 -405
- package/drivers/s3/S3DestinationDriver.js +0 -197
- package/drivers/s3/S3SourceDriver.js +0 -495
- package/engines/CryptoEngine.js +0 -75
- package/engines/Environment.js +0 -170
- package/engines/ProcessENVManager.js +0 -112
- package/engines/RandomEngine.js +0 -47
- package/engines/SecretManager.js +0 -23
- package/engines/UserManager.js +0 -66
- package/engines/ai/AutoMapperEngine.js +0 -37
- package/engines/ai/DeveloperEngine.js +0 -497
- package/engines/ai/LLM.js +0 -255
- package/engines/consumer/ConsumerManager.js +0 -218
- package/engines/consumer/ConsumerOnFinishManager.js +0 -202
- package/engines/dataset/Dataset.js +0 -824
- package/engines/dataset/DatasetManager.js +0 -211
- package/engines/dataset/DatasetRecord.js +0 -120
- package/engines/dataset/DatasetRecordPool.js +0 -77
- package/engines/execution/RequestExecutor.js +0 -67
- package/engines/parsing/CSVParser.js +0 -60
- package/engines/parsing/LineParser.js +0 -71
- package/engines/parsing/ParseCompression.js +0 -101
- package/engines/parsing/ParseHelper.js +0 -18
- package/engines/parsing/ParseManager.js +0 -54
- package/engines/parsing/XLSParser.js +0 -87
- package/engines/parsing/XMLParser.js +0 -115
- package/engines/producer/ProducerEngine.js +0 -127
- package/engines/producer/ProducerManager.js +0 -43
- package/engines/scheduler/CronScheduler.js +0 -222
- package/engines/scheduler/QueueManager.js +0 -314
- package/engines/schema/SchemaValidator.js +0 -67
- package/engines/transform/JoinEngine.js +0 -232
- package/engines/transform/TransformationEngine.js +0 -277
- package/engines/transform/TypeCaster.js +0 -59
- package/engines/usage/DataframeManager.js +0 -55
- package/engines/usage/UsageDataManager.js +0 -151
- package/engines/usage/UsageManager.js +0 -65
- package/engines/validation/Validator.js +0 -216
- package/executors/ConsumerExecutor.js +0 -280
- package/executors/Executor.js +0 -177
- package/executors/ExecutorOrchestrator.js +0 -331
- package/executors/ExecutorPerformance.js +0 -17
- package/executors/ExecutorProgress.js +0 -54
- package/executors/ExecutorScope.js +0 -52
- package/executors/OutputExecutor.js +0 -118
- package/executors/ProducerExecutor.js +0 -108
- package/helper/Helper.js +0 -149
- package/helper/Logger.js +0 -84
- package/helper/Runtime.js +0 -20
- package/helper/Settings.js +0 -13
- package/licencing/LicenceManager.js +0 -64
- package/settings.js +0 -12
package/index.js
CHANGED
|
@@ -1,123 +1,113 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
};
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
const
|
|
21
|
-
|
|
22
|
-
const
|
|
23
|
-
const
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
.
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
2
|
+
import { Command } from 'commander';
|
|
3
|
+
import dotenv from 'dotenv';
|
|
4
|
+
import { compile } from './actions/compile';
|
|
5
|
+
import { debug } from './actions/debug';
|
|
6
|
+
import { deploy } from './actions/deploy';
|
|
7
|
+
import { init } from './actions/init';
|
|
8
|
+
import { run } from './actions/run';
|
|
9
|
+
import { discover } from './actions/discover';
|
|
10
|
+
import { create_producer } from './actions/create_producer';
|
|
11
|
+
import { create_consumer } from './actions/create_consumer';
|
|
12
|
+
import { CONSTANTS } from '@remora/app-constants';
|
|
13
|
+
import { LicenceManager } from '@remora/app-auth';
|
|
14
|
+
import { Runtime } from '@remora/app-helper';
|
|
15
|
+
import { automap } from './actions/automap';
|
|
16
|
+
import { sample } from './actions/sample';
|
|
17
|
+
import { mock } from './actions/mock';
|
|
18
|
+
import { ProcessENVManager } from '@remora/app-common';
|
|
19
|
+
dotenv.configDotenv();
|
|
20
|
+
const program = new Command();
|
|
21
|
+
// Validate the remora licence
|
|
22
|
+
const remoraLicenceKey = ProcessENVManager.getEnvVariable('REMORA_LICENCE_KEY');
|
|
23
|
+
const check = LicenceManager.validate(remoraLicenceKey);
|
|
24
|
+
if (!check.valid) {
|
|
25
|
+
console.error(`Invalid Remora licence key, the product is not active: remember to set "REMORA_LICENCE_KEY" environment variable.`);
|
|
26
|
+
process.exit(1);
|
|
27
|
+
}
|
|
28
|
+
// Runtime check on heap size to warn user of insufficent runtime resources
|
|
29
|
+
const { heapSizeMB } = Runtime.getHeap();
|
|
30
|
+
if (heapSizeMB < CONSTANTS.defaults.MIN_RUNTIME_HEAP_MB)
|
|
31
|
+
console.warn(`Remora is running with ${heapSizeMB}MB of runtime heap, which is below the bare minimum of ${CONSTANTS.defaults.MIN_RUNTIME_HEAP_MB}MB (Recommended: ${CONSTANTS.defaults.RECOMMENDED_RUNTIME_HEAP_MB}MB).`);
|
|
32
|
+
else if (heapSizeMB < CONSTANTS.defaults.RECOMMENDED_RUNTIME_HEAP_MB)
|
|
33
|
+
console.warn(`Remora is running with ${heapSizeMB} MB of runtime heap, which is below the recommended of ${CONSTANTS.defaults.RECOMMENDED_RUNTIME_HEAP_MB} MB.`);
|
|
34
|
+
// Initialize all commands
|
|
35
|
+
program
|
|
36
|
+
.version(CONSTANTS.cliVersion + '', '-v, --version', 'Display the version of the CLI')
|
|
37
|
+
.description('CLI tool for setting up and managing Data-Remora');
|
|
38
|
+
program
|
|
39
|
+
.command('version')
|
|
40
|
+
.description('Display the version of the CLI')
|
|
41
|
+
.action(() => {
|
|
42
|
+
const packageJson = require('../package.json');
|
|
43
|
+
console.log(`Data-Remora CLI version: ${packageJson.version}`);
|
|
44
|
+
});
|
|
45
|
+
program
|
|
46
|
+
.command('init')
|
|
47
|
+
.description('Initialize the application and set up configuration')
|
|
48
|
+
.action(init);
|
|
49
|
+
program
|
|
50
|
+
.command('compile')
|
|
51
|
+
.description('Compile the config and check for errors')
|
|
52
|
+
.action(compile);
|
|
53
|
+
program
|
|
54
|
+
.command('debug')
|
|
55
|
+
.description('Check the connection with sources and producers')
|
|
56
|
+
.action(debug);
|
|
57
|
+
program
|
|
58
|
+
.command('deploy')
|
|
59
|
+
.description('Deploy the application to the specified environment')
|
|
60
|
+
.option('-e, --env <environment>', 'Target environment (staging, production)', 'staging')
|
|
61
|
+
.option('--skip-tests', 'Skip running tests before deployment')
|
|
62
|
+
.option('--build-only', 'Build without deploying')
|
|
63
|
+
.action(deploy);
|
|
64
|
+
program
|
|
65
|
+
.command('run [name]')
|
|
66
|
+
.description('Execute consumers. Optionally specify a single consumer name or use --project to run all consumers in a project.')
|
|
67
|
+
.option('-p, --project <name>', 'Run all consumers belonging to the specified project')
|
|
68
|
+
.action((name, options) => run(name, options));
|
|
69
|
+
program
|
|
70
|
+
.command('discover')
|
|
71
|
+
.description('Discover the data shape of a producer and automatically create the resource for it.')
|
|
72
|
+
.argument('<producer>', 'The producer to discover')
|
|
73
|
+
.action(discover);
|
|
74
|
+
program
|
|
75
|
+
.command('create-producer <name>')
|
|
76
|
+
.description('Create a new producer configuration with default settings')
|
|
77
|
+
.action(create_producer);
|
|
78
|
+
program
|
|
79
|
+
.command('automap')
|
|
80
|
+
.description('Automatically map a producer to consumers using specified schemas.')
|
|
81
|
+
.argument('<producer>', 'The producer to analyze')
|
|
82
|
+
.argument('<schemas...>', 'One or more schema names to map against')
|
|
83
|
+
.action(automap);
|
|
84
|
+
program
|
|
85
|
+
.command('create-consumer <name>')
|
|
86
|
+
.description('Create a new consumer configuration with default settings')
|
|
87
|
+
.option('-p, --producer <name>', 'Producer to create a one-to-one mapping from')
|
|
88
|
+
.action((name, options) => create_consumer(name, options.producer));
|
|
89
|
+
program
|
|
90
|
+
.command('sample <name>')
|
|
91
|
+
.description('Sample data from a producer or consumer and display it in a formatted table')
|
|
92
|
+
.option('-s, --size <number>', 'Number of sample rows to display (default: 10)', '10')
|
|
93
|
+
.action((name, options) => {
|
|
94
|
+
const sampleSize = parseInt(options.size, 10);
|
|
95
|
+
if (isNaN(sampleSize) || sampleSize <= 0) {
|
|
96
|
+
console.error('Sample size must be a positive number');
|
|
97
|
+
process.exit(1);
|
|
98
|
+
}
|
|
99
|
+
sample(name, sampleSize);
|
|
100
|
+
});
|
|
101
|
+
program
|
|
102
|
+
.command('mock <producer>')
|
|
103
|
+
.description('Generate mock data for a producer based on its dimensions')
|
|
104
|
+
.argument('<records>', 'Number of mock records to generate')
|
|
105
|
+
.action((producer, records) => {
|
|
106
|
+
const recordCount = parseInt(records, 10);
|
|
107
|
+
if (isNaN(recordCount) || recordCount <= 0) {
|
|
108
|
+
console.error('Record count must be a positive number');
|
|
32
109
|
process.exit(1);
|
|
33
110
|
}
|
|
34
|
-
|
|
35
|
-
const { heapSizeMB } = Runtime_1.default.getHeap();
|
|
36
|
-
if (heapSizeMB < Constants_1.default.defaults.MIN_RUNTIME_HEAP_MB)
|
|
37
|
-
console.warn(`Remora is running with ${heapSizeMB}MB of runtime heap, which is below the bare minimum of ${Constants_1.default.defaults.MIN_RUNTIME_HEAP_MB}MB (Recommended: ${Constants_1.default.defaults.RECOMMENDED_RUNTIME_HEAP_MB}MB).`);
|
|
38
|
-
else if (heapSizeMB < Constants_1.default.defaults.RECOMMENDED_RUNTIME_HEAP_MB)
|
|
39
|
-
console.warn(`Remora is running with ${heapSizeMB} MB of runtime heap, which is below the recommended of ${Constants_1.default.defaults.RECOMMENDED_RUNTIME_HEAP_MB} MB.`);
|
|
40
|
-
})
|
|
41
|
-
.then(() => {
|
|
42
|
-
// Initialize all commands
|
|
43
|
-
const program = new commander_1.Command();
|
|
44
|
-
program
|
|
45
|
-
.version(Constants_1.default.cliVersion + '', '-v, --version', 'Display the version of the CLI')
|
|
46
|
-
.description('CLI tool for setting up and managing Data-Remora');
|
|
47
|
-
program
|
|
48
|
-
.command('version')
|
|
49
|
-
.description('Display the version of the CLI')
|
|
50
|
-
.action(() => {
|
|
51
|
-
const packageJson = require('../package.json');
|
|
52
|
-
console.log(`Data-Remora CLI version: ${packageJson.version}`);
|
|
53
|
-
});
|
|
54
|
-
program
|
|
55
|
-
.command('init')
|
|
56
|
-
.description('Initialize the application and set up configuration')
|
|
57
|
-
.action(init_1.init);
|
|
58
|
-
program
|
|
59
|
-
.command('compile')
|
|
60
|
-
.description('Compile the config and check for errors')
|
|
61
|
-
.action(compile_1.compile);
|
|
62
|
-
program
|
|
63
|
-
.command('debug')
|
|
64
|
-
.description('Check the connection with sources and producers')
|
|
65
|
-
.action(debug_1.debug);
|
|
66
|
-
program
|
|
67
|
-
.command('deploy')
|
|
68
|
-
.description('Deploy the application to the specified environment')
|
|
69
|
-
.option('-e, --env <environment>', 'Target environment (staging, production)', 'staging')
|
|
70
|
-
.option('--skip-tests', 'Skip running tests before deployment')
|
|
71
|
-
.option('--build-only', 'Build without deploying')
|
|
72
|
-
.action(deploy_1.deploy);
|
|
73
|
-
program
|
|
74
|
-
.command('run [name]')
|
|
75
|
-
.description('Execute consumers. Optionally specify a single consumer name or use --project to run all consumers in a project.')
|
|
76
|
-
.option('-p, --project <name>', 'Run all consumers belonging to the specified project')
|
|
77
|
-
.action((name, options) => (0, run_1.run)(name, options));
|
|
78
|
-
program
|
|
79
|
-
.command('discover')
|
|
80
|
-
.description('Discover the data shape of a producer and automatically create the resource for it.')
|
|
81
|
-
.argument('<producer>', 'The producer to discover')
|
|
82
|
-
.action(discover_1.discover);
|
|
83
|
-
program
|
|
84
|
-
.command('create-producer <name>')
|
|
85
|
-
.description('Create a new producer configuration with default settings')
|
|
86
|
-
.action(create_producer_1.create_producer);
|
|
87
|
-
program
|
|
88
|
-
.command('automap')
|
|
89
|
-
.description('Automatically map a producer to consumers using specified schemas.')
|
|
90
|
-
.argument('<producer>', 'The producer to analyze')
|
|
91
|
-
.argument('<schemas...>', 'One or more schema names to map against')
|
|
92
|
-
.action(automap_1.automap);
|
|
93
|
-
program
|
|
94
|
-
.command('create-consumer <name>')
|
|
95
|
-
.description('Create a new consumer configuration with default settings')
|
|
96
|
-
.option('-p, --producer <name>', 'Producer to create a one-to-one mapping from')
|
|
97
|
-
.action((name, options) => (0, create_consumer_1.create_consumer)(name, options.producer));
|
|
98
|
-
program
|
|
99
|
-
.command('sample <name>')
|
|
100
|
-
.description('Sample data from a producer or consumer and display it in a formatted table')
|
|
101
|
-
.option('-s, --size <number>', 'Number of sample rows to display (default: 10)', '10')
|
|
102
|
-
.action((name, options) => {
|
|
103
|
-
const sampleSize = parseInt(options.size, 10);
|
|
104
|
-
if (isNaN(sampleSize) || sampleSize <= 0) {
|
|
105
|
-
console.error('Sample size must be a positive number');
|
|
106
|
-
process.exit(1);
|
|
107
|
-
}
|
|
108
|
-
(0, sample_1.sample)(name, sampleSize);
|
|
109
|
-
});
|
|
110
|
-
program
|
|
111
|
-
.command('mock <producer>')
|
|
112
|
-
.description('Generate mock data for a producer based on its dimensions')
|
|
113
|
-
.argument('<records>', 'Number of mock records to generate')
|
|
114
|
-
.action((producer, records) => {
|
|
115
|
-
const recordCount = parseInt(records, 10);
|
|
116
|
-
if (isNaN(recordCount) || recordCount <= 0) {
|
|
117
|
-
console.error('Record count must be a positive number');
|
|
118
|
-
process.exit(1);
|
|
119
|
-
}
|
|
120
|
-
(0, mock_1.mock)(producer, recordCount);
|
|
121
|
-
});
|
|
122
|
-
program.parse(process.argv);
|
|
111
|
+
mock(producer, recordCount);
|
|
123
112
|
});
|
|
113
|
+
program.parse(process.argv);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@forzalabs/remora",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.1.1",
|
|
4
4
|
"description": "A powerful CLI tool for seamless data translation.",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"private": false,
|
|
@@ -8,7 +8,6 @@
|
|
|
8
8
|
"remora": "index.js"
|
|
9
9
|
},
|
|
10
10
|
"scripts": {
|
|
11
|
-
"sync": "cd ../dev_ops && npm run sync",
|
|
12
11
|
"dev": "clear && npm run fast-build && clear && npx tsx scripts/dev.ts",
|
|
13
12
|
"dev:w": "clear && npm run fast-build && clear && npx tsx scripts/dev.ts",
|
|
14
13
|
"ts-check": "npx tsc --noemit",
|
|
@@ -20,8 +19,8 @@
|
|
|
20
19
|
"deploy": "npx tsx ./src/index.ts deploy",
|
|
21
20
|
"debug": "npx tsx ./src/index.ts debug",
|
|
22
21
|
"create-producer": "npx tsx ./src/index.ts create-producer",
|
|
23
|
-
"copy-static-file": "npx tsx ./scripts/CopyStaticFile.
|
|
24
|
-
"build": "npm i &&
|
|
22
|
+
"copy-static-file": "npx tsx ./scripts/CopyStaticFile.ts",
|
|
23
|
+
"build": "npm i && rm -rf .build && tsc --outDir .build && npm run copy-static-file",
|
|
25
24
|
"fast-build": "tsc --outDir .build",
|
|
26
25
|
"upload": "npm run build && cd .build && npm publish --access=public"
|
|
27
26
|
},
|
|
@@ -34,6 +33,7 @@
|
|
|
34
33
|
"author": "",
|
|
35
34
|
"license": "BSL",
|
|
36
35
|
"dependencies": {
|
|
36
|
+
"@remora/app-definitions": "1.0.0",
|
|
37
37
|
"@aws-sdk/client-redshift-data": "^3.699.0",
|
|
38
38
|
"@aws-sdk/client-s3": "^3.701.0",
|
|
39
39
|
"@aws-sdk/client-sqs": "^3.886.0",
|
|
@@ -1,36 +1,20 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
-
};
|
|
14
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
-
const workerpool_1 = __importDefault(require("workerpool"));
|
|
16
|
-
const dotenv_1 = __importDefault(require("dotenv"));
|
|
17
|
-
const Affirm_1 = __importDefault(require("../core/Affirm"));
|
|
18
|
-
const Environment_1 = __importDefault(require("../engines/Environment"));
|
|
19
|
-
const Executor_1 = __importDefault(require("../executors/Executor"));
|
|
20
|
-
const ProcessENVManager_1 = __importDefault(require("../engines/ProcessENVManager"));
|
|
21
|
-
dotenv_1.default.configDotenv();
|
|
22
|
-
const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
|
|
23
|
-
yield ProcessENVManager_1.default.init();
|
|
24
|
-
Environment_1.default.load('./');
|
|
1
|
+
import workerpool from 'workerpool';
|
|
2
|
+
import dotenv from 'dotenv';
|
|
3
|
+
import { affirm } from '@remora/app-core';
|
|
4
|
+
import { Environment } from '@remora/app-common';
|
|
5
|
+
import { Executor } from '@remora/app-executors';
|
|
6
|
+
dotenv.configDotenv();
|
|
7
|
+
const run = async (workerData) => {
|
|
8
|
+
Environment.load('./');
|
|
25
9
|
try {
|
|
26
10
|
const { workerId, chunk, consumer, producer, prodDimensions, scope, options } = workerData;
|
|
27
|
-
(
|
|
28
|
-
(
|
|
29
|
-
(
|
|
30
|
-
(
|
|
31
|
-
(
|
|
32
|
-
const executor = new
|
|
33
|
-
const res =
|
|
11
|
+
affirm(workerId, `Invalid worker id`);
|
|
12
|
+
affirm(consumer, `Invalid consumer`);
|
|
13
|
+
affirm(producer, `Invalid producer`);
|
|
14
|
+
affirm(chunk, `Invalid chunk`);
|
|
15
|
+
affirm(scope, `Invalid executor scope`);
|
|
16
|
+
const executor = new Executor();
|
|
17
|
+
const res = await executor.run({
|
|
34
18
|
consumer,
|
|
35
19
|
producer,
|
|
36
20
|
prodDimensions,
|
|
@@ -38,7 +22,7 @@ const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
|
|
|
38
22
|
chunk,
|
|
39
23
|
options,
|
|
40
24
|
scope,
|
|
41
|
-
reportWork: packet =>
|
|
25
|
+
reportWork: packet => workerpool.workerEmit(packet)
|
|
42
26
|
});
|
|
43
27
|
return res;
|
|
44
28
|
}
|
|
@@ -46,7 +30,7 @@ const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
|
|
|
46
30
|
console.error(error);
|
|
47
31
|
return null;
|
|
48
32
|
}
|
|
49
|
-
}
|
|
50
|
-
|
|
33
|
+
};
|
|
34
|
+
workerpool.worker({
|
|
51
35
|
executor: run
|
|
52
36
|
});
|
package/Constants.js
DELETED
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
const CONSTANTS = {
|
|
4
|
-
cliVersion: '1.0.22',
|
|
5
|
-
backendVersion: 1,
|
|
6
|
-
backendPort: 5088,
|
|
7
|
-
workerVersion: 2,
|
|
8
|
-
workerPort: 5069,
|
|
9
|
-
/**
|
|
10
|
-
* Column name for the dynamically injected source filename dimension.
|
|
11
|
-
* Prefixed with $ to indicate it's a system-generated dynamic value.
|
|
12
|
-
*/
|
|
13
|
-
SOURCE_FILENAME_COLUMN: '$source_filename',
|
|
14
|
-
defaults: {
|
|
15
|
-
REMORA_PATH: './remora',
|
|
16
|
-
PRODUCER_TEMP_FOLDER: '.temp',
|
|
17
|
-
SQL_MAX_QUERY_ROWS: 10000,
|
|
18
|
-
STRING_MAX_CHARACTERS_LENGTH: 10000000,
|
|
19
|
-
MAX_ITEMS_IN_MEMORY: 200000,
|
|
20
|
-
MIN_RUNTIME_HEAP_MB: 4000,
|
|
21
|
-
RECOMMENDED_RUNTIME_HEAP_MB: 8000,
|
|
22
|
-
INDICATIVE_THREAD_LINE_COUNT: 750000,
|
|
23
|
-
MAX_THREAD_COUNT: 8,
|
|
24
|
-
/**
|
|
25
|
-
* Minimum file size to consider parallel processing (10 MB)
|
|
26
|
-
*/
|
|
27
|
-
MIN_FILE_SIZE_FOR_PARALLEL: 10 * 1024 * 1024,
|
|
28
|
-
/**
|
|
29
|
-
* Minimum chunk size per worker to justify overhead (2 MB)
|
|
30
|
-
*/
|
|
31
|
-
MIN_CHUNK_SIZE: 2 * 1024 * 1024
|
|
32
|
-
}
|
|
33
|
-
};
|
|
34
|
-
exports.default = CONSTANTS;
|
package/core/Affirm.js
DELETED
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
/**
|
|
4
|
-
* Checks if 'value' is truthy. If not, throws an exception with custom 'msg' (if present).
|
|
5
|
-
*/
|
|
6
|
-
const affirm = (value, msg) => {
|
|
7
|
-
if (!value)
|
|
8
|
-
throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
|
|
9
|
-
};
|
|
10
|
-
affirm.hasValue = (value, msg) => {
|
|
11
|
-
if (value === null || value === undefined)
|
|
12
|
-
throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
|
|
13
|
-
};
|
|
14
|
-
affirm.hasItems = (value, msg) => {
|
|
15
|
-
if (!value || !Array.isArray(value) || !(value.length > 0))
|
|
16
|
-
throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
|
|
17
|
-
};
|
|
18
|
-
affirm.doesntInclude = (value, array, msg) => {
|
|
19
|
-
affirm(value, msg);
|
|
20
|
-
if (!array || !Array.isArray(array) || array.includes(value))
|
|
21
|
-
throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
|
|
22
|
-
};
|
|
23
|
-
affirm.doesntExist = (item, array, key, msg) => {
|
|
24
|
-
affirm(item, msg);
|
|
25
|
-
if (!array || !Array.isArray(array) || !array.some(arrayItem => arrayItem[key] === item[key]))
|
|
26
|
-
throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
|
|
27
|
-
};
|
|
28
|
-
affirm.noDupes = (array, msg) => {
|
|
29
|
-
if (!array || !Array.isArray(array) || new Set(array).size !== array.length)
|
|
30
|
-
throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
|
|
31
|
-
};
|
|
32
|
-
affirm.noDupedItems = (array, key, msg) => {
|
|
33
|
-
if (!array || !Array.isArray(array))
|
|
34
|
-
throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
|
|
35
|
-
const seen = new Set();
|
|
36
|
-
for (const item of array) {
|
|
37
|
-
if (seen.has(item[key]))
|
|
38
|
-
throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
|
|
39
|
-
seen.add(item[key]);
|
|
40
|
-
}
|
|
41
|
-
};
|
|
42
|
-
exports.default = affirm;
|
package/core/Algo.js
DELETED
|
@@ -1,160 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
const Affirm_1 = __importDefault(require("./Affirm"));
|
|
7
|
-
/**
|
|
8
|
-
* Core Algorithms that are shared between different project/problems.
|
|
9
|
-
* This file needs to be as lean as possible and must not include any external dependencies.
|
|
10
|
-
*/
|
|
11
|
-
const algo = {
|
|
12
|
-
hasVal: (value) => value !== null && value !== undefined,
|
|
13
|
-
sleep: (milliseconds) => {
|
|
14
|
-
(0, Affirm_1.default)(milliseconds > 0, 'Invalid @milliseconds parameter. Must be a positive number.');
|
|
15
|
-
return new Promise(resolve => setTimeout(resolve, milliseconds));
|
|
16
|
-
},
|
|
17
|
-
rng: (min, max) => {
|
|
18
|
-
(0, Affirm_1.default)(min, 'Invalid @min parameter. Must be a valid number.');
|
|
19
|
-
(0, Affirm_1.default)(max, 'Invalid @max parameter. Must be a valid number.');
|
|
20
|
-
(0, Affirm_1.default)(min < max, 'The minimum value must be less than the maximum value.');
|
|
21
|
-
return Math.floor(Math.random() * (max - min + 1)) + min;
|
|
22
|
-
},
|
|
23
|
-
uniqBy: (array, pk) => {
|
|
24
|
-
if (!array || array.length === 0)
|
|
25
|
-
return [];
|
|
26
|
-
return Array.from(new Set(array.map(x => x[pk])));
|
|
27
|
-
},
|
|
28
|
-
uniq: (array) => {
|
|
29
|
-
if (!array || array.length === 0)
|
|
30
|
-
return [];
|
|
31
|
-
return Array.from(new Set(array));
|
|
32
|
-
},
|
|
33
|
-
duplicatesObject: (array, pk) => {
|
|
34
|
-
if (!array || array.length === 0)
|
|
35
|
-
return [];
|
|
36
|
-
return array.filter(x => array.filter(k => k[pk] === x[pk]).length > 1);
|
|
37
|
-
},
|
|
38
|
-
duplicates: (array) => {
|
|
39
|
-
if (!array || array.length === 0)
|
|
40
|
-
return [];
|
|
41
|
-
return array.filter(x => array.filter(k => k === x).length > 1);
|
|
42
|
-
},
|
|
43
|
-
locations: (string, substring) => {
|
|
44
|
-
(0, Affirm_1.default)(string, 'Invalid @string parameter.');
|
|
45
|
-
(0, Affirm_1.default)(substring, 'Invalid @substring parameter.');
|
|
46
|
-
const a = [];
|
|
47
|
-
let i = -1;
|
|
48
|
-
while ((i = string.indexOf(substring, i + 1)) >= 0)
|
|
49
|
-
a.push(i);
|
|
50
|
-
return a;
|
|
51
|
-
},
|
|
52
|
-
isNumber: (value) => typeof value === 'number',
|
|
53
|
-
orderBy: (list, key, direction = 'asc') => {
|
|
54
|
-
(0, Affirm_1.default)(list, 'Invalid array.');
|
|
55
|
-
Affirm_1.default.hasItems(list, 'Array must be non-empty.');
|
|
56
|
-
(0, Affirm_1.default)(direction, 'Invalid direction parameter. Must be "asc" or "desc".');
|
|
57
|
-
const sortedList = [...list];
|
|
58
|
-
if (key) {
|
|
59
|
-
sortedList.sort((a, b) => {
|
|
60
|
-
const aValue = a[key];
|
|
61
|
-
const bValue = b[key];
|
|
62
|
-
if (aValue < bValue)
|
|
63
|
-
return direction === 'asc' ? -1 : 1;
|
|
64
|
-
if (aValue > bValue)
|
|
65
|
-
return direction === 'asc' ? 1 : -1;
|
|
66
|
-
return 0;
|
|
67
|
-
});
|
|
68
|
-
return sortedList;
|
|
69
|
-
}
|
|
70
|
-
else {
|
|
71
|
-
sortedList.sort((a, b) => {
|
|
72
|
-
if (a < b)
|
|
73
|
-
return direction === 'asc' ? -1 : 1;
|
|
74
|
-
if (a > b)
|
|
75
|
-
return direction === 'asc' ? 1 : -1;
|
|
76
|
-
return 0;
|
|
77
|
-
});
|
|
78
|
-
return sortedList;
|
|
79
|
-
}
|
|
80
|
-
},
|
|
81
|
-
groupBy: (array, key) => {
|
|
82
|
-
Affirm_1.default.hasItems(array, 'Array must be non-empty');
|
|
83
|
-
(0, Affirm_1.default)(key, 'key is invalid');
|
|
84
|
-
const result = new Map();
|
|
85
|
-
for (let i = 0; i < array.length; i++) {
|
|
86
|
-
const item = array[i];
|
|
87
|
-
const keyVal = item[key];
|
|
88
|
-
if (result.has(keyVal)) {
|
|
89
|
-
const current = result.get(keyVal);
|
|
90
|
-
current.push(item);
|
|
91
|
-
result.set(keyVal, current);
|
|
92
|
-
}
|
|
93
|
-
else {
|
|
94
|
-
result.set(keyVal, [item]);
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
return result;
|
|
98
|
-
},
|
|
99
|
-
chunkString: (text, chunkSize) => {
|
|
100
|
-
(0, Affirm_1.default)(text, 'Invalid text');
|
|
101
|
-
(0, Affirm_1.default)(chunkSize, 'Invalid chunk size');
|
|
102
|
-
(0, Affirm_1.default)(text.length > 0, 'Text length must be greater than 0');
|
|
103
|
-
(0, Affirm_1.default)(chunkSize > 0, 'Chunk size must be greater than 0');
|
|
104
|
-
if (text.length <= chunkSize)
|
|
105
|
-
return [text];
|
|
106
|
-
const numChunks = Math.ceil(text.length / chunkSize);
|
|
107
|
-
const chunks = new Array(numChunks);
|
|
108
|
-
for (let i = 0; i < chunks.length; i++) {
|
|
109
|
-
chunks[i] = text.substring(i * chunkSize, chunkSize);
|
|
110
|
-
}
|
|
111
|
-
return chunks;
|
|
112
|
-
},
|
|
113
|
-
first: (arr) => {
|
|
114
|
-
(0, Affirm_1.default)(algo.hasVal(arr), 'Array must not be null or undefined');
|
|
115
|
-
Affirm_1.default.hasItems(arr, 'Array must be non-empty');
|
|
116
|
-
return arr[0];
|
|
117
|
-
},
|
|
118
|
-
last: (arr) => {
|
|
119
|
-
(0, Affirm_1.default)(algo.hasVal(arr), 'Array must not be null or undefined');
|
|
120
|
-
(0, Affirm_1.default)(arr.length > 0, 'Array must be non-empty');
|
|
121
|
-
return arr[arr.length - 1];
|
|
122
|
-
},
|
|
123
|
-
mean: (numbers) => {
|
|
124
|
-
(0, Affirm_1.default)(algo.hasVal(numbers), 'Array must not be null or undefined');
|
|
125
|
-
if (numbers.length === 0)
|
|
126
|
-
return 0;
|
|
127
|
-
const total = algo.sum(numbers);
|
|
128
|
-
return total / numbers.length;
|
|
129
|
-
},
|
|
130
|
-
sum: (numbers) => {
|
|
131
|
-
(0, Affirm_1.default)(algo.hasVal(numbers), 'Array must not be null or undefined');
|
|
132
|
-
if (numbers.length === 0)
|
|
133
|
-
return 0;
|
|
134
|
-
let total = 0;
|
|
135
|
-
for (let i = 0; i < numbers.length; i++) {
|
|
136
|
-
total += numbers[i];
|
|
137
|
-
}
|
|
138
|
-
return total;
|
|
139
|
-
},
|
|
140
|
-
round: (number, precision) => {
|
|
141
|
-
(0, Affirm_1.default)(algo.hasVal(number), 'Number must not be null or undefined');
|
|
142
|
-
const factor = Math.pow(10, precision !== null && precision !== void 0 ? precision : 2);
|
|
143
|
-
return Math.round(number * factor) / factor;
|
|
144
|
-
},
|
|
145
|
-
min: (arr) => {
|
|
146
|
-
(0, Affirm_1.default)(algo.hasVal(arr), 'Array must not be null or undefined');
|
|
147
|
-
if (arr.length === 0)
|
|
148
|
-
return 0;
|
|
149
|
-
return Math.min(...arr);
|
|
150
|
-
},
|
|
151
|
-
max: (arr) => {
|
|
152
|
-
(0, Affirm_1.default)(algo.hasVal(arr), 'Array must not be null or undefined');
|
|
153
|
-
if (arr.length === 0)
|
|
154
|
-
return 0;
|
|
155
|
-
return Math.max(...arr);
|
|
156
|
-
},
|
|
157
|
-
replaceAll: (text, search, replace) => text.replace(new RegExp(search, 'g'), replace),
|
|
158
|
-
deepClone: (data) => JSON.parse(JSON.stringify(data))
|
|
159
|
-
};
|
|
160
|
-
exports.default = algo;
|