@forzalabs/remora 1.0.22 → 1.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/actions/automap.js +26 -42
  2. package/actions/compile.js +27 -43
  3. package/actions/create_consumer.js +24 -40
  4. package/actions/create_producer.js +16 -32
  5. package/actions/debug.js +18 -34
  6. package/actions/deploy.js +30 -46
  7. package/actions/discover.js +13 -29
  8. package/actions/init.js +29 -45
  9. package/actions/mock.js +16 -32
  10. package/actions/run.js +34 -52
  11. package/actions/sample.js +42 -58
  12. package/index.js +109 -119
  13. package/package.json +4 -4
  14. package/workers/ExecutorWorker.js +18 -34
  15. package/Constants.js +0 -34
  16. package/core/Affirm.js +0 -42
  17. package/core/Algo.js +0 -160
  18. package/core/dste/DSTE.js +0 -113
  19. package/core/logger/DebugLogService.js +0 -48
  20. package/core/logger/DevelopmentLogService.js +0 -70
  21. package/core/logger/LocalLogService.js +0 -70
  22. package/core/logger/Logger.js +0 -54
  23. package/database/DatabaseEngine.js +0 -149
  24. package/database/DatabaseStructure.js +0 -27
  25. package/definitions/DatasetDefinitions.js +0 -2
  26. package/definitions/ExecutorDefinitions.js +0 -2
  27. package/definitions/ProcessENV.js +0 -169
  28. package/definitions/agents/DestinationDriver.js +0 -2
  29. package/definitions/agents/SourceDriver.js +0 -2
  30. package/definitions/cli.js +0 -2
  31. package/definitions/database/ApiKeys.js +0 -2
  32. package/definitions/database/Stored.js +0 -7
  33. package/definitions/database/UsageStat.js +0 -2
  34. package/definitions/database/User.js +0 -2
  35. package/definitions/json_schemas/consumer-schema.json +0 -1226
  36. package/definitions/json_schemas/producer-schema.json +0 -308
  37. package/definitions/json_schemas/project-schema.json +0 -100
  38. package/definitions/json_schemas/source-schema.json +0 -249
  39. package/definitions/requests/ConsumerRequest.js +0 -2
  40. package/definitions/requests/Developer.js +0 -2
  41. package/definitions/requests/Mapping.js +0 -2
  42. package/definitions/requests/ProducerRequest.js +0 -2
  43. package/definitions/requests/Request.js +0 -2
  44. package/definitions/resources/Compiled.js +0 -2
  45. package/definitions/resources/Consumer.js +0 -2
  46. package/definitions/resources/Environment.js +0 -2
  47. package/definitions/resources/Library.js +0 -2
  48. package/definitions/resources/Producer.js +0 -2
  49. package/definitions/resources/Project.js +0 -2
  50. package/definitions/resources/Schema.js +0 -2
  51. package/definitions/resources/Source.js +0 -2
  52. package/definitions/temp.js +0 -2
  53. package/definitions/transform/Transformations.js +0 -2
  54. package/drivers/DeltaShareDriver.js +0 -186
  55. package/drivers/DriverFactory.js +0 -72
  56. package/drivers/DriverHelper.js +0 -248
  57. package/drivers/HttpApiDriver.js +0 -208
  58. package/drivers/RedshiftDriver.js +0 -184
  59. package/drivers/files/LocalDestinationDriver.js +0 -146
  60. package/drivers/files/LocalSourceDriver.js +0 -405
  61. package/drivers/s3/S3DestinationDriver.js +0 -197
  62. package/drivers/s3/S3SourceDriver.js +0 -495
  63. package/engines/CryptoEngine.js +0 -75
  64. package/engines/Environment.js +0 -170
  65. package/engines/ProcessENVManager.js +0 -112
  66. package/engines/RandomEngine.js +0 -47
  67. package/engines/SecretManager.js +0 -23
  68. package/engines/UserManager.js +0 -66
  69. package/engines/ai/AutoMapperEngine.js +0 -37
  70. package/engines/ai/DeveloperEngine.js +0 -497
  71. package/engines/ai/LLM.js +0 -255
  72. package/engines/consumer/ConsumerManager.js +0 -218
  73. package/engines/consumer/ConsumerOnFinishManager.js +0 -202
  74. package/engines/dataset/Dataset.js +0 -824
  75. package/engines/dataset/DatasetManager.js +0 -211
  76. package/engines/dataset/DatasetRecord.js +0 -120
  77. package/engines/dataset/DatasetRecordPool.js +0 -77
  78. package/engines/execution/RequestExecutor.js +0 -67
  79. package/engines/parsing/CSVParser.js +0 -60
  80. package/engines/parsing/LineParser.js +0 -71
  81. package/engines/parsing/ParseCompression.js +0 -101
  82. package/engines/parsing/ParseHelper.js +0 -18
  83. package/engines/parsing/ParseManager.js +0 -54
  84. package/engines/parsing/XLSParser.js +0 -87
  85. package/engines/parsing/XMLParser.js +0 -115
  86. package/engines/producer/ProducerEngine.js +0 -127
  87. package/engines/producer/ProducerManager.js +0 -43
  88. package/engines/scheduler/CronScheduler.js +0 -222
  89. package/engines/scheduler/QueueManager.js +0 -314
  90. package/engines/schema/SchemaValidator.js +0 -67
  91. package/engines/transform/JoinEngine.js +0 -232
  92. package/engines/transform/TransformationEngine.js +0 -277
  93. package/engines/transform/TypeCaster.js +0 -59
  94. package/engines/usage/DataframeManager.js +0 -55
  95. package/engines/usage/UsageDataManager.js +0 -151
  96. package/engines/usage/UsageManager.js +0 -65
  97. package/engines/validation/Validator.js +0 -216
  98. package/executors/ConsumerExecutor.js +0 -280
  99. package/executors/Executor.js +0 -177
  100. package/executors/ExecutorOrchestrator.js +0 -331
  101. package/executors/ExecutorPerformance.js +0 -17
  102. package/executors/ExecutorProgress.js +0 -54
  103. package/executors/ExecutorScope.js +0 -52
  104. package/executors/OutputExecutor.js +0 -118
  105. package/executors/ProducerExecutor.js +0 -108
  106. package/helper/Helper.js +0 -149
  107. package/helper/Logger.js +0 -84
  108. package/helper/Runtime.js +0 -20
  109. package/helper/Settings.js +0 -13
  110. package/licencing/LicenceManager.js +0 -64
  111. package/settings.js +0 -12
package/index.js CHANGED
@@ -1,123 +1,113 @@
1
1
  #!/usr/bin/env node
2
- "use strict";
3
- var __importDefault = (this && this.__importDefault) || function (mod) {
4
- return (mod && mod.__esModule) ? mod : { "default": mod };
5
- };
6
- Object.defineProperty(exports, "__esModule", { value: true });
7
- const commander_1 = require("commander");
8
- const dotenv_1 = __importDefault(require("dotenv"));
9
- const compile_1 = require("./actions/compile");
10
- const debug_1 = require("./actions/debug");
11
- const deploy_1 = require("./actions/deploy");
12
- const init_1 = require("./actions/init");
13
- const run_1 = require("./actions/run");
14
- const discover_1 = require("./actions/discover");
15
- const create_producer_1 = require("./actions/create_producer");
16
- const create_consumer_1 = require("./actions/create_consumer");
17
- const Constants_1 = __importDefault(require("./Constants"));
18
- const LicenceManager_1 = __importDefault(require("./licencing/LicenceManager"));
19
- const Runtime_1 = __importDefault(require("./helper/Runtime"));
20
- const automap_1 = require("./actions/automap");
21
- const sample_1 = require("./actions/sample");
22
- const mock_1 = require("./actions/mock");
23
- const ProcessENVManager_1 = __importDefault(require("./engines/ProcessENVManager"));
24
- dotenv_1.default.configDotenv();
25
- ProcessENVManager_1.default.init()
26
- .then(() => {
27
- // Validate the remora licence
28
- const remoraLicenceKey = ProcessENVManager_1.default.getEnvVariable('REMORA_LICENCE_KEY');
29
- const check = LicenceManager_1.default.validate(remoraLicenceKey);
30
- if (!check.valid) {
31
- console.error(`Invalid Remora licence key, the product is not active: remember to set "REMORA_LICENCE_KEY" environment variable.`);
2
+ import { Command } from 'commander';
3
+ import dotenv from 'dotenv';
4
+ import { compile } from './actions/compile';
5
+ import { debug } from './actions/debug';
6
+ import { deploy } from './actions/deploy';
7
+ import { init } from './actions/init';
8
+ import { run } from './actions/run';
9
+ import { discover } from './actions/discover';
10
+ import { create_producer } from './actions/create_producer';
11
+ import { create_consumer } from './actions/create_consumer';
12
+ import { CONSTANTS } from '@remora/app-constants';
13
+ import { LicenceManager } from '@remora/app-auth';
14
+ import { Runtime } from '@remora/app-helper';
15
+ import { automap } from './actions/automap';
16
+ import { sample } from './actions/sample';
17
+ import { mock } from './actions/mock';
18
+ import { ProcessENVManager } from '@remora/app-common';
19
+ dotenv.configDotenv();
20
+ const program = new Command();
21
+ // Validate the remora licence
22
+ const remoraLicenceKey = ProcessENVManager.getEnvVariable('REMORA_LICENCE_KEY');
23
+ const check = LicenceManager.validate(remoraLicenceKey);
24
+ if (!check.valid) {
25
+ console.error(`Invalid Remora licence key, the product is not active: remember to set "REMORA_LICENCE_KEY" environment variable.`);
26
+ process.exit(1);
27
+ }
28
+ // Runtime check on heap size to warn user of insufficent runtime resources
29
+ const { heapSizeMB } = Runtime.getHeap();
30
+ if (heapSizeMB < CONSTANTS.defaults.MIN_RUNTIME_HEAP_MB)
31
+ console.warn(`Remora is running with ${heapSizeMB}MB of runtime heap, which is below the bare minimum of ${CONSTANTS.defaults.MIN_RUNTIME_HEAP_MB}MB (Recommended: ${CONSTANTS.defaults.RECOMMENDED_RUNTIME_HEAP_MB}MB).`);
32
+ else if (heapSizeMB < CONSTANTS.defaults.RECOMMENDED_RUNTIME_HEAP_MB)
33
+ console.warn(`Remora is running with ${heapSizeMB} MB of runtime heap, which is below the recommended of ${CONSTANTS.defaults.RECOMMENDED_RUNTIME_HEAP_MB} MB.`);
34
+ // Initialize all commands
35
+ program
36
+ .version(CONSTANTS.cliVersion + '', '-v, --version', 'Display the version of the CLI')
37
+ .description('CLI tool for setting up and managing Data-Remora');
38
+ program
39
+ .command('version')
40
+ .description('Display the version of the CLI')
41
+ .action(() => {
42
+ const packageJson = require('../package.json');
43
+ console.log(`Data-Remora CLI version: ${packageJson.version}`);
44
+ });
45
+ program
46
+ .command('init')
47
+ .description('Initialize the application and set up configuration')
48
+ .action(init);
49
+ program
50
+ .command('compile')
51
+ .description('Compile the config and check for errors')
52
+ .action(compile);
53
+ program
54
+ .command('debug')
55
+ .description('Check the connection with sources and producers')
56
+ .action(debug);
57
+ program
58
+ .command('deploy')
59
+ .description('Deploy the application to the specified environment')
60
+ .option('-e, --env <environment>', 'Target environment (staging, production)', 'staging')
61
+ .option('--skip-tests', 'Skip running tests before deployment')
62
+ .option('--build-only', 'Build without deploying')
63
+ .action(deploy);
64
+ program
65
+ .command('run [name]')
66
+ .description('Execute consumers. Optionally specify a single consumer name or use --project to run all consumers in a project.')
67
+ .option('-p, --project <name>', 'Run all consumers belonging to the specified project')
68
+ .action((name, options) => run(name, options));
69
+ program
70
+ .command('discover')
71
+ .description('Discover the data shape of a producer and automatically create the resource for it.')
72
+ .argument('<producer>', 'The producer to discover')
73
+ .action(discover);
74
+ program
75
+ .command('create-producer <name>')
76
+ .description('Create a new producer configuration with default settings')
77
+ .action(create_producer);
78
+ program
79
+ .command('automap')
80
+ .description('Automatically map a producer to consumers using specified schemas.')
81
+ .argument('<producer>', 'The producer to analyze')
82
+ .argument('<schemas...>', 'One or more schema names to map against')
83
+ .action(automap);
84
+ program
85
+ .command('create-consumer <name>')
86
+ .description('Create a new consumer configuration with default settings')
87
+ .option('-p, --producer <name>', 'Producer to create a one-to-one mapping from')
88
+ .action((name, options) => create_consumer(name, options.producer));
89
+ program
90
+ .command('sample <name>')
91
+ .description('Sample data from a producer or consumer and display it in a formatted table')
92
+ .option('-s, --size <number>', 'Number of sample rows to display (default: 10)', '10')
93
+ .action((name, options) => {
94
+ const sampleSize = parseInt(options.size, 10);
95
+ if (isNaN(sampleSize) || sampleSize <= 0) {
96
+ console.error('Sample size must be a positive number');
97
+ process.exit(1);
98
+ }
99
+ sample(name, sampleSize);
100
+ });
101
+ program
102
+ .command('mock <producer>')
103
+ .description('Generate mock data for a producer based on its dimensions')
104
+ .argument('<records>', 'Number of mock records to generate')
105
+ .action((producer, records) => {
106
+ const recordCount = parseInt(records, 10);
107
+ if (isNaN(recordCount) || recordCount <= 0) {
108
+ console.error('Record count must be a positive number');
32
109
  process.exit(1);
33
110
  }
34
- // Runtime check on heap size to warn user of insufficent runtime resources
35
- const { heapSizeMB } = Runtime_1.default.getHeap();
36
- if (heapSizeMB < Constants_1.default.defaults.MIN_RUNTIME_HEAP_MB)
37
- console.warn(`Remora is running with ${heapSizeMB}MB of runtime heap, which is below the bare minimum of ${Constants_1.default.defaults.MIN_RUNTIME_HEAP_MB}MB (Recommended: ${Constants_1.default.defaults.RECOMMENDED_RUNTIME_HEAP_MB}MB).`);
38
- else if (heapSizeMB < Constants_1.default.defaults.RECOMMENDED_RUNTIME_HEAP_MB)
39
- console.warn(`Remora is running with ${heapSizeMB} MB of runtime heap, which is below the recommended of ${Constants_1.default.defaults.RECOMMENDED_RUNTIME_HEAP_MB} MB.`);
40
- })
41
- .then(() => {
42
- // Initialize all commands
43
- const program = new commander_1.Command();
44
- program
45
- .version(Constants_1.default.cliVersion + '', '-v, --version', 'Display the version of the CLI')
46
- .description('CLI tool for setting up and managing Data-Remora');
47
- program
48
- .command('version')
49
- .description('Display the version of the CLI')
50
- .action(() => {
51
- const packageJson = require('../package.json');
52
- console.log(`Data-Remora CLI version: ${packageJson.version}`);
53
- });
54
- program
55
- .command('init')
56
- .description('Initialize the application and set up configuration')
57
- .action(init_1.init);
58
- program
59
- .command('compile')
60
- .description('Compile the config and check for errors')
61
- .action(compile_1.compile);
62
- program
63
- .command('debug')
64
- .description('Check the connection with sources and producers')
65
- .action(debug_1.debug);
66
- program
67
- .command('deploy')
68
- .description('Deploy the application to the specified environment')
69
- .option('-e, --env <environment>', 'Target environment (staging, production)', 'staging')
70
- .option('--skip-tests', 'Skip running tests before deployment')
71
- .option('--build-only', 'Build without deploying')
72
- .action(deploy_1.deploy);
73
- program
74
- .command('run [name]')
75
- .description('Execute consumers. Optionally specify a single consumer name or use --project to run all consumers in a project.')
76
- .option('-p, --project <name>', 'Run all consumers belonging to the specified project')
77
- .action((name, options) => (0, run_1.run)(name, options));
78
- program
79
- .command('discover')
80
- .description('Discover the data shape of a producer and automatically create the resource for it.')
81
- .argument('<producer>', 'The producer to discover')
82
- .action(discover_1.discover);
83
- program
84
- .command('create-producer <name>')
85
- .description('Create a new producer configuration with default settings')
86
- .action(create_producer_1.create_producer);
87
- program
88
- .command('automap')
89
- .description('Automatically map a producer to consumers using specified schemas.')
90
- .argument('<producer>', 'The producer to analyze')
91
- .argument('<schemas...>', 'One or more schema names to map against')
92
- .action(automap_1.automap);
93
- program
94
- .command('create-consumer <name>')
95
- .description('Create a new consumer configuration with default settings')
96
- .option('-p, --producer <name>', 'Producer to create a one-to-one mapping from')
97
- .action((name, options) => (0, create_consumer_1.create_consumer)(name, options.producer));
98
- program
99
- .command('sample <name>')
100
- .description('Sample data from a producer or consumer and display it in a formatted table')
101
- .option('-s, --size <number>', 'Number of sample rows to display (default: 10)', '10')
102
- .action((name, options) => {
103
- const sampleSize = parseInt(options.size, 10);
104
- if (isNaN(sampleSize) || sampleSize <= 0) {
105
- console.error('Sample size must be a positive number');
106
- process.exit(1);
107
- }
108
- (0, sample_1.sample)(name, sampleSize);
109
- });
110
- program
111
- .command('mock <producer>')
112
- .description('Generate mock data for a producer based on its dimensions')
113
- .argument('<records>', 'Number of mock records to generate')
114
- .action((producer, records) => {
115
- const recordCount = parseInt(records, 10);
116
- if (isNaN(recordCount) || recordCount <= 0) {
117
- console.error('Record count must be a positive number');
118
- process.exit(1);
119
- }
120
- (0, mock_1.mock)(producer, recordCount);
121
- });
122
- program.parse(process.argv);
111
+ mock(producer, recordCount);
123
112
  });
113
+ program.parse(process.argv);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@forzalabs/remora",
3
- "version": "1.0.22",
3
+ "version": "1.1.1",
4
4
  "description": "A powerful CLI tool for seamless data translation.",
5
5
  "main": "index.js",
6
6
  "private": false,
@@ -8,7 +8,6 @@
8
8
  "remora": "index.js"
9
9
  },
10
10
  "scripts": {
11
- "sync": "cd ../dev_ops && npm run sync",
12
11
  "dev": "clear && npm run fast-build && clear && npx tsx scripts/dev.ts",
13
12
  "dev:w": "clear && npm run fast-build && clear && npx tsx scripts/dev.ts",
14
13
  "ts-check": "npx tsc --noemit",
@@ -20,8 +19,8 @@
20
19
  "deploy": "npx tsx ./src/index.ts deploy",
21
20
  "debug": "npx tsx ./src/index.ts debug",
22
21
  "create-producer": "npx tsx ./src/index.ts create-producer",
23
- "copy-static-file": "npx tsx ./scripts/CopyStaticFile.js",
24
- "build": "npm i && npm run sync && rm -rf .build && tsc --outDir .build && npm run copy-static-file",
22
+ "copy-static-file": "npx tsx ./scripts/CopyStaticFile.ts",
23
+ "build": "npm i && rm -rf .build && tsc --outDir .build && npm run copy-static-file",
25
24
  "fast-build": "tsc --outDir .build",
26
25
  "upload": "npm run build && cd .build && npm publish --access=public"
27
26
  },
@@ -34,6 +33,7 @@
34
33
  "author": "",
35
34
  "license": "BSL",
36
35
  "dependencies": {
36
+ "@remora/app-definitions": "1.0.0",
37
37
  "@aws-sdk/client-redshift-data": "^3.699.0",
38
38
  "@aws-sdk/client-s3": "^3.701.0",
39
39
  "@aws-sdk/client-sqs": "^3.886.0",
@@ -1,36 +1,20 @@
1
- "use strict";
2
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
- return new (P || (P = Promise))(function (resolve, reject) {
5
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
- step((generator = generator.apply(thisArg, _arguments || [])).next());
9
- });
10
- };
11
- var __importDefault = (this && this.__importDefault) || function (mod) {
12
- return (mod && mod.__esModule) ? mod : { "default": mod };
13
- };
14
- Object.defineProperty(exports, "__esModule", { value: true });
15
- const workerpool_1 = __importDefault(require("workerpool"));
16
- const dotenv_1 = __importDefault(require("dotenv"));
17
- const Affirm_1 = __importDefault(require("../core/Affirm"));
18
- const Environment_1 = __importDefault(require("../engines/Environment"));
19
- const Executor_1 = __importDefault(require("../executors/Executor"));
20
- const ProcessENVManager_1 = __importDefault(require("../engines/ProcessENVManager"));
21
- dotenv_1.default.configDotenv();
22
- const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
23
- yield ProcessENVManager_1.default.init();
24
- Environment_1.default.load('./');
1
+ import workerpool from 'workerpool';
2
+ import dotenv from 'dotenv';
3
+ import { affirm } from '@remora/app-core';
4
+ import { Environment } from '@remora/app-common';
5
+ import { Executor } from '@remora/app-executors';
6
+ dotenv.configDotenv();
7
+ const run = async (workerData) => {
8
+ Environment.load('./');
25
9
  try {
26
10
  const { workerId, chunk, consumer, producer, prodDimensions, scope, options } = workerData;
27
- (0, Affirm_1.default)(workerId, `Invalid worker id`);
28
- (0, Affirm_1.default)(consumer, `Invalid consumer`);
29
- (0, Affirm_1.default)(producer, `Invalid producer`);
30
- (0, Affirm_1.default)(chunk, `Invalid chunk`);
31
- (0, Affirm_1.default)(scope, `Invalid executor scope`);
32
- const executor = new Executor_1.default();
33
- const res = yield executor.run({
11
+ affirm(workerId, `Invalid worker id`);
12
+ affirm(consumer, `Invalid consumer`);
13
+ affirm(producer, `Invalid producer`);
14
+ affirm(chunk, `Invalid chunk`);
15
+ affirm(scope, `Invalid executor scope`);
16
+ const executor = new Executor();
17
+ const res = await executor.run({
34
18
  consumer,
35
19
  producer,
36
20
  prodDimensions,
@@ -38,7 +22,7 @@ const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
38
22
  chunk,
39
23
  options,
40
24
  scope,
41
- reportWork: packet => workerpool_1.default.workerEmit(packet)
25
+ reportWork: packet => workerpool.workerEmit(packet)
42
26
  });
43
27
  return res;
44
28
  }
@@ -46,7 +30,7 @@ const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
46
30
  console.error(error);
47
31
  return null;
48
32
  }
49
- });
50
- workerpool_1.default.worker({
33
+ };
34
+ workerpool.worker({
51
35
  executor: run
52
36
  });
package/Constants.js DELETED
@@ -1,34 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- const CONSTANTS = {
4
- cliVersion: '1.0.22',
5
- backendVersion: 1,
6
- backendPort: 5088,
7
- workerVersion: 2,
8
- workerPort: 5069,
9
- /**
10
- * Column name for the dynamically injected source filename dimension.
11
- * Prefixed with $ to indicate it's a system-generated dynamic value.
12
- */
13
- SOURCE_FILENAME_COLUMN: '$source_filename',
14
- defaults: {
15
- REMORA_PATH: './remora',
16
- PRODUCER_TEMP_FOLDER: '.temp',
17
- SQL_MAX_QUERY_ROWS: 10000,
18
- STRING_MAX_CHARACTERS_LENGTH: 10000000,
19
- MAX_ITEMS_IN_MEMORY: 200000,
20
- MIN_RUNTIME_HEAP_MB: 4000,
21
- RECOMMENDED_RUNTIME_HEAP_MB: 8000,
22
- INDICATIVE_THREAD_LINE_COUNT: 750000,
23
- MAX_THREAD_COUNT: 8,
24
- /**
25
- * Minimum file size to consider parallel processing (10 MB)
26
- */
27
- MIN_FILE_SIZE_FOR_PARALLEL: 10 * 1024 * 1024,
28
- /**
29
- * Minimum chunk size per worker to justify overhead (2 MB)
30
- */
31
- MIN_CHUNK_SIZE: 2 * 1024 * 1024
32
- }
33
- };
34
- exports.default = CONSTANTS;
package/core/Affirm.js DELETED
@@ -1,42 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- /**
4
- * Checks if 'value' is truthy. If not, throws an exception with custom 'msg' (if present).
5
- */
6
- const affirm = (value, msg) => {
7
- if (!value)
8
- throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
9
- };
10
- affirm.hasValue = (value, msg) => {
11
- if (value === null || value === undefined)
12
- throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
13
- };
14
- affirm.hasItems = (value, msg) => {
15
- if (!value || !Array.isArray(value) || !(value.length > 0))
16
- throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
17
- };
18
- affirm.doesntInclude = (value, array, msg) => {
19
- affirm(value, msg);
20
- if (!array || !Array.isArray(array) || array.includes(value))
21
- throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
22
- };
23
- affirm.doesntExist = (item, array, key, msg) => {
24
- affirm(item, msg);
25
- if (!array || !Array.isArray(array) || !array.some(arrayItem => arrayItem[key] === item[key]))
26
- throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
27
- };
28
- affirm.noDupes = (array, msg) => {
29
- if (!array || !Array.isArray(array) || new Set(array).size !== array.length)
30
- throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
31
- };
32
- affirm.noDupedItems = (array, key, msg) => {
33
- if (!array || !Array.isArray(array))
34
- throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
35
- const seen = new Set();
36
- for (const item of array) {
37
- if (seen.has(item[key]))
38
- throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
39
- seen.add(item[key]);
40
- }
41
- };
42
- exports.default = affirm;
package/core/Algo.js DELETED
@@ -1,160 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- const Affirm_1 = __importDefault(require("./Affirm"));
7
- /**
8
- * Core Algorithms that are shared between different project/problems.
9
- * This file needs to be as lean as possible and must not include any external dependencies.
10
- */
11
- const algo = {
12
- hasVal: (value) => value !== null && value !== undefined,
13
- sleep: (milliseconds) => {
14
- (0, Affirm_1.default)(milliseconds > 0, 'Invalid @milliseconds parameter. Must be a positive number.');
15
- return new Promise(resolve => setTimeout(resolve, milliseconds));
16
- },
17
- rng: (min, max) => {
18
- (0, Affirm_1.default)(min, 'Invalid @min parameter. Must be a valid number.');
19
- (0, Affirm_1.default)(max, 'Invalid @max parameter. Must be a valid number.');
20
- (0, Affirm_1.default)(min < max, 'The minimum value must be less than the maximum value.');
21
- return Math.floor(Math.random() * (max - min + 1)) + min;
22
- },
23
- uniqBy: (array, pk) => {
24
- if (!array || array.length === 0)
25
- return [];
26
- return Array.from(new Set(array.map(x => x[pk])));
27
- },
28
- uniq: (array) => {
29
- if (!array || array.length === 0)
30
- return [];
31
- return Array.from(new Set(array));
32
- },
33
- duplicatesObject: (array, pk) => {
34
- if (!array || array.length === 0)
35
- return [];
36
- return array.filter(x => array.filter(k => k[pk] === x[pk]).length > 1);
37
- },
38
- duplicates: (array) => {
39
- if (!array || array.length === 0)
40
- return [];
41
- return array.filter(x => array.filter(k => k === x).length > 1);
42
- },
43
- locations: (string, substring) => {
44
- (0, Affirm_1.default)(string, 'Invalid @string parameter.');
45
- (0, Affirm_1.default)(substring, 'Invalid @substring parameter.');
46
- const a = [];
47
- let i = -1;
48
- while ((i = string.indexOf(substring, i + 1)) >= 0)
49
- a.push(i);
50
- return a;
51
- },
52
- isNumber: (value) => typeof value === 'number',
53
- orderBy: (list, key, direction = 'asc') => {
54
- (0, Affirm_1.default)(list, 'Invalid array.');
55
- Affirm_1.default.hasItems(list, 'Array must be non-empty.');
56
- (0, Affirm_1.default)(direction, 'Invalid direction parameter. Must be "asc" or "desc".');
57
- const sortedList = [...list];
58
- if (key) {
59
- sortedList.sort((a, b) => {
60
- const aValue = a[key];
61
- const bValue = b[key];
62
- if (aValue < bValue)
63
- return direction === 'asc' ? -1 : 1;
64
- if (aValue > bValue)
65
- return direction === 'asc' ? 1 : -1;
66
- return 0;
67
- });
68
- return sortedList;
69
- }
70
- else {
71
- sortedList.sort((a, b) => {
72
- if (a < b)
73
- return direction === 'asc' ? -1 : 1;
74
- if (a > b)
75
- return direction === 'asc' ? 1 : -1;
76
- return 0;
77
- });
78
- return sortedList;
79
- }
80
- },
81
- groupBy: (array, key) => {
82
- Affirm_1.default.hasItems(array, 'Array must be non-empty');
83
- (0, Affirm_1.default)(key, 'key is invalid');
84
- const result = new Map();
85
- for (let i = 0; i < array.length; i++) {
86
- const item = array[i];
87
- const keyVal = item[key];
88
- if (result.has(keyVal)) {
89
- const current = result.get(keyVal);
90
- current.push(item);
91
- result.set(keyVal, current);
92
- }
93
- else {
94
- result.set(keyVal, [item]);
95
- }
96
- }
97
- return result;
98
- },
99
- chunkString: (text, chunkSize) => {
100
- (0, Affirm_1.default)(text, 'Invalid text');
101
- (0, Affirm_1.default)(chunkSize, 'Invalid chunk size');
102
- (0, Affirm_1.default)(text.length > 0, 'Text length must be greater than 0');
103
- (0, Affirm_1.default)(chunkSize > 0, 'Chunk size must be greater than 0');
104
- if (text.length <= chunkSize)
105
- return [text];
106
- const numChunks = Math.ceil(text.length / chunkSize);
107
- const chunks = new Array(numChunks);
108
- for (let i = 0; i < chunks.length; i++) {
109
- chunks[i] = text.substring(i * chunkSize, chunkSize);
110
- }
111
- return chunks;
112
- },
113
- first: (arr) => {
114
- (0, Affirm_1.default)(algo.hasVal(arr), 'Array must not be null or undefined');
115
- Affirm_1.default.hasItems(arr, 'Array must be non-empty');
116
- return arr[0];
117
- },
118
- last: (arr) => {
119
- (0, Affirm_1.default)(algo.hasVal(arr), 'Array must not be null or undefined');
120
- (0, Affirm_1.default)(arr.length > 0, 'Array must be non-empty');
121
- return arr[arr.length - 1];
122
- },
123
- mean: (numbers) => {
124
- (0, Affirm_1.default)(algo.hasVal(numbers), 'Array must not be null or undefined');
125
- if (numbers.length === 0)
126
- return 0;
127
- const total = algo.sum(numbers);
128
- return total / numbers.length;
129
- },
130
- sum: (numbers) => {
131
- (0, Affirm_1.default)(algo.hasVal(numbers), 'Array must not be null or undefined');
132
- if (numbers.length === 0)
133
- return 0;
134
- let total = 0;
135
- for (let i = 0; i < numbers.length; i++) {
136
- total += numbers[i];
137
- }
138
- return total;
139
- },
140
- round: (number, precision) => {
141
- (0, Affirm_1.default)(algo.hasVal(number), 'Number must not be null or undefined');
142
- const factor = Math.pow(10, precision !== null && precision !== void 0 ? precision : 2);
143
- return Math.round(number * factor) / factor;
144
- },
145
- min: (arr) => {
146
- (0, Affirm_1.default)(algo.hasVal(arr), 'Array must not be null or undefined');
147
- if (arr.length === 0)
148
- return 0;
149
- return Math.min(...arr);
150
- },
151
- max: (arr) => {
152
- (0, Affirm_1.default)(algo.hasVal(arr), 'Array must not be null or undefined');
153
- if (arr.length === 0)
154
- return 0;
155
- return Math.max(...arr);
156
- },
157
- replaceAll: (text, search, replace) => text.replace(new RegExp(search, 'g'), replace),
158
- deepClone: (data) => JSON.parse(JSON.stringify(data))
159
- };
160
- exports.default = algo;