@forzalabs/remora 1.0.21 → 1.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/actions/automap.js +26 -42
  2. package/actions/compile.js +27 -43
  3. package/actions/create_consumer.js +24 -40
  4. package/actions/create_producer.js +16 -32
  5. package/actions/debug.js +18 -34
  6. package/actions/deploy.js +30 -46
  7. package/actions/discover.js +13 -29
  8. package/actions/init.js +29 -45
  9. package/actions/mock.js +16 -32
  10. package/actions/run.js +34 -52
  11. package/actions/sample.js +42 -58
  12. package/index.js +38 -43
  13. package/package.json +4 -4
  14. package/workers/ExecutorWorker.js +18 -32
  15. package/Constants.js +0 -34
  16. package/core/Affirm.js +0 -42
  17. package/core/Algo.js +0 -160
  18. package/core/dste/DSTE.js +0 -113
  19. package/core/logger/DebugLogService.js +0 -48
  20. package/core/logger/DevelopmentLogService.js +0 -70
  21. package/core/logger/LocalLogService.js +0 -70
  22. package/core/logger/Logger.js +0 -54
  23. package/database/DatabaseEngine.js +0 -149
  24. package/database/DatabaseStructure.js +0 -27
  25. package/definitions/DatasetDefinitions.js +0 -2
  26. package/definitions/ExecutorDefinitions.js +0 -2
  27. package/definitions/ProcessENV.js +0 -2
  28. package/definitions/agents/DestinationDriver.js +0 -2
  29. package/definitions/agents/SourceDriver.js +0 -2
  30. package/definitions/cli.js +0 -2
  31. package/definitions/database/ApiKeys.js +0 -2
  32. package/definitions/database/Stored.js +0 -7
  33. package/definitions/database/UsageStat.js +0 -2
  34. package/definitions/database/User.js +0 -2
  35. package/definitions/json_schemas/consumer-schema.json +0 -1226
  36. package/definitions/json_schemas/producer-schema.json +0 -308
  37. package/definitions/json_schemas/project-schema.json +0 -100
  38. package/definitions/json_schemas/source-schema.json +0 -249
  39. package/definitions/requests/ConsumerRequest.js +0 -2
  40. package/definitions/requests/Developer.js +0 -2
  41. package/definitions/requests/Mapping.js +0 -2
  42. package/definitions/requests/ProducerRequest.js +0 -2
  43. package/definitions/requests/Request.js +0 -2
  44. package/definitions/resources/Compiled.js +0 -2
  45. package/definitions/resources/Consumer.js +0 -2
  46. package/definitions/resources/Environment.js +0 -2
  47. package/definitions/resources/Library.js +0 -2
  48. package/definitions/resources/Producer.js +0 -2
  49. package/definitions/resources/Project.js +0 -2
  50. package/definitions/resources/Schema.js +0 -2
  51. package/definitions/resources/Source.js +0 -2
  52. package/definitions/temp.js +0 -2
  53. package/definitions/transform/Transformations.js +0 -2
  54. package/drivers/DeltaShareDriver.js +0 -186
  55. package/drivers/DriverFactory.js +0 -72
  56. package/drivers/DriverHelper.js +0 -248
  57. package/drivers/HttpApiDriver.js +0 -208
  58. package/drivers/RedshiftDriver.js +0 -184
  59. package/drivers/files/LocalDestinationDriver.js +0 -146
  60. package/drivers/files/LocalSourceDriver.js +0 -405
  61. package/drivers/s3/S3DestinationDriver.js +0 -197
  62. package/drivers/s3/S3SourceDriver.js +0 -495
  63. package/engines/CryptoEngine.js +0 -75
  64. package/engines/Environment.js +0 -170
  65. package/engines/ProcessENVManager.js +0 -83
  66. package/engines/RandomEngine.js +0 -47
  67. package/engines/SecretManager.js +0 -23
  68. package/engines/UserManager.js +0 -66
  69. package/engines/ai/AutoMapperEngine.js +0 -37
  70. package/engines/ai/DeveloperEngine.js +0 -497
  71. package/engines/ai/LLM.js +0 -255
  72. package/engines/consumer/ConsumerManager.js +0 -218
  73. package/engines/consumer/ConsumerOnFinishManager.js +0 -202
  74. package/engines/dataset/Dataset.js +0 -824
  75. package/engines/dataset/DatasetManager.js +0 -211
  76. package/engines/dataset/DatasetRecord.js +0 -120
  77. package/engines/dataset/DatasetRecordPool.js +0 -77
  78. package/engines/execution/RequestExecutor.js +0 -67
  79. package/engines/parsing/CSVParser.js +0 -60
  80. package/engines/parsing/LineParser.js +0 -71
  81. package/engines/parsing/ParseCompression.js +0 -101
  82. package/engines/parsing/ParseHelper.js +0 -18
  83. package/engines/parsing/ParseManager.js +0 -54
  84. package/engines/parsing/XLSParser.js +0 -87
  85. package/engines/parsing/XMLParser.js +0 -115
  86. package/engines/producer/ProducerEngine.js +0 -127
  87. package/engines/producer/ProducerManager.js +0 -43
  88. package/engines/scheduler/CronScheduler.js +0 -222
  89. package/engines/scheduler/QueueManager.js +0 -314
  90. package/engines/schema/SchemaValidator.js +0 -67
  91. package/engines/transform/JoinEngine.js +0 -232
  92. package/engines/transform/TransformationEngine.js +0 -277
  93. package/engines/transform/TypeCaster.js +0 -59
  94. package/engines/usage/DataframeManager.js +0 -55
  95. package/engines/usage/UsageDataManager.js +0 -151
  96. package/engines/usage/UsageManager.js +0 -65
  97. package/engines/validation/Validator.js +0 -216
  98. package/executors/ConsumerExecutor.js +0 -280
  99. package/executors/Executor.js +0 -177
  100. package/executors/ExecutorOrchestrator.js +0 -331
  101. package/executors/ExecutorPerformance.js +0 -17
  102. package/executors/ExecutorProgress.js +0 -54
  103. package/executors/ExecutorScope.js +0 -52
  104. package/executors/OutputExecutor.js +0 -118
  105. package/executors/ProducerExecutor.js +0 -108
  106. package/helper/Helper.js +0 -149
  107. package/helper/Logger.js +0 -84
  108. package/helper/Runtime.js +0 -20
  109. package/helper/Settings.js +0 -13
  110. package/licencing/LicenceManager.js +0 -64
  111. package/settings.js +0 -12
package/index.js CHANGED
@@ -1,44 +1,39 @@
1
1
  #!/usr/bin/env node
2
- "use strict";
3
- var __importDefault = (this && this.__importDefault) || function (mod) {
4
- return (mod && mod.__esModule) ? mod : { "default": mod };
5
- };
6
- Object.defineProperty(exports, "__esModule", { value: true });
7
- const commander_1 = require("commander");
8
- const dotenv_1 = __importDefault(require("dotenv"));
9
- const compile_1 = require("./actions/compile");
10
- const debug_1 = require("./actions/debug");
11
- const deploy_1 = require("./actions/deploy");
12
- const init_1 = require("./actions/init");
13
- const run_1 = require("./actions/run");
14
- const discover_1 = require("./actions/discover");
15
- const create_producer_1 = require("./actions/create_producer");
16
- const create_consumer_1 = require("./actions/create_consumer");
17
- const Constants_1 = __importDefault(require("./Constants"));
18
- const LicenceManager_1 = __importDefault(require("./licencing/LicenceManager"));
19
- const Runtime_1 = __importDefault(require("./helper/Runtime"));
20
- const automap_1 = require("./actions/automap");
21
- const sample_1 = require("./actions/sample");
22
- const mock_1 = require("./actions/mock");
23
- const ProcessENVManager_1 = __importDefault(require("./engines/ProcessENVManager"));
24
- dotenv_1.default.configDotenv();
25
- const program = new commander_1.Command();
2
+ import { Command } from 'commander';
3
+ import dotenv from 'dotenv';
4
+ import { compile } from './actions/compile';
5
+ import { debug } from './actions/debug';
6
+ import { deploy } from './actions/deploy';
7
+ import { init } from './actions/init';
8
+ import { run } from './actions/run';
9
+ import { discover } from './actions/discover';
10
+ import { create_producer } from './actions/create_producer';
11
+ import { create_consumer } from './actions/create_consumer';
12
+ import { CONSTANTS } from '@remora/app-constants';
13
+ import { LicenceManager } from '@remora/app-auth';
14
+ import { Runtime } from '@remora/app-helper';
15
+ import { automap } from './actions/automap';
16
+ import { sample } from './actions/sample';
17
+ import { mock } from './actions/mock';
18
+ import { ProcessENVManager } from '@remora/app-common';
19
+ dotenv.configDotenv();
20
+ const program = new Command();
26
21
  // Validate the remora licence
27
- const remoraLicenceKey = ProcessENVManager_1.default.getEnvVariable('REMORA_LICENCE_KEY');
28
- const check = LicenceManager_1.default.validate(remoraLicenceKey);
22
+ const remoraLicenceKey = ProcessENVManager.getEnvVariable('REMORA_LICENCE_KEY');
23
+ const check = LicenceManager.validate(remoraLicenceKey);
29
24
  if (!check.valid) {
30
25
  console.error(`Invalid Remora licence key, the product is not active: remember to set "REMORA_LICENCE_KEY" environment variable.`);
31
26
  process.exit(1);
32
27
  }
33
28
  // Runtime check on heap size to warn user of insufficent runtime resources
34
- const { heapSizeMB } = Runtime_1.default.getHeap();
35
- if (heapSizeMB < Constants_1.default.defaults.MIN_RUNTIME_HEAP_MB)
36
- console.warn(`Remora is running with ${heapSizeMB}MB of runtime heap, which is below the bare minimum of ${Constants_1.default.defaults.MIN_RUNTIME_HEAP_MB}MB (Recommended: ${Constants_1.default.defaults.RECOMMENDED_RUNTIME_HEAP_MB}MB).`);
37
- else if (heapSizeMB < Constants_1.default.defaults.RECOMMENDED_RUNTIME_HEAP_MB)
38
- console.warn(`Remora is running with ${heapSizeMB} MB of runtime heap, which is below the recommended of ${Constants_1.default.defaults.RECOMMENDED_RUNTIME_HEAP_MB} MB.`);
29
+ const { heapSizeMB } = Runtime.getHeap();
30
+ if (heapSizeMB < CONSTANTS.defaults.MIN_RUNTIME_HEAP_MB)
31
+ console.warn(`Remora is running with ${heapSizeMB}MB of runtime heap, which is below the bare minimum of ${CONSTANTS.defaults.MIN_RUNTIME_HEAP_MB}MB (Recommended: ${CONSTANTS.defaults.RECOMMENDED_RUNTIME_HEAP_MB}MB).`);
32
+ else if (heapSizeMB < CONSTANTS.defaults.RECOMMENDED_RUNTIME_HEAP_MB)
33
+ console.warn(`Remora is running with ${heapSizeMB} MB of runtime heap, which is below the recommended of ${CONSTANTS.defaults.RECOMMENDED_RUNTIME_HEAP_MB} MB.`);
39
34
  // Initialize all commands
40
35
  program
41
- .version(Constants_1.default.cliVersion + '', '-v, --version', 'Display the version of the CLI')
36
+ .version(CONSTANTS.cliVersion + '', '-v, --version', 'Display the version of the CLI')
42
37
  .description('CLI tool for setting up and managing Data-Remora');
43
38
  program
44
39
  .command('version')
@@ -50,47 +45,47 @@ program
50
45
  program
51
46
  .command('init')
52
47
  .description('Initialize the application and set up configuration')
53
- .action(init_1.init);
48
+ .action(init);
54
49
  program
55
50
  .command('compile')
56
51
  .description('Compile the config and check for errors')
57
- .action(compile_1.compile);
52
+ .action(compile);
58
53
  program
59
54
  .command('debug')
60
55
  .description('Check the connection with sources and producers')
61
- .action(debug_1.debug);
56
+ .action(debug);
62
57
  program
63
58
  .command('deploy')
64
59
  .description('Deploy the application to the specified environment')
65
60
  .option('-e, --env <environment>', 'Target environment (staging, production)', 'staging')
66
61
  .option('--skip-tests', 'Skip running tests before deployment')
67
62
  .option('--build-only', 'Build without deploying')
68
- .action(deploy_1.deploy);
63
+ .action(deploy);
69
64
  program
70
65
  .command('run [name]')
71
66
  .description('Execute consumers. Optionally specify a single consumer name or use --project to run all consumers in a project.')
72
67
  .option('-p, --project <name>', 'Run all consumers belonging to the specified project')
73
- .action((name, options) => (0, run_1.run)(name, options));
68
+ .action((name, options) => run(name, options));
74
69
  program
75
70
  .command('discover')
76
71
  .description('Discover the data shape of a producer and automatically create the resource for it.')
77
72
  .argument('<producer>', 'The producer to discover')
78
- .action(discover_1.discover);
73
+ .action(discover);
79
74
  program
80
75
  .command('create-producer <name>')
81
76
  .description('Create a new producer configuration with default settings')
82
- .action(create_producer_1.create_producer);
77
+ .action(create_producer);
83
78
  program
84
79
  .command('automap')
85
80
  .description('Automatically map a producer to consumers using specified schemas.')
86
81
  .argument('<producer>', 'The producer to analyze')
87
82
  .argument('<schemas...>', 'One or more schema names to map against')
88
- .action(automap_1.automap);
83
+ .action(automap);
89
84
  program
90
85
  .command('create-consumer <name>')
91
86
  .description('Create a new consumer configuration with default settings')
92
87
  .option('-p, --producer <name>', 'Producer to create a one-to-one mapping from')
93
- .action((name, options) => (0, create_consumer_1.create_consumer)(name, options.producer));
88
+ .action((name, options) => create_consumer(name, options.producer));
94
89
  program
95
90
  .command('sample <name>')
96
91
  .description('Sample data from a producer or consumer and display it in a formatted table')
@@ -101,7 +96,7 @@ program
101
96
  console.error('Sample size must be a positive number');
102
97
  process.exit(1);
103
98
  }
104
- (0, sample_1.sample)(name, sampleSize);
99
+ sample(name, sampleSize);
105
100
  });
106
101
  program
107
102
  .command('mock <producer>')
@@ -113,6 +108,6 @@ program
113
108
  console.error('Record count must be a positive number');
114
109
  process.exit(1);
115
110
  }
116
- (0, mock_1.mock)(producer, recordCount);
111
+ mock(producer, recordCount);
117
112
  });
118
113
  program.parse(process.argv);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@forzalabs/remora",
3
- "version": "1.0.21",
3
+ "version": "1.1.1",
4
4
  "description": "A powerful CLI tool for seamless data translation.",
5
5
  "main": "index.js",
6
6
  "private": false,
@@ -8,7 +8,6 @@
8
8
  "remora": "index.js"
9
9
  },
10
10
  "scripts": {
11
- "sync": "cd ../dev_ops && npm run sync",
12
11
  "dev": "clear && npm run fast-build && clear && npx tsx scripts/dev.ts",
13
12
  "dev:w": "clear && npm run fast-build && clear && npx tsx scripts/dev.ts",
14
13
  "ts-check": "npx tsc --noemit",
@@ -20,8 +19,8 @@
20
19
  "deploy": "npx tsx ./src/index.ts deploy",
21
20
  "debug": "npx tsx ./src/index.ts debug",
22
21
  "create-producer": "npx tsx ./src/index.ts create-producer",
23
- "copy-static-file": "npx tsx ./scripts/CopyStaticFile.js",
24
- "build": "npm i && npm run sync && rm -rf .build && tsc --outDir .build && npm run copy-static-file",
22
+ "copy-static-file": "npx tsx ./scripts/CopyStaticFile.ts",
23
+ "build": "npm i && rm -rf .build && tsc --outDir .build && npm run copy-static-file",
25
24
  "fast-build": "tsc --outDir .build",
26
25
  "upload": "npm run build && cd .build && npm publish --access=public"
27
26
  },
@@ -34,6 +33,7 @@
34
33
  "author": "",
35
34
  "license": "BSL",
36
35
  "dependencies": {
36
+ "@remora/app-definitions": "1.0.0",
37
37
  "@aws-sdk/client-redshift-data": "^3.699.0",
38
38
  "@aws-sdk/client-s3": "^3.701.0",
39
39
  "@aws-sdk/client-sqs": "^3.886.0",
@@ -1,34 +1,20 @@
1
- "use strict";
2
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
- return new (P || (P = Promise))(function (resolve, reject) {
5
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
- step((generator = generator.apply(thisArg, _arguments || [])).next());
9
- });
10
- };
11
- var __importDefault = (this && this.__importDefault) || function (mod) {
12
- return (mod && mod.__esModule) ? mod : { "default": mod };
13
- };
14
- Object.defineProperty(exports, "__esModule", { value: true });
15
- const workerpool_1 = __importDefault(require("workerpool"));
16
- const dotenv_1 = __importDefault(require("dotenv"));
17
- const Affirm_1 = __importDefault(require("../core/Affirm"));
18
- const Environment_1 = __importDefault(require("../engines/Environment"));
19
- const Executor_1 = __importDefault(require("../executors/Executor"));
20
- dotenv_1.default.configDotenv();
21
- const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
22
- Environment_1.default.load('./');
1
+ import workerpool from 'workerpool';
2
+ import dotenv from 'dotenv';
3
+ import { affirm } from '@remora/app-core';
4
+ import { Environment } from '@remora/app-common';
5
+ import { Executor } from '@remora/app-executors';
6
+ dotenv.configDotenv();
7
+ const run = async (workerData) => {
8
+ Environment.load('./');
23
9
  try {
24
10
  const { workerId, chunk, consumer, producer, prodDimensions, scope, options } = workerData;
25
- (0, Affirm_1.default)(workerId, `Invalid worker id`);
26
- (0, Affirm_1.default)(consumer, `Invalid consumer`);
27
- (0, Affirm_1.default)(producer, `Invalid producer`);
28
- (0, Affirm_1.default)(chunk, `Invalid chunk`);
29
- (0, Affirm_1.default)(scope, `Invalid executor scope`);
30
- const executor = new Executor_1.default();
31
- const res = yield executor.run({
11
+ affirm(workerId, `Invalid worker id`);
12
+ affirm(consumer, `Invalid consumer`);
13
+ affirm(producer, `Invalid producer`);
14
+ affirm(chunk, `Invalid chunk`);
15
+ affirm(scope, `Invalid executor scope`);
16
+ const executor = new Executor();
17
+ const res = await executor.run({
32
18
  consumer,
33
19
  producer,
34
20
  prodDimensions,
@@ -36,7 +22,7 @@ const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
36
22
  chunk,
37
23
  options,
38
24
  scope,
39
- reportWork: packet => workerpool_1.default.workerEmit(packet)
25
+ reportWork: packet => workerpool.workerEmit(packet)
40
26
  });
41
27
  return res;
42
28
  }
@@ -44,7 +30,7 @@ const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
44
30
  console.error(error);
45
31
  return null;
46
32
  }
47
- });
48
- workerpool_1.default.worker({
33
+ };
34
+ workerpool.worker({
49
35
  executor: run
50
36
  });
package/Constants.js DELETED
@@ -1,34 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- const CONSTANTS = {
4
- cliVersion: '1.0.21',
5
- backendVersion: 1,
6
- backendPort: 5088,
7
- workerVersion: 2,
8
- workerPort: 5069,
9
- /**
10
- * Column name for the dynamically injected source filename dimension.
11
- * Prefixed with $ to indicate it's a system-generated dynamic value.
12
- */
13
- SOURCE_FILENAME_COLUMN: '$source_filename',
14
- defaults: {
15
- REMORA_PATH: './remora',
16
- PRODUCER_TEMP_FOLDER: '.temp',
17
- SQL_MAX_QUERY_ROWS: 10000,
18
- STRING_MAX_CHARACTERS_LENGTH: 10000000,
19
- MAX_ITEMS_IN_MEMORY: 200000,
20
- MIN_RUNTIME_HEAP_MB: 4000,
21
- RECOMMENDED_RUNTIME_HEAP_MB: 8000,
22
- INDICATIVE_THREAD_LINE_COUNT: 750000,
23
- MAX_THREAD_COUNT: 8,
24
- /**
25
- * Minimum file size to consider parallel processing (10 MB)
26
- */
27
- MIN_FILE_SIZE_FOR_PARALLEL: 10 * 1024 * 1024,
28
- /**
29
- * Minimum chunk size per worker to justify overhead (2 MB)
30
- */
31
- MIN_CHUNK_SIZE: 2 * 1024 * 1024
32
- }
33
- };
34
- exports.default = CONSTANTS;
package/core/Affirm.js DELETED
@@ -1,42 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- /**
4
- * Checks if 'value' is truthy. If not, throws an exception with custom 'msg' (if present).
5
- */
6
- const affirm = (value, msg) => {
7
- if (!value)
8
- throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
9
- };
10
- affirm.hasValue = (value, msg) => {
11
- if (value === null || value === undefined)
12
- throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
13
- };
14
- affirm.hasItems = (value, msg) => {
15
- if (!value || !Array.isArray(value) || !(value.length > 0))
16
- throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
17
- };
18
- affirm.doesntInclude = (value, array, msg) => {
19
- affirm(value, msg);
20
- if (!array || !Array.isArray(array) || array.includes(value))
21
- throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
22
- };
23
- affirm.doesntExist = (item, array, key, msg) => {
24
- affirm(item, msg);
25
- if (!array || !Array.isArray(array) || !array.some(arrayItem => arrayItem[key] === item[key]))
26
- throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
27
- };
28
- affirm.noDupes = (array, msg) => {
29
- if (!array || !Array.isArray(array) || new Set(array).size !== array.length)
30
- throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
31
- };
32
- affirm.noDupedItems = (array, key, msg) => {
33
- if (!array || !Array.isArray(array))
34
- throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
35
- const seen = new Set();
36
- for (const item of array) {
37
- if (seen.has(item[key]))
38
- throw new Error(`Affirm failed${(msg && msg.length > 0) ? `: ${msg}` : ''}`);
39
- seen.add(item[key]);
40
- }
41
- };
42
- exports.default = affirm;
package/core/Algo.js DELETED
@@ -1,160 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- const Affirm_1 = __importDefault(require("./Affirm"));
7
- /**
8
- * Core Algorithms that are shared between different project/problems.
9
- * This file needs to be as lean as possible and must not include any external dependencies.
10
- */
11
- const algo = {
12
- hasVal: (value) => value !== null && value !== undefined,
13
- sleep: (milliseconds) => {
14
- (0, Affirm_1.default)(milliseconds > 0, 'Invalid @milliseconds parameter. Must be a positive number.');
15
- return new Promise(resolve => setTimeout(resolve, milliseconds));
16
- },
17
- rng: (min, max) => {
18
- (0, Affirm_1.default)(min, 'Invalid @min parameter. Must be a valid number.');
19
- (0, Affirm_1.default)(max, 'Invalid @max parameter. Must be a valid number.');
20
- (0, Affirm_1.default)(min < max, 'The minimum value must be less than the maximum value.');
21
- return Math.floor(Math.random() * (max - min + 1)) + min;
22
- },
23
- uniqBy: (array, pk) => {
24
- if (!array || array.length === 0)
25
- return [];
26
- return Array.from(new Set(array.map(x => x[pk])));
27
- },
28
- uniq: (array) => {
29
- if (!array || array.length === 0)
30
- return [];
31
- return Array.from(new Set(array));
32
- },
33
- duplicatesObject: (array, pk) => {
34
- if (!array || array.length === 0)
35
- return [];
36
- return array.filter(x => array.filter(k => k[pk] === x[pk]).length > 1);
37
- },
38
- duplicates: (array) => {
39
- if (!array || array.length === 0)
40
- return [];
41
- return array.filter(x => array.filter(k => k === x).length > 1);
42
- },
43
- locations: (string, substring) => {
44
- (0, Affirm_1.default)(string, 'Invalid @string parameter.');
45
- (0, Affirm_1.default)(substring, 'Invalid @substring parameter.');
46
- const a = [];
47
- let i = -1;
48
- while ((i = string.indexOf(substring, i + 1)) >= 0)
49
- a.push(i);
50
- return a;
51
- },
52
- isNumber: (value) => typeof value === 'number',
53
- orderBy: (list, key, direction = 'asc') => {
54
- (0, Affirm_1.default)(list, 'Invalid array.');
55
- Affirm_1.default.hasItems(list, 'Array must be non-empty.');
56
- (0, Affirm_1.default)(direction, 'Invalid direction parameter. Must be "asc" or "desc".');
57
- const sortedList = [...list];
58
- if (key) {
59
- sortedList.sort((a, b) => {
60
- const aValue = a[key];
61
- const bValue = b[key];
62
- if (aValue < bValue)
63
- return direction === 'asc' ? -1 : 1;
64
- if (aValue > bValue)
65
- return direction === 'asc' ? 1 : -1;
66
- return 0;
67
- });
68
- return sortedList;
69
- }
70
- else {
71
- sortedList.sort((a, b) => {
72
- if (a < b)
73
- return direction === 'asc' ? -1 : 1;
74
- if (a > b)
75
- return direction === 'asc' ? 1 : -1;
76
- return 0;
77
- });
78
- return sortedList;
79
- }
80
- },
81
- groupBy: (array, key) => {
82
- Affirm_1.default.hasItems(array, 'Array must be non-empty');
83
- (0, Affirm_1.default)(key, 'key is invalid');
84
- const result = new Map();
85
- for (let i = 0; i < array.length; i++) {
86
- const item = array[i];
87
- const keyVal = item[key];
88
- if (result.has(keyVal)) {
89
- const current = result.get(keyVal);
90
- current.push(item);
91
- result.set(keyVal, current);
92
- }
93
- else {
94
- result.set(keyVal, [item]);
95
- }
96
- }
97
- return result;
98
- },
99
- chunkString: (text, chunkSize) => {
100
- (0, Affirm_1.default)(text, 'Invalid text');
101
- (0, Affirm_1.default)(chunkSize, 'Invalid chunk size');
102
- (0, Affirm_1.default)(text.length > 0, 'Text length must be greater than 0');
103
- (0, Affirm_1.default)(chunkSize > 0, 'Chunk size must be greater than 0');
104
- if (text.length <= chunkSize)
105
- return [text];
106
- const numChunks = Math.ceil(text.length / chunkSize);
107
- const chunks = new Array(numChunks);
108
- for (let i = 0; i < chunks.length; i++) {
109
- chunks[i] = text.substring(i * chunkSize, chunkSize);
110
- }
111
- return chunks;
112
- },
113
- first: (arr) => {
114
- (0, Affirm_1.default)(algo.hasVal(arr), 'Array must not be null or undefined');
115
- Affirm_1.default.hasItems(arr, 'Array must be non-empty');
116
- return arr[0];
117
- },
118
- last: (arr) => {
119
- (0, Affirm_1.default)(algo.hasVal(arr), 'Array must not be null or undefined');
120
- (0, Affirm_1.default)(arr.length > 0, 'Array must be non-empty');
121
- return arr[arr.length - 1];
122
- },
123
- mean: (numbers) => {
124
- (0, Affirm_1.default)(algo.hasVal(numbers), 'Array must not be null or undefined');
125
- if (numbers.length === 0)
126
- return 0;
127
- const total = algo.sum(numbers);
128
- return total / numbers.length;
129
- },
130
- sum: (numbers) => {
131
- (0, Affirm_1.default)(algo.hasVal(numbers), 'Array must not be null or undefined');
132
- if (numbers.length === 0)
133
- return 0;
134
- let total = 0;
135
- for (let i = 0; i < numbers.length; i++) {
136
- total += numbers[i];
137
- }
138
- return total;
139
- },
140
- round: (number, precision) => {
141
- (0, Affirm_1.default)(algo.hasVal(number), 'Number must not be null or undefined');
142
- const factor = Math.pow(10, precision !== null && precision !== void 0 ? precision : 2);
143
- return Math.round(number * factor) / factor;
144
- },
145
- min: (arr) => {
146
- (0, Affirm_1.default)(algo.hasVal(arr), 'Array must not be null or undefined');
147
- if (arr.length === 0)
148
- return 0;
149
- return Math.min(...arr);
150
- },
151
- max: (arr) => {
152
- (0, Affirm_1.default)(algo.hasVal(arr), 'Array must not be null or undefined');
153
- if (arr.length === 0)
154
- return 0;
155
- return Math.max(...arr);
156
- },
157
- replaceAll: (text, search, replace) => text.replace(new RegExp(search, 'g'), replace),
158
- deepClone: (data) => JSON.parse(JSON.stringify(data))
159
- };
160
- exports.default = algo;
package/core/dste/DSTE.js DELETED
@@ -1,113 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || (function () {
19
- var ownKeys = function(o) {
20
- ownKeys = Object.getOwnPropertyNames || function (o) {
21
- var ar = [];
22
- for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
- return ar;
24
- };
25
- return ownKeys(o);
26
- };
27
- return function (mod) {
28
- if (mod && mod.__esModule) return mod;
29
- var result = {};
30
- if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
- __setModuleDefault(result, mod);
32
- return result;
33
- };
34
- })();
35
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
36
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
37
- return new (P || (P = Promise))(function (resolve, reject) {
38
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
39
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
40
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
41
- step((generator = generator.apply(thisArg, _arguments || [])).next());
42
- });
43
- };
44
- var __importDefault = (this && this.__importDefault) || function (mod) {
45
- return (mod && mod.__esModule) ? mod : { "default": mod };
46
- };
47
- Object.defineProperty(exports, "__esModule", { value: true });
48
- const child_process_1 = require("child_process");
49
- const Affirm_1 = __importDefault(require("../Affirm"));
50
- const fs = __importStar(require("fs"));
51
- const path_1 = __importDefault(require("path"));
52
- /**
53
- * This class is used to abstract away all calls that are non-deterministic.
54
- * As of now they just return non-deterministic values, but in the future the implementation can be easily replaced with deterministic code
55
- */
56
- class DeterministicSimulationTestingEngine {
57
- constructor() {
58
- this.init = (options) => {
59
- (0, Affirm_1.default)(options, 'Invalid options');
60
- this._options = options;
61
- };
62
- /**
63
- * returns a random number between 0 and 1
64
- */
65
- this.random = () => Math.random();
66
- this.now = () => new Date();
67
- this.simulate = () => {
68
- const dir = __dirname;
69
- const root = path_1.default.join(dir, '..', '..', '..');
70
- const folders = fs.readdirSync(root);
71
- const testFolder = folders.find(x => x === '_tests');
72
- if (testFolder) {
73
- const baseCommand = `npx tsx ./${testFolder}`;
74
- const testFiles = fs.readdirSync(testFolder);
75
- for (let i = 0; i < testFiles.length; i++) {
76
- const testFile = testFiles[i];
77
- const command = `${baseCommand}/${testFile}`;
78
- console.log('starting command', command);
79
- const tt = (0, child_process_1.execSync)(command);
80
- console.log('completed command', tt);
81
- // TODO: I can't get the output back from this execution and not even the console gets logged
82
- }
83
- }
84
- };
85
- this.tests = (tests) => __awaiter(this, void 0, void 0, function* () {
86
- const results = yield Promise.all(tests);
87
- console.log(results.map(x => `- ${x.message}`).join('\n'));
88
- const count = results.length;
89
- const successCount = results.filter(x => x.success).length;
90
- const failCount = results.filter(x => !x.success).length;
91
- console.log(`\nTests summary:\n- test count: ${count}\n- successes: \x1b[42m${successCount}\x1b[0m\n- failures: \x1b[41m${failCount}\x1b[0m`);
92
- });
93
- this.test = (name, fn) => __awaiter(this, void 0, void 0, function* () {
94
- try {
95
- const res = yield Promise.resolve(fn());
96
- if (!res)
97
- return { message: `test "${name}"\x1b[41m failed.\x1b[0m`, success: false };
98
- else
99
- return { message: `test "${name}"\x1b[42m succeded.\x1b[0m`, success: true };
100
- }
101
- catch (error) {
102
- const myerror = error;
103
- return { message: `test "${name}"\x1b[41m crashed\x1b[0m with message: ${myerror.message}; trace: ${myerror.stack}`, success: false };
104
- }
105
- });
106
- }
107
- }
108
- /**
109
- * Deterministic Simulation Testing Engine
110
- */
111
- const DSTE = new DeterministicSimulationTestingEngine();
112
- DSTE.init({ seed: 0 }); // this is going to bite me in the butt...
113
- exports.default = DSTE;
@@ -1,48 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- class DebugLogService {
4
- constructor() {
5
- this.id = 'DevelopmentLogService';
6
- this._folder = 'local_logs';
7
- this._file = 'logs.txt';
8
- this._path = this._folder + '/' + this._file;
9
- this.init = () => {
10
- return null;
11
- };
12
- this.log = (text) => {
13
- try {
14
- console.log(text);
15
- }
16
- catch (error) {
17
- console.error('ERROR on log', error);
18
- }
19
- finally {
20
- return null;
21
- }
22
- };
23
- this.info = (text) => {
24
- try {
25
- console.info(text);
26
- }
27
- catch (error) {
28
- console.error('ERROR on info', error);
29
- }
30
- finally {
31
- return null;
32
- }
33
- };
34
- this.error = (error) => {
35
- try {
36
- console.error(error);
37
- }
38
- catch (error) {
39
- console.error('ERROR on error', error);
40
- }
41
- finally {
42
- return null;
43
- }
44
- };
45
- }
46
- }
47
- const DebugLogger = new DebugLogService();
48
- exports.default = DebugLogger;