@forzalabs/remora 0.0.20 → 0.0.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Constants.js +1 -1
- package/actions/automap.js +5 -1
- package/actions/deploy.js +0 -1
- package/actions/run.js +1 -1
- package/auth/JWTManager.js +1 -1
- package/database/DatabaseEngine.js +13 -3
- package/definitions/json_schemas/consumer-schema.json +392 -0
- package/definitions/json_schemas/producer-schema.json +4 -0
- package/definitions/transform/Transformations.js +2 -0
- package/engines/DataframeManager.js +55 -0
- package/engines/Environment.js +1 -1
- package/engines/UsageDataManager.js +110 -0
- package/engines/UserManager.js +2 -2
- package/engines/ai/AutoMapperEngine.js +2 -2
- package/engines/ai/LLM.js +51 -26
- package/engines/consumer/ConsumerManager.js +2 -1
- package/engines/execution/ExecutionEnvironment.js +8 -1
- package/engines/execution/ExecutionPlanner.js +6 -2
- package/engines/transform/TransformationEngine.js +220 -0
- package/engines/transform/TypeCaster.js +33 -0
- package/engines/validation/Validator.js +22 -5
- package/helper/Helper.js +7 -0
- package/index.js +7 -0
- package/licencing/LicenceManager.js +64 -0
- package/package.json +1 -1
package/Constants.js
CHANGED
package/actions/automap.js
CHANGED
|
@@ -33,6 +33,10 @@ const automap = (producerName, schemaNames) => __awaiter(void 0, void 0, void 0,
|
|
|
33
33
|
if (!producer) {
|
|
34
34
|
throw new Error(`Producer ${producerName} not found`);
|
|
35
35
|
}
|
|
36
|
+
const source = Environment_1.default.getSource(producer.source);
|
|
37
|
+
if (!source) {
|
|
38
|
+
throw new Error(`Source ${producer.source} not found`);
|
|
39
|
+
}
|
|
36
40
|
// Get the specified schemas
|
|
37
41
|
const schemas = [];
|
|
38
42
|
for (const schemaName of schemaNames) {
|
|
@@ -47,7 +51,7 @@ const automap = (producerName, schemaNames) => __awaiter(void 0, void 0, void 0,
|
|
|
47
51
|
// Convert sample data to strings for AutoMapperEngine
|
|
48
52
|
const sampleStrings = sampleData.map(item => JSON.stringify(item));
|
|
49
53
|
// Call the automapper
|
|
50
|
-
const mapResult = yield AutoMapperEngine_1.default.map(sampleStrings, schemas);
|
|
54
|
+
const mapResult = yield AutoMapperEngine_1.default.map(sampleStrings, schemas, producer.settings.fileKey, [source]);
|
|
51
55
|
// Create the producers based on the mapping
|
|
52
56
|
for (const producer of mapResult.producers) {
|
|
53
57
|
const producerPath = path_1.default.join('remora/producers', `${producer.name}.json`);
|
package/actions/deploy.js
CHANGED
|
@@ -59,7 +59,6 @@ const deploy = (options) => __awaiter(void 0, void 0, void 0, function* () {
|
|
|
59
59
|
const apiKey = process.env.REMORA_LICENCE_KEY;
|
|
60
60
|
if (!apiKey)
|
|
61
61
|
throw new Error('REMORA_LICENCE_KEY environment variable is not set');
|
|
62
|
-
console.log('workerAPI', workerAPI);
|
|
63
62
|
const response = yield fetch(workerAPI, {
|
|
64
63
|
method: 'POST',
|
|
65
64
|
headers: {
|
package/actions/run.js
CHANGED
|
@@ -49,7 +49,7 @@ const run = (consumerName) => __awaiter(void 0, void 0, void 0, function* () {
|
|
|
49
49
|
if (response.fileUri)
|
|
50
50
|
console.log(chalk_1.default.green(`• Consumer ${consumer.name} -> ${response.fileUri}`));
|
|
51
51
|
else
|
|
52
|
-
console.log(chalk_1.default.green(`• Consumer ${consumer.name} -> ${response.data.slice(0, 5).join('\n')}
|
|
52
|
+
console.log(chalk_1.default.green(`• Consumer ${consumer.name} -> \n${response.data.slice(0, 5).map(x => JSON.stringify(x)).join('\n')}\n and ${response.data.length - 5} more...`));
|
|
53
53
|
}
|
|
54
54
|
else {
|
|
55
55
|
console.log(chalk_1.default.red(`• Consumer ${consumer.name} -> Failed: ${error}`));
|
package/auth/JWTManager.js
CHANGED
|
@@ -46,7 +46,7 @@ class JWTManagerClass {
|
|
|
46
46
|
this.verifyCLI = (token) => {
|
|
47
47
|
(0, Affirm_1.default)(token, 'Invalid token');
|
|
48
48
|
// ⚠️ ATTENTION!! The remoteSecretKey is saved in clear text because we are studying a way to not make it visible to the customer. ATTENTION!!
|
|
49
|
-
const REMORA_KEY_SECRET =
|
|
49
|
+
const REMORA_KEY_SECRET = process.env.REMORA_KEY_SECRET;
|
|
50
50
|
(0, Affirm_1.default)(REMORA_KEY_SECRET, 'Invalid CLI JWT SECRET');
|
|
51
51
|
return jsonwebtoken_1.default.verify(token, REMORA_KEY_SECRET);
|
|
52
52
|
};
|
|
@@ -13,8 +13,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
13
13
|
};
|
|
14
14
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
15
|
const mongodb_1 = require("mongodb");
|
|
16
|
-
const Settings_1 = __importDefault(require("../helper/Settings"));
|
|
17
16
|
const Helper_1 = __importDefault(require("../helper/Helper"));
|
|
17
|
+
const Settings_1 = __importDefault(require("../helper/Settings"));
|
|
18
18
|
class DatabaseEngineClass {
|
|
19
19
|
constructor() {
|
|
20
20
|
this.MAX_TRY_CONNECTION = 3;
|
|
@@ -47,10 +47,10 @@ class DatabaseEngineClass {
|
|
|
47
47
|
console.error('Error disconnecting from MongoDB:', error);
|
|
48
48
|
}
|
|
49
49
|
});
|
|
50
|
-
this.query = (collectionName,
|
|
50
|
+
this.query = (collectionName, filter, options) => __awaiter(this, void 0, void 0, function* () {
|
|
51
51
|
try {
|
|
52
52
|
const collection = this._db.collection(collectionName);
|
|
53
|
-
const result = yield collection.find(
|
|
53
|
+
const result = yield collection.find(filter, options).toArray();
|
|
54
54
|
return result;
|
|
55
55
|
}
|
|
56
56
|
catch (error) {
|
|
@@ -58,6 +58,16 @@ class DatabaseEngineClass {
|
|
|
58
58
|
throw error;
|
|
59
59
|
}
|
|
60
60
|
});
|
|
61
|
+
this.aggregate = (collectionName, aggregation) => __awaiter(this, void 0, void 0, function* () {
|
|
62
|
+
try {
|
|
63
|
+
const collection = this._db.collection(collectionName);
|
|
64
|
+
return yield collection.aggregate(aggregation).toArray();
|
|
65
|
+
}
|
|
66
|
+
catch (error) {
|
|
67
|
+
console.error('Error aggregate documents:', error);
|
|
68
|
+
throw error;
|
|
69
|
+
}
|
|
70
|
+
});
|
|
61
71
|
this.get = (collectionName, id) => __awaiter(this, void 0, void 0, function* () {
|
|
62
72
|
try {
|
|
63
73
|
const collection = this._db.collection(collectionName);
|
|
@@ -127,6 +127,52 @@
|
|
|
127
127
|
"subFields"
|
|
128
128
|
],
|
|
129
129
|
"additionalProperties": false
|
|
130
|
+
},
|
|
131
|
+
"transform": {
|
|
132
|
+
"description": "Optional list of transformations to apply to the dataset before returning the data",
|
|
133
|
+
"oneOf": [
|
|
134
|
+
{
|
|
135
|
+
"$ref": "#/definitions/singleTransformation"
|
|
136
|
+
},
|
|
137
|
+
{
|
|
138
|
+
"type": "array",
|
|
139
|
+
"items": {
|
|
140
|
+
"$ref": "#/definitions/singleTransformation"
|
|
141
|
+
},
|
|
142
|
+
"description": "A list of transformations to be applied in sequence"
|
|
143
|
+
}
|
|
144
|
+
]
|
|
145
|
+
},
|
|
146
|
+
"validate": {
|
|
147
|
+
"type": "object",
|
|
148
|
+
"description": "Rules to check field value compliance and data quality",
|
|
149
|
+
"properties": {
|
|
150
|
+
"min": {
|
|
151
|
+
"type": "number",
|
|
152
|
+
"description": "Minimum value for numeric fields"
|
|
153
|
+
},
|
|
154
|
+
"max": {
|
|
155
|
+
"type": "number",
|
|
156
|
+
"description": "Maximum value for numeric fields"
|
|
157
|
+
},
|
|
158
|
+
"regex": {
|
|
159
|
+
"type": "string",
|
|
160
|
+
"description": "Regular expression pattern to validate string fields"
|
|
161
|
+
},
|
|
162
|
+
"required": {
|
|
163
|
+
"type": "boolean",
|
|
164
|
+
"description": "Whether the field is required"
|
|
165
|
+
}
|
|
166
|
+
},
|
|
167
|
+
"additionalProperties": false
|
|
168
|
+
},
|
|
169
|
+
"onError": {
|
|
170
|
+
"type": "string",
|
|
171
|
+
"description": "Action to take if an error occurs during transformations or validation",
|
|
172
|
+
"enum": ["set_default", "skip", "fail"]
|
|
173
|
+
},
|
|
174
|
+
"default": {
|
|
175
|
+
"description": "Default value of the field if it is missing (or on error if specified)"
|
|
130
176
|
}
|
|
131
177
|
},
|
|
132
178
|
"required": [
|
|
@@ -325,12 +371,358 @@
|
|
|
325
371
|
"subFields"
|
|
326
372
|
],
|
|
327
373
|
"additionalProperties": false
|
|
374
|
+
},
|
|
375
|
+
"transform": {
|
|
376
|
+
"description": "Optional list of transformations to apply to the dataset before returning the data",
|
|
377
|
+
"oneOf": [
|
|
378
|
+
{
|
|
379
|
+
"$ref": "#/definitions/singleTransformation"
|
|
380
|
+
},
|
|
381
|
+
{
|
|
382
|
+
"type": "array",
|
|
383
|
+
"items": {
|
|
384
|
+
"$ref": "#/definitions/singleTransformation"
|
|
385
|
+
},
|
|
386
|
+
"description": "A list of transformations to be applied in sequence"
|
|
387
|
+
}
|
|
388
|
+
]
|
|
389
|
+
},
|
|
390
|
+
"validate": {
|
|
391
|
+
"type": "object",
|
|
392
|
+
"description": "Rules to check field value compliance and data quality",
|
|
393
|
+
"properties": {
|
|
394
|
+
"min": {
|
|
395
|
+
"type": "number",
|
|
396
|
+
"description": "Minimum value for numeric fields"
|
|
397
|
+
},
|
|
398
|
+
"max": {
|
|
399
|
+
"type": "number",
|
|
400
|
+
"description": "Maximum value for numeric fields"
|
|
401
|
+
},
|
|
402
|
+
"regex": {
|
|
403
|
+
"type": "string",
|
|
404
|
+
"description": "Regular expression pattern to validate string fields"
|
|
405
|
+
},
|
|
406
|
+
"required": {
|
|
407
|
+
"type": "boolean",
|
|
408
|
+
"description": "Whether the field is required"
|
|
409
|
+
}
|
|
410
|
+
},
|
|
411
|
+
"additionalProperties": false
|
|
412
|
+
},
|
|
413
|
+
"onError": {
|
|
414
|
+
"type": "string",
|
|
415
|
+
"description": "Action to take if an error occurs during transformations or validation",
|
|
416
|
+
"enum": ["set_default", "skip", "fail"]
|
|
417
|
+
},
|
|
418
|
+
"default": {
|
|
419
|
+
"description": "Default value of the field if it is missing (or on error if specified)"
|
|
328
420
|
}
|
|
329
421
|
},
|
|
330
422
|
"required": [
|
|
331
423
|
"key"
|
|
332
424
|
],
|
|
333
425
|
"additionalProperties": false
|
|
426
|
+
},
|
|
427
|
+
"singleTransformation": {
|
|
428
|
+
"oneOf": [
|
|
429
|
+
{
|
|
430
|
+
"type": "object",
|
|
431
|
+
"properties": {
|
|
432
|
+
"cast": {
|
|
433
|
+
"type": "string",
|
|
434
|
+
"description": "Cast the value to a specific type",
|
|
435
|
+
"enum": ["string", "number", "date", "boolean"]
|
|
436
|
+
}
|
|
437
|
+
},
|
|
438
|
+
"required": ["cast"],
|
|
439
|
+
"additionalProperties": false
|
|
440
|
+
},
|
|
441
|
+
{
|
|
442
|
+
"type": "object",
|
|
443
|
+
"properties": {
|
|
444
|
+
"multiply": {
|
|
445
|
+
"type": "number",
|
|
446
|
+
"description": "Multiply the numeric value by this factor"
|
|
447
|
+
}
|
|
448
|
+
},
|
|
449
|
+
"required": ["multiply"],
|
|
450
|
+
"additionalProperties": false
|
|
451
|
+
},
|
|
452
|
+
{
|
|
453
|
+
"type": "object",
|
|
454
|
+
"properties": {
|
|
455
|
+
"add": {
|
|
456
|
+
"type": "number",
|
|
457
|
+
"description": "Add this number to the numeric value"
|
|
458
|
+
}
|
|
459
|
+
},
|
|
460
|
+
"required": ["add"],
|
|
461
|
+
"additionalProperties": false
|
|
462
|
+
},
|
|
463
|
+
{
|
|
464
|
+
"type": "object",
|
|
465
|
+
"properties": {
|
|
466
|
+
"extract": {
|
|
467
|
+
"type": "string",
|
|
468
|
+
"enum": ["year", "month", "day", "hour", "minute"],
|
|
469
|
+
"description": "Extract a component from a date value"
|
|
470
|
+
}
|
|
471
|
+
},
|
|
472
|
+
"required": ["extract"],
|
|
473
|
+
"additionalProperties": false
|
|
474
|
+
},
|
|
475
|
+
{
|
|
476
|
+
"type": "object",
|
|
477
|
+
"properties": {
|
|
478
|
+
"concat": {
|
|
479
|
+
"type": "object",
|
|
480
|
+
"properties": {
|
|
481
|
+
"separator": {
|
|
482
|
+
"type": "string",
|
|
483
|
+
"description": "The separator to use when joining array elements"
|
|
484
|
+
}
|
|
485
|
+
},
|
|
486
|
+
"required": ["separator"],
|
|
487
|
+
"additionalProperties": false
|
|
488
|
+
}
|
|
489
|
+
},
|
|
490
|
+
"required": ["concat"],
|
|
491
|
+
"additionalProperties": false
|
|
492
|
+
},
|
|
493
|
+
{
|
|
494
|
+
"type": "object",
|
|
495
|
+
"properties": {
|
|
496
|
+
"split": {
|
|
497
|
+
"type": "object",
|
|
498
|
+
"properties": {
|
|
499
|
+
"separator": {
|
|
500
|
+
"type": "string",
|
|
501
|
+
"description": "The separator to use when splitting the string"
|
|
502
|
+
},
|
|
503
|
+
"index": {
|
|
504
|
+
"type": "number",
|
|
505
|
+
"description": "The index of the split part to keep"
|
|
506
|
+
}
|
|
507
|
+
},
|
|
508
|
+
"required": ["separator", "index"],
|
|
509
|
+
"additionalProperties": false
|
|
510
|
+
}
|
|
511
|
+
},
|
|
512
|
+
"required": ["split"],
|
|
513
|
+
"additionalProperties": false
|
|
514
|
+
},
|
|
515
|
+
{
|
|
516
|
+
"type": "object",
|
|
517
|
+
"properties": {
|
|
518
|
+
"regex_match": {
|
|
519
|
+
"type": "object",
|
|
520
|
+
"properties": {
|
|
521
|
+
"pattern": {
|
|
522
|
+
"type": "string",
|
|
523
|
+
"description": "The regex pattern to test against"
|
|
524
|
+
},
|
|
525
|
+
"flags": {
|
|
526
|
+
"type": "string",
|
|
527
|
+
"description": "Regex flags (e.g., 'i' for case-insensitive)"
|
|
528
|
+
}
|
|
529
|
+
},
|
|
530
|
+
"required": ["pattern"],
|
|
531
|
+
"additionalProperties": false
|
|
532
|
+
}
|
|
533
|
+
},
|
|
534
|
+
"required": ["regex_match"],
|
|
535
|
+
"additionalProperties": false
|
|
536
|
+
},
|
|
537
|
+
{
|
|
538
|
+
"type": "object",
|
|
539
|
+
"properties": {
|
|
540
|
+
"regex_replace": {
|
|
541
|
+
"type": "object",
|
|
542
|
+
"properties": {
|
|
543
|
+
"pattern": {
|
|
544
|
+
"type": "string",
|
|
545
|
+
"description": "The regex pattern to match"
|
|
546
|
+
},
|
|
547
|
+
"replacement": {
|
|
548
|
+
"type": "string",
|
|
549
|
+
"description": "The replacement string"
|
|
550
|
+
},
|
|
551
|
+
"flags": {
|
|
552
|
+
"type": "string",
|
|
553
|
+
"description": "Regex flags (e.g., 'g' for global)"
|
|
554
|
+
}
|
|
555
|
+
},
|
|
556
|
+
"required": ["pattern", "replacement"],
|
|
557
|
+
"additionalProperties": false
|
|
558
|
+
}
|
|
559
|
+
},
|
|
560
|
+
"required": ["regex_replace"],
|
|
561
|
+
"additionalProperties": false
|
|
562
|
+
},
|
|
563
|
+
{
|
|
564
|
+
"type": "object",
|
|
565
|
+
"properties": {
|
|
566
|
+
"regex_extract": {
|
|
567
|
+
"type": "object",
|
|
568
|
+
"properties": {
|
|
569
|
+
"pattern": {
|
|
570
|
+
"type": "string",
|
|
571
|
+
"description": "The regex pattern to extract with"
|
|
572
|
+
},
|
|
573
|
+
"group": {
|
|
574
|
+
"type": "number",
|
|
575
|
+
"description": "The capture group index to extract (0 for full match)"
|
|
576
|
+
},
|
|
577
|
+
"flags": {
|
|
578
|
+
"type": "string",
|
|
579
|
+
"description": "Regex flags (e.g., 'i' for case-insensitive)"
|
|
580
|
+
}
|
|
581
|
+
},
|
|
582
|
+
"required": ["pattern", "group"],
|
|
583
|
+
"additionalProperties": false
|
|
584
|
+
}
|
|
585
|
+
},
|
|
586
|
+
"required": ["regex_extract"],
|
|
587
|
+
"additionalProperties": false
|
|
588
|
+
},
|
|
589
|
+
{
|
|
590
|
+
"type": "object",
|
|
591
|
+
"properties": {
|
|
592
|
+
"trim": {
|
|
593
|
+
"type": "boolean",
|
|
594
|
+
"description": "Trim whitespace from both ends of the string"
|
|
595
|
+
}
|
|
596
|
+
},
|
|
597
|
+
"required": ["trim"],
|
|
598
|
+
"additionalProperties": false
|
|
599
|
+
},
|
|
600
|
+
{
|
|
601
|
+
"type": "object",
|
|
602
|
+
"properties": {
|
|
603
|
+
"to_lowercase": {
|
|
604
|
+
"type": "boolean",
|
|
605
|
+
"description": "Convert string to lowercase"
|
|
606
|
+
}
|
|
607
|
+
},
|
|
608
|
+
"required": ["to_lowercase"],
|
|
609
|
+
"additionalProperties": false
|
|
610
|
+
},
|
|
611
|
+
{
|
|
612
|
+
"type": "object",
|
|
613
|
+
"properties": {
|
|
614
|
+
"to_uppercase": {
|
|
615
|
+
"type": "boolean",
|
|
616
|
+
"description": "Convert string to uppercase"
|
|
617
|
+
}
|
|
618
|
+
},
|
|
619
|
+
"required": ["to_uppercase"],
|
|
620
|
+
"additionalProperties": false
|
|
621
|
+
},
|
|
622
|
+
{
|
|
623
|
+
"type": "object",
|
|
624
|
+
"properties": {
|
|
625
|
+
"capitalize": {
|
|
626
|
+
"type": "boolean",
|
|
627
|
+
"description": "Capitalize the first letter of the string"
|
|
628
|
+
}
|
|
629
|
+
},
|
|
630
|
+
"required": ["capitalize"],
|
|
631
|
+
"additionalProperties": false
|
|
632
|
+
},
|
|
633
|
+
{
|
|
634
|
+
"type": "object",
|
|
635
|
+
"properties": {
|
|
636
|
+
"substring": {
|
|
637
|
+
"type": "object",
|
|
638
|
+
"properties": {
|
|
639
|
+
"start": {
|
|
640
|
+
"type": "number",
|
|
641
|
+
"description": "Starting position of substring"
|
|
642
|
+
},
|
|
643
|
+
"end": {
|
|
644
|
+
"type": "number",
|
|
645
|
+
"description": "Optional ending position of substring"
|
|
646
|
+
}
|
|
647
|
+
},
|
|
648
|
+
"required": ["start"],
|
|
649
|
+
"additionalProperties": false
|
|
650
|
+
}
|
|
651
|
+
},
|
|
652
|
+
"required": ["substring"],
|
|
653
|
+
"additionalProperties": false
|
|
654
|
+
},
|
|
655
|
+
{
|
|
656
|
+
"type": "object",
|
|
657
|
+
"properties": {
|
|
658
|
+
"pad_start": {
|
|
659
|
+
"type": "object",
|
|
660
|
+
"properties": {
|
|
661
|
+
"length": {
|
|
662
|
+
"type": "number",
|
|
663
|
+
"description": "Desired string length after padding"
|
|
664
|
+
},
|
|
665
|
+
"char": {
|
|
666
|
+
"type": "string",
|
|
667
|
+
"description": "Character to pad with",
|
|
668
|
+
"minLength": 1,
|
|
669
|
+
"maxLength": 1
|
|
670
|
+
}
|
|
671
|
+
},
|
|
672
|
+
"required": ["length", "char"],
|
|
673
|
+
"additionalProperties": false
|
|
674
|
+
}
|
|
675
|
+
},
|
|
676
|
+
"required": ["pad_start"],
|
|
677
|
+
"additionalProperties": false
|
|
678
|
+
},
|
|
679
|
+
{
|
|
680
|
+
"type": "object",
|
|
681
|
+
"properties": {
|
|
682
|
+
"pad_end": {
|
|
683
|
+
"type": "object",
|
|
684
|
+
"properties": {
|
|
685
|
+
"length": {
|
|
686
|
+
"type": "number",
|
|
687
|
+
"description": "Desired string length after padding"
|
|
688
|
+
},
|
|
689
|
+
"char": {
|
|
690
|
+
"type": "string",
|
|
691
|
+
"description": "Character to pad with",
|
|
692
|
+
"minLength": 1,
|
|
693
|
+
"maxLength": 1
|
|
694
|
+
}
|
|
695
|
+
},
|
|
696
|
+
"required": ["length", "char"],
|
|
697
|
+
"additionalProperties": false
|
|
698
|
+
}
|
|
699
|
+
},
|
|
700
|
+
"required": ["pad_end"],
|
|
701
|
+
"additionalProperties": false
|
|
702
|
+
},
|
|
703
|
+
{
|
|
704
|
+
"type": "object",
|
|
705
|
+
"properties": {
|
|
706
|
+
"prepend": {
|
|
707
|
+
"type": "string",
|
|
708
|
+
"description": "String to add at the beginning"
|
|
709
|
+
}
|
|
710
|
+
},
|
|
711
|
+
"required": ["prepend"],
|
|
712
|
+
"additionalProperties": false
|
|
713
|
+
},
|
|
714
|
+
{
|
|
715
|
+
"type": "object",
|
|
716
|
+
"properties": {
|
|
717
|
+
"append": {
|
|
718
|
+
"type": "string",
|
|
719
|
+
"description": "String to add at the end"
|
|
720
|
+
}
|
|
721
|
+
},
|
|
722
|
+
"required": ["append"],
|
|
723
|
+
"additionalProperties": false
|
|
724
|
+
}
|
|
725
|
+
]
|
|
334
726
|
}
|
|
335
727
|
},
|
|
336
728
|
"examples": [
|
|
@@ -135,6 +135,10 @@
|
|
|
135
135
|
"CSV"
|
|
136
136
|
],
|
|
137
137
|
"description": "The type of file to read"
|
|
138
|
+
},
|
|
139
|
+
"delimiter": {
|
|
140
|
+
"type": "string",
|
|
141
|
+
"description": "The column delimiter for the CSV file if different from the default (,)."
|
|
138
142
|
}
|
|
139
143
|
},
|
|
140
144
|
"additionalProperties": false
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const Algo_1 = __importDefault(require("../core/Algo"));
|
|
7
|
+
const Helper_1 = __importDefault(require("../helper/Helper"));
|
|
8
|
+
class DataframeManagerClass {
|
|
9
|
+
fill(points, from, to, onlyLastValue, maintainLastValue) {
|
|
10
|
+
const min = from !== null && from !== void 0 ? from : this.getMinDate(points);
|
|
11
|
+
const max = to !== null && to !== void 0 ? to : this.getMaxDate(points);
|
|
12
|
+
const orderPoints = points.length > 0 ? Algo_1.default.orderBy(points, 'x') : [];
|
|
13
|
+
const filledPoints = [];
|
|
14
|
+
const currentDate = new Date(min);
|
|
15
|
+
while (currentDate <= max) {
|
|
16
|
+
const monthKey = Helper_1.default.formatDateToYYYYMM(currentDate);
|
|
17
|
+
filledPoints.push({ x: monthKey, y: 0 });
|
|
18
|
+
currentDate.setMonth(currentDate.getMonth() + 1);
|
|
19
|
+
}
|
|
20
|
+
for (let i = 0; i < orderPoints.length; i++) {
|
|
21
|
+
const point = orderPoints[i];
|
|
22
|
+
const date = new Date(point.x);
|
|
23
|
+
const filledPoint = filledPoints.find(x => x.x === Helper_1.default.formatDateToYYYYMM(date));
|
|
24
|
+
if (filledPoint) {
|
|
25
|
+
if (!onlyLastValue)
|
|
26
|
+
filledPoint.y += point.y;
|
|
27
|
+
else
|
|
28
|
+
filledPoint.y = point.y;
|
|
29
|
+
if (maintainLastValue) {
|
|
30
|
+
const index = filledPoints.findIndex(x => x.x === Helper_1.default.formatDateToYYYYMM(date));
|
|
31
|
+
for (let k = index; k < filledPoints.length; k++) {
|
|
32
|
+
const nextFilledPoint = filledPoints[k];
|
|
33
|
+
nextFilledPoint.y = filledPoint.y;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
return filledPoints;
|
|
39
|
+
}
|
|
40
|
+
getMinDate(points) {
|
|
41
|
+
if (!points || points.length === 0) {
|
|
42
|
+
const currentDate = new Date();
|
|
43
|
+
return new Date(currentDate.getFullYear() - 1, currentDate.getMonth(), currentDate.getDate());
|
|
44
|
+
}
|
|
45
|
+
return points.reduce((min, point) => (new Date(point.x) < min ? new Date(point === null || point === void 0 ? void 0 : point.x) : min), new Date(points[0].x));
|
|
46
|
+
}
|
|
47
|
+
getMaxDate(points) {
|
|
48
|
+
if (!points || points.length === 0) {
|
|
49
|
+
return new Date();
|
|
50
|
+
}
|
|
51
|
+
return points.reduce((max, point) => (new Date(point.x) > max ? new Date(point.x) : max), new Date(points[0].x));
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
const DataframeManager = new DataframeManagerClass();
|
|
55
|
+
exports.default = DataframeManager;
|
package/engines/Environment.js
CHANGED
|
@@ -7,7 +7,7 @@ const fs_1 = __importDefault(require("fs"));
|
|
|
7
7
|
const path_1 = __importDefault(require("path"));
|
|
8
8
|
const Affirm_1 = __importDefault(require("../core/Affirm"));
|
|
9
9
|
const SchemaValidator_1 = __importDefault(require("./schema/SchemaValidator"));
|
|
10
|
-
const Validator_1 = __importDefault(require("./Validator"));
|
|
10
|
+
const Validator_1 = __importDefault(require("./validation/Validator"));
|
|
11
11
|
const Constants_1 = __importDefault(require("../Constants"));
|
|
12
12
|
class EnvironmentClass {
|
|
13
13
|
constructor() {
|