mx-cloud 0.0.19 → 0.0.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/interpret.d.ts +1 -0
- package/build/interpret.js +80 -38
- package/package.json +1 -1
package/build/interpret.d.ts
CHANGED
|
@@ -61,6 +61,7 @@ export default class Interpreter extends EventEmitter {
|
|
|
61
61
|
private autohealFailures;
|
|
62
62
|
private namedResults;
|
|
63
63
|
private screenshotCounter;
|
|
64
|
+
private scrapeListCounter;
|
|
64
65
|
private serializableDataByType;
|
|
65
66
|
constructor(workflow: WorkflowFile, options?: Partial<InterpreterOptions>);
|
|
66
67
|
trackAutohealFailure(error: string): void;
|
package/build/interpret.js
CHANGED
|
@@ -69,6 +69,7 @@ class Interpreter extends events_1.EventEmitter {
|
|
|
69
69
|
this.autohealFailures = [];
|
|
70
70
|
this.namedResults = {};
|
|
71
71
|
this.screenshotCounter = 0;
|
|
72
|
+
this.scrapeListCounter = 0;
|
|
72
73
|
this.serializableDataByType = {
|
|
73
74
|
scrapeList: {},
|
|
74
75
|
scrapeSchema: {}
|
|
@@ -360,11 +361,20 @@ class Interpreter extends events_1.EventEmitter {
|
|
|
360
361
|
this.options.debugChannel.setActionType("screenshot");
|
|
361
362
|
}
|
|
362
363
|
const screenshotBuffer = yield page.screenshot(Object.assign(Object.assign({}, params), { path: undefined }));
|
|
363
|
-
|
|
364
|
-
//
|
|
365
|
-
|
|
366
|
-
const
|
|
367
|
-
|
|
364
|
+
// Prefer explicit nameOverride (from workflow step.name or computed action name)
|
|
365
|
+
// If nameOverride is provided (non-empty) use it *as-is*.
|
|
366
|
+
// Only use counter-appended name when no nameOverride is available.
|
|
367
|
+
const explicitName = (typeof nameOverride === 'string' && nameOverride.trim().length > 0) ? nameOverride.trim() : null;
|
|
368
|
+
let screenshotName;
|
|
369
|
+
if (explicitName) {
|
|
370
|
+
screenshotName = explicitName;
|
|
371
|
+
}
|
|
372
|
+
else {
|
|
373
|
+
// If no explicit name, produce a readable generated name with a counter
|
|
374
|
+
this.screenshotCounter += 1;
|
|
375
|
+
screenshotName = `Screenshot ${this.screenshotCounter}`;
|
|
376
|
+
}
|
|
377
|
+
// Pass structured metadata (name included) to binaryCallback
|
|
368
378
|
yield this.options.binaryCallback({
|
|
369
379
|
name: screenshotName,
|
|
370
380
|
data: screenshotBuffer,
|
|
@@ -471,20 +481,13 @@ class Interpreter extends events_1.EventEmitter {
|
|
|
471
481
|
if (!this.namedResults[actionType])
|
|
472
482
|
this.namedResults[actionType] = {};
|
|
473
483
|
this.namedResults[actionType][actionName] = this.cumulativeResults;
|
|
474
|
-
if (!this.serializableDataByType[actionType])
|
|
475
|
-
this.serializableDataByType[actionType] = {};
|
|
476
484
|
if (!this.serializableDataByType[actionType])
|
|
477
485
|
this.serializableDataByType[actionType] = {};
|
|
478
486
|
if (!this.serializableDataByType[actionType][actionName]) {
|
|
479
|
-
this.serializableDataByType[actionType][actionName] =
|
|
480
|
-
}
|
|
481
|
-
for (const row of this.cumulativeResults) {
|
|
482
|
-
for (const [key, value] of Object.entries(row)) {
|
|
483
|
-
if (value !== undefined) {
|
|
484
|
-
this.serializableDataByType[actionType][actionName][key] = value;
|
|
485
|
-
}
|
|
486
|
-
}
|
|
487
|
+
this.serializableDataByType[actionType][actionName] = [];
|
|
487
488
|
}
|
|
489
|
+
// Store as array (matching cumulativeResults structure)
|
|
490
|
+
this.serializableDataByType[actionType][actionName] = [...this.cumulativeResults];
|
|
488
491
|
// now emit full structured object
|
|
489
492
|
yield this.options.serializableCallback({
|
|
490
493
|
scrapeList: this.serializableDataByType.scrapeList,
|
|
@@ -511,6 +514,7 @@ class Interpreter extends events_1.EventEmitter {
|
|
|
511
514
|
this.options.debugChannel.incrementScrapeListIndex();
|
|
512
515
|
}
|
|
513
516
|
let scrapeResults = [];
|
|
517
|
+
let paginationUsed = false;
|
|
514
518
|
if (!config.pagination) {
|
|
515
519
|
scrapeResults = yield page.evaluate((cfg) => {
|
|
516
520
|
try {
|
|
@@ -523,6 +527,7 @@ class Interpreter extends events_1.EventEmitter {
|
|
|
523
527
|
}, config);
|
|
524
528
|
}
|
|
525
529
|
else {
|
|
530
|
+
paginationUsed = true;
|
|
526
531
|
scrapeResults = yield this.handlePagination(page, config);
|
|
527
532
|
}
|
|
528
533
|
// Ensure we always have an array
|
|
@@ -530,19 +535,28 @@ class Interpreter extends events_1.EventEmitter {
|
|
|
530
535
|
scrapeResults = [];
|
|
531
536
|
}
|
|
532
537
|
console.log(`ScrapeList completed with ${scrapeResults.length} results`);
|
|
533
|
-
//
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
538
|
+
// Only process and callback if pagination wasn't used
|
|
539
|
+
// (handlePagination already handles storage and callbacks internally)
|
|
540
|
+
if (!paginationUsed) {
|
|
541
|
+
// ✅ Append list results under "scrapeList" → name
|
|
542
|
+
const actionType = "scrapeList";
|
|
543
|
+
let actionName = config.__name || "";
|
|
544
|
+
// If no name provided, generate a unique one
|
|
545
|
+
if (!actionName || actionName.trim() === "") {
|
|
546
|
+
this.scrapeListCounter++;
|
|
547
|
+
actionName = `List ${this.scrapeListCounter}`;
|
|
548
|
+
}
|
|
549
|
+
if (!this.serializableDataByType[actionType])
|
|
550
|
+
this.serializableDataByType[actionType] = {};
|
|
551
|
+
if (!this.serializableDataByType[actionType][actionName]) {
|
|
552
|
+
this.serializableDataByType[actionType][actionName] = [];
|
|
553
|
+
}
|
|
554
|
+
this.serializableDataByType[actionType][actionName].push(...scrapeResults);
|
|
555
|
+
yield this.options.serializableCallback({
|
|
556
|
+
scrapeList: this.serializableDataByType.scrapeList,
|
|
557
|
+
scrapeSchema: this.serializableDataByType.scrapeSchema
|
|
558
|
+
});
|
|
540
559
|
}
|
|
541
|
-
this.serializableDataByType[actionType][actionName].push(...scrapeResults);
|
|
542
|
-
yield this.options.serializableCallback({
|
|
543
|
-
scrapeList: this.serializableDataByType.scrapeList,
|
|
544
|
-
scrapeSchema: this.serializableDataByType.scrapeSchema
|
|
545
|
-
});
|
|
546
560
|
}
|
|
547
561
|
catch (error) {
|
|
548
562
|
console.error('ScrapeList action failed completely:', error.message);
|
|
@@ -552,7 +566,13 @@ class Interpreter extends events_1.EventEmitter {
|
|
|
552
566
|
if (!this.namedResults[actionType])
|
|
553
567
|
this.namedResults[actionType] = {};
|
|
554
568
|
this.namedResults[actionType][actionName] = [];
|
|
555
|
-
|
|
569
|
+
if (!this.serializableDataByType[actionType])
|
|
570
|
+
this.serializableDataByType[actionType] = {};
|
|
571
|
+
this.serializableDataByType[actionType][actionName] = [];
|
|
572
|
+
yield this.options.serializableCallback({
|
|
573
|
+
scrapeList: this.serializableDataByType.scrapeList,
|
|
574
|
+
scrapeSchema: this.serializableDataByType.scrapeSchema
|
|
575
|
+
});
|
|
556
576
|
}
|
|
557
577
|
}),
|
|
558
578
|
scrapeListAuto: (config) => __awaiter(this, void 0, void 0, function* () {
|
|
@@ -625,25 +645,22 @@ class Interpreter extends events_1.EventEmitter {
|
|
|
625
645
|
return;
|
|
626
646
|
}
|
|
627
647
|
this.log(`Launching ${String(step.action)}`, logger_1.Level.LOG);
|
|
648
|
+
let stepName = null;
|
|
628
649
|
try {
|
|
629
650
|
const debug = this.options.debugChannel;
|
|
630
651
|
if (debug === null || debug === void 0 ? void 0 : debug.setActionType) {
|
|
631
652
|
debug.setActionType(String(step.action));
|
|
632
653
|
}
|
|
633
654
|
// Safely extract name for this step
|
|
634
|
-
let stepName = null;
|
|
635
|
-
// If the workflow step itself has a name
|
|
636
655
|
if (step === null || step === void 0 ? void 0 : step.name) {
|
|
637
656
|
stepName = step.name;
|
|
638
657
|
}
|
|
639
|
-
// If the first arg is an object with a __name property
|
|
640
658
|
else if (Array.isArray(step === null || step === void 0 ? void 0 : step.args) &&
|
|
641
659
|
step.args.length > 0 &&
|
|
642
660
|
typeof step.args[0] === "object" &&
|
|
643
661
|
"__name" in step.args[0]) {
|
|
644
662
|
stepName = step.args[0].__name;
|
|
645
663
|
}
|
|
646
|
-
// If args itself is an object with a __name property
|
|
647
664
|
else if (typeof (step === null || step === void 0 ? void 0 : step.args) === "object" &&
|
|
648
665
|
(step === null || step === void 0 ? void 0 : step.args) !== null &&
|
|
649
666
|
"__name" in step.args) {
|
|
@@ -664,7 +681,13 @@ class Interpreter extends events_1.EventEmitter {
|
|
|
664
681
|
if (step.action in wawActions) {
|
|
665
682
|
// "Arrayifying" here should not be needed (TS + syntax checker - only arrays; but why not)
|
|
666
683
|
const params = !step.args || Array.isArray(step.args) ? step.args : [step.args];
|
|
667
|
-
|
|
684
|
+
if (step.action === 'screenshot') {
|
|
685
|
+
// call the screenshot handler directly to allow the extra name parameter
|
|
686
|
+
yield wawActions.screenshot(...(params !== null && params !== void 0 ? params : []), stepName !== null && stepName !== void 0 ? stepName : undefined);
|
|
687
|
+
}
|
|
688
|
+
else {
|
|
689
|
+
yield wawActions[step.action](...(params !== null && params !== void 0 ? params : []));
|
|
690
|
+
}
|
|
668
691
|
}
|
|
669
692
|
else {
|
|
670
693
|
if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
|
|
@@ -728,6 +751,20 @@ class Interpreter extends events_1.EventEmitter {
|
|
|
728
751
|
this.log('Workflow aborted, stopping pagination', logger_1.Level.WARN);
|
|
729
752
|
return [];
|
|
730
753
|
}
|
|
754
|
+
// Generate action name for this scrapeList
|
|
755
|
+
const actionType = "scrapeList";
|
|
756
|
+
let actionName = config.__name || "";
|
|
757
|
+
if (!actionName || actionName.trim() === "") {
|
|
758
|
+
this.scrapeListCounter++;
|
|
759
|
+
actionName = `List ${this.scrapeListCounter}`;
|
|
760
|
+
}
|
|
761
|
+
// Initialize storage for this action
|
|
762
|
+
if (!this.serializableDataByType[actionType]) {
|
|
763
|
+
this.serializableDataByType[actionType] = {};
|
|
764
|
+
}
|
|
765
|
+
if (!this.serializableDataByType[actionType][actionName]) {
|
|
766
|
+
this.serializableDataByType[actionType][actionName] = [];
|
|
767
|
+
}
|
|
731
768
|
let allResults = [];
|
|
732
769
|
let previousHeight = 0;
|
|
733
770
|
let scrapedItems = new Set();
|
|
@@ -764,7 +801,12 @@ class Interpreter extends events_1.EventEmitter {
|
|
|
764
801
|
});
|
|
765
802
|
allResults = allResults.concat(newResults);
|
|
766
803
|
debugLog("Results collected:", allResults.length);
|
|
767
|
-
|
|
804
|
+
// Store in serializableDataByType and send structured callback
|
|
805
|
+
this.serializableDataByType[actionType][actionName] = [...allResults];
|
|
806
|
+
yield this.options.serializableCallback({
|
|
807
|
+
scrapeList: this.serializableDataByType.scrapeList,
|
|
808
|
+
scrapeSchema: this.serializableDataByType.scrapeSchema
|
|
809
|
+
});
|
|
768
810
|
});
|
|
769
811
|
const checkLimit = () => {
|
|
770
812
|
if (config.limit && allResults.length >= config.limit) {
|
|
@@ -1051,7 +1093,7 @@ class Interpreter extends events_1.EventEmitter {
|
|
|
1051
1093
|
}).catch(e => {
|
|
1052
1094
|
throw e;
|
|
1053
1095
|
}),
|
|
1054
|
-
|
|
1096
|
+
page.locator(workingSelector).first().click()
|
|
1055
1097
|
]);
|
|
1056
1098
|
debugLog("Navigation successful after regular click");
|
|
1057
1099
|
yield page.waitForTimeout(2000);
|
|
@@ -1067,7 +1109,7 @@ class Interpreter extends events_1.EventEmitter {
|
|
|
1067
1109
|
}).catch(e => {
|
|
1068
1110
|
throw e;
|
|
1069
1111
|
}),
|
|
1070
|
-
|
|
1112
|
+
page.locator(workingSelector).first().dispatchEvent('click')
|
|
1071
1113
|
]);
|
|
1072
1114
|
debugLog("Navigation successful after dispatch event");
|
|
1073
1115
|
yield page.waitForTimeout(2000);
|
|
@@ -1075,11 +1117,11 @@ class Interpreter extends events_1.EventEmitter {
|
|
|
1075
1117
|
}
|
|
1076
1118
|
catch (dispatchNavError) {
|
|
1077
1119
|
try {
|
|
1078
|
-
yield
|
|
1120
|
+
yield page.locator(workingSelector).first().click();
|
|
1079
1121
|
yield page.waitForTimeout(2000);
|
|
1080
1122
|
}
|
|
1081
1123
|
catch (clickError) {
|
|
1082
|
-
yield
|
|
1124
|
+
yield page.locator(workingSelector).first().dispatchEvent('click');
|
|
1083
1125
|
yield page.waitForTimeout(2000);
|
|
1084
1126
|
}
|
|
1085
1127
|
}
|