mx-cloud 0.0.23 → 0.0.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/interpret.js +30 -60
- package/package.json +1 -1
package/build/interpret.js
CHANGED
|
@@ -430,9 +430,8 @@ class Interpreter extends events_1.EventEmitter {
|
|
|
430
430
|
const scrapeResults = yield page.evaluate((s) => window.scrape(s !== null && s !== void 0 ? s : null), selector);
|
|
431
431
|
yield this.options.serializableCallback(scrapeResults);
|
|
432
432
|
}),
|
|
433
|
-
scrapeSchema: (
|
|
433
|
+
scrapeSchema: (schema_1, ...args_1) => __awaiter(this, [schema_1, ...args_1], void 0, function* (schema, actionName = "") {
|
|
434
434
|
var _a;
|
|
435
|
-
// Check abort flag at start of scraping
|
|
436
435
|
if (this.isAborted) {
|
|
437
436
|
this.log('Workflow aborted, stopping scrapeSchema', logger_1.Level.WARN);
|
|
438
437
|
return;
|
|
@@ -451,7 +450,6 @@ class Interpreter extends events_1.EventEmitter {
|
|
|
451
450
|
}
|
|
452
451
|
const resultToProcess = Array.isArray(scrapeResult) ? scrapeResult[0] : scrapeResult;
|
|
453
452
|
if (this.cumulativeResults.length === 0) {
|
|
454
|
-
// First execution - create initial row
|
|
455
453
|
const newRow = {};
|
|
456
454
|
Object.entries(resultToProcess).forEach(([key, value]) => {
|
|
457
455
|
if (value !== undefined) {
|
|
@@ -461,12 +459,10 @@ class Interpreter extends events_1.EventEmitter {
|
|
|
461
459
|
this.cumulativeResults.push(newRow);
|
|
462
460
|
}
|
|
463
461
|
else {
|
|
464
|
-
// Check if any keys from new result already exist in the last row
|
|
465
462
|
const lastRow = this.cumulativeResults[this.cumulativeResults.length - 1];
|
|
466
463
|
const newResultKeys = Object.keys(resultToProcess).filter(key => resultToProcess[key] !== undefined);
|
|
467
464
|
const hasRepeatedKeys = newResultKeys.some(key => lastRow.hasOwnProperty(key));
|
|
468
465
|
if (hasRepeatedKeys) {
|
|
469
|
-
// Keys are repeated - create a new row
|
|
470
466
|
const newRow = {};
|
|
471
467
|
Object.entries(resultToProcess).forEach(([key, value]) => {
|
|
472
468
|
if (value !== undefined) {
|
|
@@ -476,7 +472,6 @@ class Interpreter extends events_1.EventEmitter {
|
|
|
476
472
|
this.cumulativeResults.push(newRow);
|
|
477
473
|
}
|
|
478
474
|
else {
|
|
479
|
-
// No repeated keys - merge with the last row
|
|
480
475
|
Object.entries(resultToProcess).forEach(([key, value]) => {
|
|
481
476
|
if (value !== undefined) {
|
|
482
477
|
lastRow[key] = value;
|
|
@@ -484,30 +479,24 @@ class Interpreter extends events_1.EventEmitter {
|
|
|
484
479
|
});
|
|
485
480
|
}
|
|
486
481
|
}
|
|
487
|
-
console.log("Total accumulated rows:", this.cumulativeResults.length);
|
|
488
|
-
console.log("Current results:", this.cumulativeResults);
|
|
489
|
-
// ✅ Append schema results under "scrapeSchema" → name
|
|
490
482
|
const actionType = "scrapeSchema";
|
|
491
|
-
const
|
|
483
|
+
const name = actionName || "Texts";
|
|
492
484
|
if (!this.namedResults[actionType])
|
|
493
485
|
this.namedResults[actionType] = {};
|
|
494
|
-
this.namedResults[actionType][
|
|
486
|
+
this.namedResults[actionType][name] = this.cumulativeResults;
|
|
495
487
|
if (!this.serializableDataByType[actionType])
|
|
496
488
|
this.serializableDataByType[actionType] = {};
|
|
497
|
-
if (!this.serializableDataByType[actionType][
|
|
498
|
-
this.serializableDataByType[actionType][
|
|
489
|
+
if (!this.serializableDataByType[actionType][name]) {
|
|
490
|
+
this.serializableDataByType[actionType][name] = [];
|
|
499
491
|
}
|
|
500
|
-
|
|
501
|
-
this.serializableDataByType[actionType][actionName] = [...this.cumulativeResults];
|
|
502
|
-
// now emit full structured object
|
|
492
|
+
this.serializableDataByType[actionType][name] = [...this.cumulativeResults];
|
|
503
493
|
yield this.options.serializableCallback({
|
|
504
494
|
scrapeList: this.serializableDataByType.scrapeList,
|
|
505
495
|
scrapeSchema: this.serializableDataByType.scrapeSchema
|
|
506
496
|
});
|
|
507
497
|
}),
|
|
508
|
-
scrapeList: (
|
|
498
|
+
scrapeList: (config_1, ...args_1) => __awaiter(this, [config_1, ...args_1], void 0, function* (config, actionName = "") {
|
|
509
499
|
var _a, _b;
|
|
510
|
-
// Check abort flag at start of scraping
|
|
511
500
|
if (this.isAborted) {
|
|
512
501
|
this.log('Workflow aborted, stopping scrapeList', logger_1.Level.WARN);
|
|
513
502
|
return;
|
|
@@ -533,36 +522,31 @@ class Interpreter extends events_1.EventEmitter {
|
|
|
533
522
|
}
|
|
534
523
|
catch (error) {
|
|
535
524
|
console.warn('ScrapeList evaluation failed:', error.message);
|
|
536
|
-
return [];
|
|
525
|
+
return [];
|
|
537
526
|
}
|
|
538
527
|
}, config);
|
|
539
528
|
}
|
|
540
529
|
else {
|
|
541
530
|
paginationUsed = true;
|
|
542
|
-
scrapeResults = yield this.handlePagination(page, config);
|
|
531
|
+
scrapeResults = yield this.handlePagination(page, config, actionName);
|
|
543
532
|
}
|
|
544
|
-
// Ensure we always have an array
|
|
545
533
|
if (!Array.isArray(scrapeResults)) {
|
|
546
534
|
scrapeResults = [];
|
|
547
535
|
}
|
|
548
536
|
console.log(`ScrapeList completed with ${scrapeResults.length} results`);
|
|
549
|
-
// Only process and callback if pagination wasn't used
|
|
550
|
-
// (handlePagination already handles storage and callbacks internally)
|
|
551
537
|
if (!paginationUsed) {
|
|
552
|
-
// ✅ Append list results under "scrapeList" → name
|
|
553
538
|
const actionType = "scrapeList";
|
|
554
|
-
let
|
|
555
|
-
|
|
556
|
-
if (!actionName || actionName.trim() === "") {
|
|
539
|
+
let name = actionName || "";
|
|
540
|
+
if (!name || name.trim() === "") {
|
|
557
541
|
this.scrapeListCounter++;
|
|
558
|
-
|
|
542
|
+
name = `List ${this.scrapeListCounter}`;
|
|
559
543
|
}
|
|
560
544
|
if (!this.serializableDataByType[actionType])
|
|
561
545
|
this.serializableDataByType[actionType] = {};
|
|
562
|
-
if (!this.serializableDataByType[actionType][
|
|
563
|
-
this.serializableDataByType[actionType][
|
|
546
|
+
if (!this.serializableDataByType[actionType][name]) {
|
|
547
|
+
this.serializableDataByType[actionType][name] = [];
|
|
564
548
|
}
|
|
565
|
-
this.serializableDataByType[actionType][
|
|
549
|
+
this.serializableDataByType[actionType][name].push(...scrapeResults);
|
|
566
550
|
yield this.options.serializableCallback({
|
|
567
551
|
scrapeList: this.serializableDataByType.scrapeList,
|
|
568
552
|
scrapeSchema: this.serializableDataByType.scrapeSchema
|
|
@@ -571,15 +555,18 @@ class Interpreter extends events_1.EventEmitter {
|
|
|
571
555
|
}
|
|
572
556
|
catch (error) {
|
|
573
557
|
console.error('ScrapeList action failed completely:', error.message);
|
|
574
|
-
// Don't throw error, just return empty array
|
|
575
558
|
const actionType = "scrapeList";
|
|
576
|
-
|
|
559
|
+
let name = actionName || "";
|
|
560
|
+
if (!name || name.trim() === "") {
|
|
561
|
+
this.scrapeListCounter++;
|
|
562
|
+
name = `List ${this.scrapeListCounter}`;
|
|
563
|
+
}
|
|
577
564
|
if (!this.namedResults[actionType])
|
|
578
565
|
this.namedResults[actionType] = {};
|
|
579
|
-
this.namedResults[actionType][
|
|
566
|
+
this.namedResults[actionType][name] = [];
|
|
580
567
|
if (!this.serializableDataByType[actionType])
|
|
581
568
|
this.serializableDataByType[actionType] = {};
|
|
582
|
-
this.serializableDataByType[actionType][
|
|
569
|
+
this.serializableDataByType[actionType][name] = [];
|
|
583
570
|
yield this.options.serializableCallback({
|
|
584
571
|
scrapeList: this.serializableDataByType.scrapeList,
|
|
585
572
|
scrapeSchema: this.serializableDataByType.scrapeSchema
|
|
@@ -662,25 +649,7 @@ class Interpreter extends events_1.EventEmitter {
|
|
|
662
649
|
if (debug === null || debug === void 0 ? void 0 : debug.setActionType) {
|
|
663
650
|
debug.setActionType(String(step.action));
|
|
664
651
|
}
|
|
665
|
-
|
|
666
|
-
if (step === null || step === void 0 ? void 0 : step.name) {
|
|
667
|
-
stepName = step.name;
|
|
668
|
-
}
|
|
669
|
-
else if (Array.isArray(step === null || step === void 0 ? void 0 : step.args) &&
|
|
670
|
-
step.args.length > 0 &&
|
|
671
|
-
typeof step.args[0] === "object" &&
|
|
672
|
-
"__name" in step.args[0]) {
|
|
673
|
-
stepName = step.args[0].__name;
|
|
674
|
-
}
|
|
675
|
-
else if (typeof (step === null || step === void 0 ? void 0 : step.args) === "object" &&
|
|
676
|
-
(step === null || step === void 0 ? void 0 : step.args) !== null &&
|
|
677
|
-
"__name" in step.args) {
|
|
678
|
-
stepName = step.args.__name;
|
|
679
|
-
}
|
|
680
|
-
// Default fallback
|
|
681
|
-
if (!stepName) {
|
|
682
|
-
stepName = String(step.action);
|
|
683
|
-
}
|
|
652
|
+
stepName = (step === null || step === void 0 ? void 0 : step.name) || String(step.action);
|
|
684
653
|
if (debug && typeof debug.setActionName === "function") {
|
|
685
654
|
debug.setActionName(stepName);
|
|
686
655
|
}
|
|
@@ -693,9 +662,12 @@ class Interpreter extends events_1.EventEmitter {
|
|
|
693
662
|
// "Arrayifying" here should not be needed (TS + syntax checker - only arrays; but why not)
|
|
694
663
|
const params = !step.args || Array.isArray(step.args) ? step.args : [step.args];
|
|
695
664
|
if (step.action === 'screenshot') {
|
|
696
|
-
// call the screenshot handler directly to allow the extra name parameter
|
|
697
665
|
yield wawActions.screenshot(...(params !== null && params !== void 0 ? params : []), stepName !== null && stepName !== void 0 ? stepName : undefined);
|
|
698
666
|
}
|
|
667
|
+
else if (step.action === 'scrapeList' || step.action === 'scrapeSchema') {
|
|
668
|
+
const actionName = step.name || "";
|
|
669
|
+
yield wawActions[step.action](...(params !== null && params !== void 0 ? params : []), actionName);
|
|
670
|
+
}
|
|
699
671
|
else {
|
|
700
672
|
yield wawActions[step.action](...(params !== null && params !== void 0 ? params : []));
|
|
701
673
|
}
|
|
@@ -755,16 +727,14 @@ class Interpreter extends events_1.EventEmitter {
|
|
|
755
727
|
}
|
|
756
728
|
});
|
|
757
729
|
}
|
|
758
|
-
handlePagination(
|
|
759
|
-
return __awaiter(this,
|
|
760
|
-
// Check abort flag at start of pagination
|
|
730
|
+
handlePagination(page_1, config_1) {
|
|
731
|
+
return __awaiter(this, arguments, void 0, function* (page, config, providedActionName = "") {
|
|
761
732
|
if (this.isAborted) {
|
|
762
733
|
this.log('Workflow aborted, stopping pagination', logger_1.Level.WARN);
|
|
763
734
|
return [];
|
|
764
735
|
}
|
|
765
|
-
// Generate action name for this scrapeList
|
|
766
736
|
const actionType = "scrapeList";
|
|
767
|
-
let actionName =
|
|
737
|
+
let actionName = providedActionName || "";
|
|
768
738
|
if (!actionName || actionName.trim() === "") {
|
|
769
739
|
this.scrapeListCounter++;
|
|
770
740
|
actionName = `List ${this.scrapeListCounter}`;
|