maxun-core 0.0.24 → 0.0.25

package/build/interpret.d.ts

@@ -57,6 +57,9 @@ export default class Interpreter extends EventEmitter {
     private log;
     private blocker;
     private cumulativeResults;
+    private namedResults;
+    private screenshotCounter;
+    private serializableDataByType;
     constructor(workflow: WorkflowFile, options?: Partial<InterpreterOptions>);
     /**
      * Sets the abort flag to immediately stop all operations

package/build/interpret.js

@@ -55,6 +55,12 @@ class Interpreter extends events_1.EventEmitter {
         this.isAborted = false;
         this.blocker = null;
         this.cumulativeResults = [];
+        this.namedResults = {};
+        this.screenshotCounter = 0;
+        this.serializableDataByType = {
+            scrapeList: {},
+            scrapeSchema: {}
+        };
         this.workflow = workflow.workflow;
         this.initializedWorkflow = null;
         this.options = Object.assign({ maxRepeats: 5, maxConcurrency: 5, serializableCallback: (data) => {
@@ -330,13 +336,26 @@ class Interpreter extends events_1.EventEmitter {
              * Beware of false linter errors - here, we know better!
              */
             const wawActions = {
-                screenshot: (params) => __awaiter(this, void 0, void 0, function* () {
+                screenshot: (params, nameOverride) => __awaiter(this, void 0, void 0, function* () {
                     var _b;
                     if ((_b = this.options.debugChannel) === null || _b === void 0 ? void 0 : _b.setActionType) {
-                        this.options.debugChannel.setActionType('screenshot');
+                        this.options.debugChannel.setActionType("screenshot");
                     }
                     const screenshotBuffer = yield page.screenshot(Object.assign(Object.assign({}, params), { path: undefined }));
-                    yield this.options.binaryCallback(screenshotBuffer, 'image/png');
+                    const explicitName = (typeof nameOverride === 'string' && nameOverride.trim().length > 0) ? nameOverride.trim() : null;
+                    let screenshotName;
+                    if (explicitName) {
+                        screenshotName = explicitName;
+                    }
+                    else {
+                        this.screenshotCounter += 1;
+                        screenshotName = `Screenshot ${this.screenshotCounter}`;
+                    }
+                    yield this.options.binaryCallback({
+                        name: screenshotName,
+                        data: screenshotBuffer,
+                        mimeType: "image/png",
+                    }, "image/png");
                 }),
                 enqueueLinks: (selector) => __awaiter(this, void 0, void 0, function* () {
                     var _c;
@@ -394,18 +413,52 @@ class Interpreter extends events_1.EventEmitter {
                     if (!this.cumulativeResults || !Array.isArray(this.cumulativeResults)) {
                         this.cumulativeResults = [];
                     }
+                    const resultToProcess = Array.isArray(scrapeResult) ? scrapeResult[0] : scrapeResult;
                     if (this.cumulativeResults.length === 0) {
-                        this.cumulativeResults.push({});
+                        const newRow = {};
+                        Object.entries(resultToProcess).forEach(([key, value]) => {
+                            if (value !== undefined) {
+                                newRow[key] = value;
+                            }
+                        });
+                        this.cumulativeResults.push(newRow);
                     }
-                    const mergedResult = this.cumulativeResults[0];
-                    const resultToProcess = Array.isArray(scrapeResult) ? scrapeResult[0] : scrapeResult;
-                    Object.entries(resultToProcess).forEach(([key, value]) => {
-                        if (value !== undefined) {
-                            mergedResult[key] = value;
+                    else {
+                        const lastRow = this.cumulativeResults[this.cumulativeResults.length - 1];
+                        const newResultKeys = Object.keys(resultToProcess).filter(key => resultToProcess[key] !== undefined);
+                        const hasRepeatedKeys = newResultKeys.some(key => lastRow.hasOwnProperty(key));
+                        if (hasRepeatedKeys) {
+                            const newRow = {};
+                            Object.entries(resultToProcess).forEach(([key, value]) => {
+                                if (value !== undefined) {
+                                    newRow[key] = value;
+                                }
+                            });
+                            this.cumulativeResults.push(newRow);
+                        }
+                        else {
+                            Object.entries(resultToProcess).forEach(([key, value]) => {
+                                if (value !== undefined) {
+                                    lastRow[key] = value;
+                                }
+                            });
                         }
+                    }
+                    const actionType = "scrapeSchema";
+                    const actionName = schema.__name || "Texts";
+                    if (!this.namedResults[actionType])
+                        this.namedResults[actionType] = {};
+                    this.namedResults[actionType][actionName] = this.cumulativeResults;
+                    if (!this.serializableDataByType[actionType])
+                        this.serializableDataByType[actionType] = {};
+                    if (!this.serializableDataByType[actionType][actionName]) {
+                        this.serializableDataByType[actionType][actionName] = [];
+                    }
+                    this.serializableDataByType[actionType][actionName] = [...this.cumulativeResults];
+                    yield this.options.serializableCallback({
+                        scrapeList: this.serializableDataByType.scrapeList,
+                        scrapeSchema: this.serializableDataByType.scrapeSchema
                     });
-                    console.log("Updated merged result:", mergedResult);
-                    yield this.options.serializableCallback([mergedResult]);
                 }),
                 scrapeList: (config) => __awaiter(this, void 0, void 0, function* () {
                     var _f, _g;
@@ -420,17 +473,56 @@ class Interpreter extends events_1.EventEmitter {
                         yield this.options.serializableCallback({});
                         return;
                     }
-                    yield this.ensureScriptsLoaded(page);
-                    if ((_g = this.options.debugChannel) === null || _g === void 0 ? void 0 : _g.incrementScrapeListIndex) {
-                        this.options.debugChannel.incrementScrapeListIndex();
-                    }
-                    if (!config.pagination) {
-                        const scrapeResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
-                        yield this.options.serializableCallback(scrapeResults);
+                    try {
+                        yield this.ensureScriptsLoaded(page);
+                        if ((_g = this.options.debugChannel) === null || _g === void 0 ? void 0 : _g.incrementScrapeListIndex) {
+                            this.options.debugChannel.incrementScrapeListIndex();
+                        }
+                        let scrapeResults = [];
+                        if (!config.pagination) {
+                            scrapeResults = yield page.evaluate((cfg) => {
+                                try {
+                                    return window.scrapeList(cfg);
+                                }
+                                catch (error) {
+                                    console.warn('ScrapeList evaluation failed:', error.message);
+                                    return [];
+                                }
+                            }, config);
+                        }
+                        else {
+                            scrapeResults = yield this.handlePagination(page, config);
+                        }
+                        if (!Array.isArray(scrapeResults)) {
+                            scrapeResults = [];
+                        }
+                        const actionType = "scrapeList";
+                        const actionName = config.__name || "List";
+                        if (!this.serializableDataByType[actionType])
+                            this.serializableDataByType[actionType] = {};
+                        if (!this.serializableDataByType[actionType][actionName]) {
+                            this.serializableDataByType[actionType][actionName] = [];
+                        }
+                        this.serializableDataByType[actionType][actionName].push(...scrapeResults);
+                        yield this.options.serializableCallback({
+                            scrapeList: this.serializableDataByType.scrapeList,
+                            scrapeSchema: this.serializableDataByType.scrapeSchema
+                        });
                     }
-                    else {
-                        const scrapeResults = yield this.handlePagination(page, config);
-                        yield this.options.serializableCallback(scrapeResults);
+                    catch (error) {
+                        console.error('ScrapeList action failed completely:', error.message);
+                        const actionType = "scrapeList";
+                        const actionName = config.__name || "List";
+                        if (!this.namedResults[actionType])
+                            this.namedResults[actionType] = {};
+                        this.namedResults[actionType][actionName] = [];
+                        if (!this.serializableDataByType[actionType])
+                            this.serializableDataByType[actionType] = {};
+                        this.serializableDataByType[actionType][actionName] = [];
+                        yield this.options.serializableCallback({
+                            scrapeList: this.serializableDataByType.scrapeList,
+                            scrapeSchema: this.serializableDataByType.scrapeSchema
+                        });
                     }
                 }),
                 scrapeListAuto: (config) => __awaiter(this, void 0, void 0, function* () {
@@ -497,11 +589,50 @@ class Interpreter extends events_1.EventEmitter {
                 }
             });
             for (const step of steps) {
+                if (this.isAborted) {
+                    this.log('Workflow aborted during step execution', logger_1.Level.WARN);
+                    return;
+                }
                 this.log(`Launching ${String(step.action)}`, logger_1.Level.LOG);
+                let stepName = null;
+                try {
+                    const debug = this.options.debugChannel;
+                    if (debug === null || debug === void 0 ? void 0 : debug.setActionType) {
+                        debug.setActionType(String(step.action));
+                    }
+                    if (step === null || step === void 0 ? void 0 : step.name) {
+                        stepName = step.name;
+                    }
+                    else if (Array.isArray(step === null || step === void 0 ? void 0 : step.args) &&
+                        step.args.length > 0 &&
+                        typeof step.args[0] === "object" &&
+                        "__name" in step.args[0]) {
+                        stepName = step.args[0].__name;
+                    }
+                    else if (typeof (step === null || step === void 0 ? void 0 : step.args) === "object" &&
+                        (step === null || step === void 0 ? void 0 : step.args) !== null &&
+                        "__name" in step.args) {
+                        stepName = step.args.__name;
+                    }
+                    if (!stepName) {
+                        stepName = String(step.action);
+                    }
+                    if (debug && typeof debug.setActionName === "function") {
+                        debug.setActionName(stepName);
+                    }
+                }
+                catch (err) {
+                    this.log(`Failed to set action name/type: ${err.message}`, logger_1.Level.WARN);
+                }
                 if (step.action in wawActions) {
                     // "Arrayifying" here should not be needed (TS + syntax checker - only arrays; but why not)
                     const params = !step.args || Array.isArray(step.args) ? step.args : [step.args];
-                    yield wawActions[step.action](...(params !== null && params !== void 0 ? params : []));
+                    if (step.action === 'screenshot') {
+                        yield wawActions.screenshot(...(params !== null && params !== void 0 ? params : []), stepName !== null && stepName !== void 0 ? stepName : undefined);
+                    }
+                    else {
+                        yield wawActions[step.action](...(params !== null && params !== void 0 ? params : []));
+                    }
                 }
                 else {
                     if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {

package/build/preprocessor.js

@@ -34,6 +34,8 @@ class Preprocessor {
                 what: joi_1.default.array().items({
                     action: joi_1.default.string().required(),
                     args: joi_1.default.array().items(joi_1.default.any()),
+                    name: joi_1.default.string(),
+                    actionId: joi_1.default.string()
                 }).required(),
             })).required(),
         });

package/build/types/workflow.d.ts

@@ -27,6 +27,8 @@ export type CustomFunctions = 'scrape' | 'scrapeSchema' | 'scroll' | 'screenshot
 export type What = {
     action: MethodNames<Page> | CustomFunctions;
     args?: any[];
+    name?: string;
+    actionId?: string;
 };
 export type PageState = Partial<BaseConditions>;
 export type ParamType = Record<string, any>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxun-core",
-  "version": "0.0.24",
+  "version": "0.0.25",
   "description": "Core package for Maxun, responsible for data extraction",
   "main": "build/index.js",
   "typings": "build/index.d.ts",