mx-cloud 0.0.24 → 0.0.26

package/build/interpret.d.ts

@@ -38,11 +38,13 @@ interface InterpreterOptions {
     serializableCallback: (output: any) => (void | Promise<void>);
     binaryCallback: (output: any, mimeType: string) => (void | Promise<void>);
     debug: boolean;
+    robotType?: 'extract' | 'scrape' | 'deep-extract';
     debugChannel: Partial<{
         activeId: (id: number) => void;
         debugMessage: (msg: string) => void;
         setActionType: (type: string) => void;
         incrementScrapeListIndex: () => void;
+        progressUpdate: (current: number, total: number, percentage: number) => void;
     }>;
 }
 /**
@@ -63,28 +65,16 @@ export default class Interpreter extends EventEmitter {
     private screenshotCounter;
     private scrapeListCounter;
     private serializableDataByType;
+    private pendingDeepExtraction;
+    private isInDeepExtractionPhase;
+    private deepExtractionStats;
+    private totalActions;
+    private executedActions;
     constructor(workflow: WorkflowFile, options?: Partial<InterpreterOptions>);
     trackAutohealFailure(error: string): void;
     private applyAdBlocker;
     private disableAdBlocker;
-    private getSelectors;
-    /**
-      * Returns the context object from given Page and the current workflow.\
-      * \
-      * `workflow` is used for selector extraction - function searches for used selectors to
-      * look for later in the page's context.
-      * @param page Playwright Page object
-      * @param workflow Current **initialized** workflow (array of where-what pairs).
-      * @returns {PageState} State of the current page.
-      */
-    private getState;
-    /**
-     * Tests if the given action is applicable with the given context.
-     * @param where Tested *where* condition
-     * @param context Current browser context.
-     * @returns True if `where` is applicable in the given context, false otherwise
-     */
-    private applicable;
+    private callWithTimeout;
     /**
      * Sets the abort flag to immediately stop all operations
      */
@@ -104,12 +94,7 @@ export default class Interpreter extends EventEmitter {
    */
     private carryOutSteps;
     private handlePagination;
-    private getMatchingActionId;
-    private removeShadowSelectors;
     private removeSpecialSelectors;
-    private generatePageNodeInformation;
-    private detectElementChanges;
-    private validateWorkflowAction;
     /**
      * Test if a selector is working on the current page
      * @param {Page} page - Playwright page object
@@ -148,6 +133,66 @@ export default class Interpreter extends EventEmitter {
      * @returns {Promise<WhereWhatPair>} - The potentially modified action
      */
     private validateAndFixSelectors;
+    /**
+     * Extracts URLs from the current page's list elements.
+     * Used during pagination to maintain sync between scraped results and extracted URLs.
+     *
+     * @param page - Playwright page object
+     * @param listSelector - The selector used to identify list elements
+     * @param limit - Maximum number of elements to process (should match number of scraped items)
+     * @returns Array of URL arrays, one per list element
+     */
+    private extractUrlsFromCurrentPage;
+    /**
+     * Builds a hierarchical deep extraction plan by analyzing the workflow structure.
+     * Identifies goto actions and determines what actions to execute at each level.
+     * Workflow is bottom-to-top, so we scan from end to start.
+     */
+    private buildDeepExtractionHierarchy;
+    /**
+     * Extracts hrefs directly from the page based on scrapeSchema selectors.
+     * Checks ALL selectors from the schema config - if they point to anchor elements, extract href.
+     * This is called after scrapeSchema executes to capture hrefs for deep extraction.
+     */
+    private extractHrefsFromPage;
+    /**
+     * Filters URLs for deep extraction based on the goto action pattern.
+     * This is called immediately after the first capture action (scrapeList).
+     * Returns the filtered URL mappings that should be processed after workflow completion.
+     * Each mapping maintains alignment with the original scrapeList index.
+     */
+    private filterDeepExtractionUrls;
+    /**
+     * Filters pre-extracted URLs for deep extraction based on the goto action pattern.
+     * This is used for paginated lists where URLs were extracted during pagination.
+     * Returns the filtered URL mappings that maintain alignment with scrapeList indices.
+     */
+    private filterDeepExtractionUrlsFromExtracted;
+    /**
+     * Helper function to check if a URL matches a goto pattern.
+     */
+    /**
+     * Generic pattern matching for deep extraction URLs.
+     * Works across any website by analyzing URL structure rather than relying on keywords.
+     *
+     * Strategy:
+     * 1. Match URLs with same origin and path length
+     * 2. Identify "structural" segments (numbers, short words, etc.) that should match exactly
+     * 3. Allow other segments to vary (dynamic content like IDs, slugs, names)
+     * 4. Skip exact matches to avoid duplicates
+     */
+    private matchesGotoPattern;
+    /**
+     * Executes hierarchical deep extraction by processing each level recursively.
+     * URLs are already stored in each hierarchy level's urlMappings during workflow execution.
+     */
+    private executeHierarchicalDeepExtraction;
+    /**
+     * Executes deep extraction for a single level.
+     * URLs are already extracted and stored in hierarchy during workflow execution.
+     * This function just navigates to URLs and executes the capture actions.
+     */
+    private executeDeepExtractionLevel;
     private runLoop;
     private ensureScriptsLoaded;
     /**