flow-frame-core 0.1.0

package/dist/run-flow.js

@@ -0,0 +1,220 @@
+import { ensureDir, getContext } from "./utils.js";
+import fs from 'fs';
+import path from 'path';
+import { executeFlow } from "./services/executor.js";
+export function runflow(row_flow_directory_func, workflow_trained_model_folder, workflow_folder_func) {
+    if (!row_flow_directory_func()) {
+        console.log('Row flow directory function is not valid');
+        return;
+    }
+    if (!workflow_trained_model_folder?.()) {
+        console.log('Workflow trained model folder function is not valid');
+        return;
+    }
+    if (!workflow_folder_func?.()) {
+        console.log('Workflow folder function is not valid');
+        return;
+    }
+    ensureDir(row_flow_directory_func());
+    ensureDir(workflow_trained_model_folder());
+    ensureDir(workflow_folder_func());
+    ensureQueueFile();
+    let promise = Promise.resolve();
+    async function handleRunFlowMessage(data) {
+        promise = promise.then(async () => {
+            console.log('→ Running flow with data:', data);
+            // Write data to a file {guid}.json.
+            let name = `run-flow-${guid()}.json`;
+            fs.writeFileSync(path.join(row_flow_directory_func(), name), JSON.stringify(data));
+            // and the name to a file named queue.json
+            fs.appendFileSync(path.join(row_flow_directory_func(), 'queue.json'), JSON.stringify({ name }) + "\n");
+            updateStatus();
+            return true;
+        });
+    }
+    function clearProgress() {
+        getContext().ws.send(JSON.stringify({
+            type: 'run-flow-clear-progress', data: {}
+        }));
+    }
+    function handleRunStatus() {
+        let count = getQueueCount();
+        getContext().ws.send(JSON.stringify({
+            type: 'run-flow-status', data: {
+                running_flow: getContext().running_flow || false,
+                count
+            }
+        }));
+    }
+    function updateStatus() {
+        ensureQueueFile();
+        handleRunStatus();
+    }
+    function ensureQueueFile() {
+        if (fs.existsSync(path.join(row_flow_directory_func(), 'queue.json')) === false) {
+            fs.writeFileSync(path.join(row_flow_directory_func(), 'queue.json'), '');
+        }
+    }
+    function getQueueCount() {
+        ensureQueueFile();
+        let count = fs.readFileSync(path.join(row_flow_directory_func(), 'queue.json'), 'utf-8').split('\n').filter(x => x.trim().length > 0).length;
+        return count;
+    }
+    function stopFlow() {
+        getContext().running_flow = false;
+        updateStatus();
+    }
+    function clearFlow() {
+        fs.writeFileSync(path.join(row_flow_directory_func(), 'queue.json'), '');
+        clearProgress();
+        updateStatus();
+    }
+    function startFlow() {
+        getContext().running_flow = true;
+        updateStatus();
+    }
+    async function run() {
+        let items = readQueue();
+        clearProgress();
+        if (items.length === 0) {
+            return false;
+        }
+        else {
+            console.log(`Processing : ${items[0].name}`);
+            let success = await processItem(items[0]);
+            if (success) {
+                promise = promise.then(() => {
+                    // Remove the processed item from the queue
+                    fs.appendFileSync(path.join(row_flow_directory_func(), 'processed.json'), JSON.stringify(items[0]) + "\n");
+                    items.shift();
+                });
+                await promise;
+            }
+            else {
+                items[0].retryCount = (items[0].retryCount || 0) + 1;
+                throw new Error('Failed to process item, will retry later');
+            }
+            promise = promise.then(() => {
+                fs.writeFileSync(path.join(row_flow_directory_func(), 'queue.json'), items.map(x => JSON.stringify(x)).join('\n'));
+            });
+            await promise;
+        }
+        updateStatus();
+        return true;
+    }
+    async function processItem(queueItem) {
+        // process the item
+        const { name } = queueItem;
+        let itemContents = fs.readFileSync(path.join(row_flow_directory_func(), name), 'utf-8');
+        let item;
+        try {
+            item = JSON.parse(itemContents);
+        }
+        catch (e) {
+            throw new Error('Invalid item, unable to parse JSON');
+        }
+        const { flowData, workflows, instanceId } = item;
+        if (!instanceId) {
+            throw new Error('Invalid item, missing instanceId');
+        }
+        if (!flowData) {
+            throw new Error('Invalid item, missing flowData');
+        }
+        if (!workflows) {
+            throw new Error('Invalid item, missing workflows');
+        }
+        loadWorkflowsIntoFolder(workflows, workflow_folder_func());
+        const { edges, nodes, modelContext } = flowData;
+        if (!edges || !nodes || !modelContext) {
+            throw new Error(`Run flow can't process this item`);
+        }
+        let currentContext = getContext(instanceId);
+        if (!currentContext) {
+            currentContext = {
+                status: 'running',
+                progress: {},
+                result: null,
+                error: null
+            };
+        }
+        try {
+            await executeFlow(nodes, edges, workflows, modelContext, {
+                [`@modelFolder`]: workflow_trained_model_folder()
+            }, (nodeId, progress) => {
+                console.log(`executing flow for node: ${nodeId} with progress: ${JSON.stringify(progress)}`);
+                getContext().ws.send(JSON.stringify({
+                    type: 'run-flow-progress', data: { instanceId, nodeId, progress }
+                }));
+            });
+            return true;
+        }
+        catch (e) {
+            console.error(`Error executing flow for node: ${e}`);
+            return false;
+        }
+    }
+    function readQueue() {
+        // returns data from queue
+        let data = fs.readFileSync(path.join(row_flow_directory_func(), 'queue.json'), 'utf-8');
+        return data.split('\n').filter(x => x.trim().length > 0).map(x => JSON.parse(x));
+    }
+    (async () => {
+        let didSomething = false;
+        while (true) {
+            if (getContext().running_flow) {
+                try {
+                    didSomething = await run();
+                }
+                catch (e) {
+                    console.error('Error running flow:', e);
+                }
+            }
+            await new Promise(resolve => setTimeout(resolve, didSomething ? 1000 : 10 * 1000));
+        }
+    })();
+    return {
+        handleRunFlowMessage,
+        handleRunStatus,
+        startFlow,
+        stopFlow,
+        clearFlow
+    };
+}
+function loadWorkflowsIntoFolder(workflows, workflow_folder) {
+    ensureDir(workflow_folder);
+    console.log(`Loading ${workflows.length} workflows into folder: ${workflow_folder}`);
+    const getAutomationFiles = (dir) => {
+        let results = [];
+        const list = fs.readdirSync(dir);
+        list.forEach(file => {
+            const filePath = path.join(dir, file);
+            const stat = fs.statSync(filePath);
+            if (stat && stat.isDirectory()) {
+                results = results.concat(getAutomationFiles(filePath));
+            }
+            else if (file.endsWith('.automation')) {
+                results.push(filePath);
+            }
+        });
+        return results;
+    };
+    const workflowFiles = getAutomationFiles(workflow_folder);
+    workflowFiles.forEach(filePath => {
+        try {
+            const workflow = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
+            workflows.push({ workflow });
+        }
+        catch (e) {
+            console.error(`Failed to load workflow from ${filePath}:`, e);
+        }
+    });
+    console.log(`Loaded ${workflows.length} workflows into folder: ${workflow_folder}`);
+}
+function guid() {
+    return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) {
+        const r = Math.random() * 16 | 0;
+        const v = c === 'x' ? r : (r & 0x3 | 0x8);
+        return v.toString(16);
+    });
+}
+//# sourceMappingURL=run-flow.js.map

package/dist/run-flow.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-flow.js","sourceRoot":"","sources":["../src/run-flow.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,EAAY,MAAM,IAAI,CAAC;AAC9B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,MAAM,UAAU,OAAO,CAAC,uBAAuB,EAAE,6BAA6B,EAAE,oBAAoB;IAChG,IAAI,CAAC,uBAAuB,EAAE,EAAE,CAAC;QAC7B,OAAO,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC;QACxD,OAAO;IACX,CAAC;IACD,IAAI,CAAC,6BAA6B,EAAE,EAAE,EAAE,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,qDAAqD,CAAC,CAAC;QACnE,OAAO;IACX,CAAC;IACD,IAAI,CAAC,oBAAoB,EAAE,EAAE,EAAE,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;QACrD,OAAO;IACX,CAAC;IACD,SAAS,CAAC,uBAAuB,EAAE,CAAC,CAAC;IACrC,SAAS,CAAC,6BAA6B,EAAE,CAAC,CAAC;IAC3C,SAAS,CAAC,oBAAoB,EAAE,CAAC,CAAC;IAClC,eAAe,EAAE,CAAC;IAClB,IAAI,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAEhC,KAAK,UAAU,oBAAoB,CAAC,IAAI;QACpC,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YAC9B,OAAO,CAAC,GAAG,CAAC,2BAA2B,EAAE,IAAI,CAAC,CAAC;YAC/C,oCAAoC;YACpC,IAAI,IAAI,GAAG,YAAY,IAAI,EAAE,OAAO,CAAA;YACpC,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,uBAAuB,EAAE,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;YACnF,0CAA0C;YAC1C,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,uBAAuB,EAAE,EAAE,YAAY,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;YAEvG,YAAY,EAAE,CAAC;YACf,OAAO,IAAI,CAAC;QAChB,CAAC,CAAC,CAAA;IACN,CAAC;IACD,SAAS,aAAa;QAClB,UAAU,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;YAChC,IAAI,EAAE,yBAAyB,EAAE,IAAI,EAAE,EAAE;SAC5C,CAAC,CAAC,CAAC;IACR,CAAC;IACD,SAAS,eAAe;QAEpB,IAAI,KAAK,GAAG,aAAa,EAAE,CAAC;QAC5B,UAAU,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;YAChC,IAAI,EAAE,iBAAiB,EAAE,IAAI,EAAE;gBAC3B,YAAY,EAAE,UAAU,EAAE,CAAC,YAAY,IAAI,KAAK;gBAChD,KAAK;aACR;SACJ,CAAC,CAAC,CAAC;IACR,CAAC;IAED,SAAS,YAAY;QACjB,eAAe,EAAE,CAAC;QAClB,eAAe,EAAE,CAAC;IACtB,CAAC;IACD,SAAS,eAAe;QACpB,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,uBAAuB,EAAE,EAAE,YAAY,CAAC,CAAC,KAAK,KAAK,EAAE,CAAC;YAC9E,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,uBAAuB,EAAE,EAAE,YAAY,CAAC,EAAE,EAAE,CAAC,CAAC;QAC7E,CAAC;IACL,CAAC;IACD,SAAS,aAAa;QAClB,eAAe,EAAE,CAAC;QAClB,IAAI,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,uBAAuB,EAAE,EAAE,YAAY,CAAC,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;QAC7I,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,SAAS,QAAQ;QACb,UAAU,EAAE,CAAC,YAAY,GAAG,KAAK,CAAC;QAClC,YAAY,EAAE,CAAC;IACnB,CAAC;IAED,SAAS,SAAS;QACd,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,uBAAuB,EAAE,EAAE,YAAY,CAAC,EAAE,EAAE,CAAC,CAAC;QACzE,aAAa,EAAE,CAAC;QAChB,YAAY,EAAE,CAAC;IACnB,CAAC;IAED,SAAS,SAAS;QACd,UAAU,EAAE,CAAC,YAAY,GAAG,IAAI,CAAC;QACjC,YAAY,EAAE,CAAC;IACnB,CAAC;IAED,KAAK,UAAU,GAAG;QACd,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;QACxB,aAAa,EAAE,CAAC;QAChB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrB,OAAO,KAAK,CAAC;QACjB,CAAC;aACI,CAAC;YACF,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YAC7C,IAAI,OAAO,GAAG,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1C,IAAI,OAAO,EAAE,CAAC;gBACV,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE;oBACxB,2CAA2C;oBAC3C,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,uBAAuB,EAAE,EAAE,gBAAgB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;oBAC3G,KAAK,CAAC,KAAK,EAAE,CAAC;gBAElB,CAAC,CAAC,CAAC;gBACH,MAAM,OAAO,CAAC;YAClB,CAAC;iBACI,CAAC;gBACF,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;gBACrD,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;YAChE,CAAC;YACD,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE;gBACxB,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,uBAAuB,EAAE,EAAE,YAAY,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YACvH,CAAC,CAAC,CAAA;YACF,MAAM,OAAO,CAAC;QAClB,CAAC;QAED,YAAY,EAAE,CAAC;QACf,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,KAAK,UAAU,WAAW,CAAC,SAAS;QAChC,mBAAmB;QACnB,MAAM,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC;QAC3B,IAAI,YAAY,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,uBAAuB,EAAE,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;QACxF,IAAI,IAAI,CAAC;QACT,IAAI,CAAC;YACD,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QACpC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QAC1D,CAAC;QACD,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC;QACjD,IAAI,CAAC,UAAU,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;QACxD,CAAC;QACD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QACtD,CAAC;QACD,IAAI,CAAC,SAAS,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;QACvD,CAAC;QACD,uBAAuB,CAAC,SAAS,EAAE,oBAAoB,EAAE,CAAC,CAAC;QAE3D,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,QAAQ,CAAC;QAChD,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,IAAI,CAAC,YAAY,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;QACxD,CAAC;QACD,IAAI,cAAc,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;QAC5C,IAAI,CAAC,cAAc,EAAE,CAAC;YAClB,cAAc,GAAG;gBACb,MAAM,EAAE,SAAS;gBACjB,QAAQ,EAAE,EAAE;gBACZ,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,IAAI;aACd,CAAC;QACN,CAAC;QACD,IAAI,CAAC;YACD,MAAM,WAAW,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE;gBACrD,CAAC,cAAc,CAAC,EAAE,6BAA6B,EAAE;aACpD,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,EAAE;gBACpB,OAAO,CAAC,GAAG,CAAC,4BAA4B,MAAM,mBAAmB,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBAC7F,UAAU,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;oBAChC,IAAI,EAAE,mBAAmB,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE;iBACpE,CAAC,CAAC,CAAC;YACR,CAAC,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QAChB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,OAAO,CAAC,KAAK,CAAC,kCAAkC,CAAC,EAAE,CAAC,CAAC;YACrD,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;IACD,SAAS,SAAS;QACd,0BAA0B;QAC1B,IAAI,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,uBAAuB,EAAE,EAAE,YAAY,CAAC,EAAE,OAAO,CAAC,CAAC;QACxF,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACrF,CAAC;IAED,CAAC,KAAK,IAAI,EAAE;QACR,IAAI,YAAY,GAAG,KAAK,CAAC;QACzB,OAAO,IAAI,EAAE,CAAC;YACV,IAAI,UAAU,EAAE,CAAC,YAAY,EAAE,CAAC;gBAC5B,IAAI,CAAC;oBACD,YAAY,GAAG,MAAM,GAAG,EAAE,CAAC;gBAC/B,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACT,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,CAAC,CAAC,CAAC;gBAC5C,CAAC;YACL,CAAC;YACD,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;QACvF,CAAC;IACL,CAAC,CAAC,EAAE,CAAC;IAEL,OAAO;QACH,oBAAoB;QACpB,eAAe;QACf,SAAS;QACT,QAAQ;QACR,SAAS;KACZ,CAAA;AACL,CAAC;AACD,SAAS,uBAAuB,CAAC,SAAS,EAAE,eAAe;IACvD,SAAS,CAAC,eAAe,CAAC,CAAC;IAC3B,OAAO,CAAC,GAAG,CAAC,WAAW,SAAS,CAAC,MAAM,2BAA2B,eAAe,EAAE,CAAC,CAAC;IAErF,MAAM,kBAAkB,GAAG,CAAC,GAAG,EAAE,EAAE;QAC/B,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,MAAM,IAAI,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;YAChB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YACtC,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACnC,IAAI,IAAI,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;gBAC7B,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC3D,CAAC;iBAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;gBACtC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC3B,CAAC;QACL,CAAC,CAAC,CAAC;QACH,OAAO,OAAO,CAAC;IACnB,CAAC,CAAC;IAEF,MAAM,aAAa,GAAG,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,aAAa,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE;QAC7B,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;YAChE,SAAS,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;QACjC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,OAAO,CAAC,KAAK,CAAC,gCAAgC,QAAQ,GAAG,EAAE,CAAC,CAAC,CAAC;QAClE,CAAC;IACL,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,GAAG,CAAC,UAAU,SAAS,CAAC,MAAM,2BAA2B,eAAe,EAAE,CAAC,CAAC;AACxF,CAAC;AAED,SAAS,IAAI;IACT,OAAO,sCAAsC,CAAC,OAAO,CAAC,OAAO,EAAE,UAAU,CAAC;QACtE,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;QACjC,MAAM,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,GAAG,CAAC,CAAC;QAC1C,OAAO,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC1B,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/scraper.d.ts

@@ -0,0 +1,2 @@
+export declare function crawl(startUrl: any): Promise<{}>;
+//# sourceMappingURL=scraper.d.ts.map

package/dist/scraper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scraper.d.ts","sourceRoot":"","sources":["../src/scraper.ts"],"names":[],"mappings":"AAYA,wBAAsB,KAAK,CAAC,QAAQ,KAAA,eAoDnC"}

package/dist/scraper.js

@@ -0,0 +1,75 @@
+// scraper.js
+import axios from 'axios';
+import * as cheerio from 'cheerio'; // Changed from default import to namespace import
+import { URL } from 'url';
+import fs from 'fs';
+import { fileURLToPath } from 'url';
+// Get __filename equivalent for ES modules
+const __filename = fileURLToPath(import.meta.url);
+// Crawl starting at `startUrl`, only following links on the same hostname.
+// Returns a map { url: textContent }
+export async function crawl(startUrl) {
+    const startHost = new URL(startUrl).hostname;
+    const visited = new Set();
+    const results = {};
+    // Depth-first recursion
+    async function visit(pageUrl) {
+        if (visited.has(pageUrl))
+            return;
+        visited.add(pageUrl);
+        let html;
+        try {
+            const res = await axios.get(pageUrl);
+            html = res.data;
+        }
+        catch (err) {
+            console.error(`❌ Failed to fetch ${pageUrl}: ${err.message}`);
+            return;
+        }
+        const $ = cheerio.load(html);
+        // Remove scripts and styles
+        $('script, style, noscript').remove();
+        // Get visible text
+        const text = $('body').text()
+            .replace(/\s+/g, ' ') // collapse whitespace
+            .trim();
+        results[pageUrl] = text;
+        // Find all links
+        const links = $('a[href]')
+            .map((_, a) => $(a).attr('href'))
+            .get()
+            .map(href => {
+            try {
+                return new URL(href, pageUrl).toString();
+            }
+            catch {
+                return null;
+            }
+        })
+            .filter(u => u && new URL(u).hostname === startHost);
+        // Recurse (you could throttle/concurrency-limit here)
+        for (const link of links) {
+            await visit(link);
+        }
+    }
+    await visit(startUrl);
+    return results;
+}
+// CLI entrypoint - ES module equivalent of require.main === module
+if (import.meta.url === `file://${process.argv[1]}`) {
+    const [, , startUrl] = process.argv;
+    if (!startUrl) {
+        console.error('Usage: node scraper.js <start-url>');
+        process.exit(1);
+    }
+    crawl(startUrl)
+        .then(results => {
+        // e.g. write to file
+        fs.writeFileSync('scraped.json', JSON.stringify(results, null, 2));
+        console.log(`✅ Crawled ${Object.keys(results).length} pages. Output → scraped.json`);
+    })
+        .catch(err => {
+        console.error(`Fatal error: ${err.stack}`);
+    });
+}
+//# sourceMappingURL=scraper.js.map

package/dist/scraper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scraper.js","sourceRoot":"","sources":["../src/scraper.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC,CAAE,kDAAkD;AACvF,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAC1B,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AAEpC,2CAA2C;AAC3C,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAElD,2EAA2E;AAC3E,qCAAqC;AACrC,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,QAAQ;IAChC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC;IAC7C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAE,CAAC;IAC1B,MAAM,OAAO,GAAG,EAAE,CAAC;IAEnB,wBAAwB;IACxB,KAAK,UAAU,KAAK,CAAC,OAAO;QACxB,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC;YAAE,OAAO;QACjC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAErB,IAAI,IAAI,CAAC;QACT,IAAI,CAAC;YACD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACrC,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;QACpB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,qBAAqB,OAAO,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;YAC9D,OAAO;QACX,CAAC;QAED,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE7B,4BAA4B;QAC5B,CAAC,CAAC,yBAAyB,CAAC,CAAC,MAAM,EAAE,CAAC;QAEtC,mBAAmB;QACnB,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE;aACxB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAM,sBAAsB;aAChD,IAAI,EAAE,CAAC;QAEZ,OAAO,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC;QAExB,iBAAiB;QACjB,MAAM,KAAK,GAAG,CAAC,CAAC,SAAS,CAAC;aACrB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aAChC,GAAG,EAAE;aACL,GAAG,CAAC,IAAI,CAAC,EAAE;YACR,IAAI,CAAC;gBACD,OAAO,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;YAC7C,CAAC;YAAC,MAAM,CAAC;gBACL,OAAO,IAAI,CAAC;YAChB,CAAC;QACL,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC;QAEzD,sDAAsD;QACtD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;QACtB,CAAC;IACL,CAAC;IAED,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;IACtB,OAAO,OAAO,CAAC;AACnB,CAAC;AAED,mEAAmE;AACnE,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,UAAU,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IAClD,MAAM,CAAC,EAAE,AAAD,EAAG,QAAQ,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IACpC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACZ,OAAO,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACpD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC;SACV,IAAI,CAAC,OAAO,CAAC,EAAE;QACZ,qBAAqB;QACrB,EAAE,CAAC,aAAa,CAAC,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QACnE,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,+BAA+B,CAAC,CAAC;IACzF,CAAC,CAAC;SACD,KAAK,CAAC,GAAG,CAAC,EAAE;QACT,OAAO,CAAC,KAAK,CAAC,gBAAgB,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;AACX,CAAC"}

package/dist/scraper_endpoints.d.ts

@@ -0,0 +1,2 @@
+export declare function scraper_endpoints(app: any): void;
+//# sourceMappingURL=scraper_endpoints.d.ts.map

package/dist/scraper_endpoints.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scraper_endpoints.d.ts","sourceRoot":"","sources":["../src/scraper_endpoints.ts"],"names":[],"mappings":"AAIA,wBAAgB,iBAAiB,CAAC,GAAG,KAAA,QAkCpC"}

package/dist/scraper_endpoints.js

@@ -0,0 +1,40 @@
+// index.js
+import { URL } from 'url';
+import { crawl } from './scraper.js'; // path to the module you already wrote
+export function scraper_endpoints(app) {
+    // POST /scrape
+    // body: { domain: string }
+    app.post('/scrape', async (req, res) => {
+        const { domain } = req.body;
+        if (!domain || typeof domain !== 'string') {
+            return res.status(400).json({ error: '`domain` must be a non-empty string' });
+        }
+        // Normalize into a full URL
+        let startUrl;
+        try {
+            // if user passed "https://foo.com" or "http://foo.com"
+            startUrl = new URL(domain).toString();
+        }
+        catch {
+            // fall back to http://
+            try {
+                startUrl = new URL(`http://${domain}`).toString();
+            }
+            catch (err) {
+                return res.status(400).json({ error: 'Invalid domain format' });
+            }
+        }
+        try {
+            const pages = await crawl(startUrl);
+            return res.json({ startUrl, pages });
+        }
+        catch (err) {
+            console.error('Scrape error:', err);
+            return res.status(500).json({
+                error: 'Failed to scrape',
+                details: err.message
+            });
+        }
+    });
+}
+//# sourceMappingURL=scraper_endpoints.js.map

package/dist/scraper_endpoints.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scraper_endpoints.js","sourceRoot":"","sources":["../src/scraper_endpoints.ts"],"names":[],"mappings":"AAAA,WAAW;AACX,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAC1B,OAAO,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC,CAAC,uCAAuC;AAE7E,MAAM,UAAU,iBAAiB,CAAC,GAAG;IACjC,eAAe;IACf,2BAA2B;IAC3B,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE;QACnC,MAAM,EAAE,MAAM,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC;QAC5B,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YACxC,OAAO,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,qCAAqC,EAAE,CAAC,CAAC;QAClF,CAAC;QAED,4BAA4B;QAC5B,IAAI,QAAQ,CAAC;QACb,IAAI,CAAC;YACD,uDAAuD;YACvD,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC1C,CAAC;QAAC,MAAM,CAAC;YACL,uBAAuB;YACvB,IAAI,CAAC;gBACD,QAAQ,GAAG,IAAI,GAAG,CAAC,UAAU,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;YACtD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,OAAO,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC,CAAC;YACpE,CAAC;QACL,CAAC;QAED,IAAI,CAAC;YACD,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;YACpC,OAAO,GAAG,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QACzC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;YACpC,OAAO,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBACxB,KAAK,EAAE,kBAAkB;gBACzB,OAAO,EAAE,GAAG,CAAC,OAAO;aACvB,CAAC,CAAC;QACP,CAAC;IACL,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/server.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=server.d.ts.map

package/dist/server.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":""}