octoparse-cli 0.1.14

package/node_modules/@octopus/engine/dist/utils/index.js

@@ -0,0 +1,356 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.repairURL = exports.transformDate = exports.random = exports.hashCode = exports.executeJavaScriptInWorker = exports.correctUrl = exports.isNullOrEmpty = exports.getGlobalCookie = exports.generateExtractIndexMap = exports.sleep = exports.parseXMLToObj = exports.parseXML = void 0;
+const moment_1 = __importDefault(require("moment"));
+const xmldom_1 = require("xmldom");
+const fast_xml_parser_1 = require("fast-xml-parser");
+const worker_threads_1 = require("worker_threads");
+const fs_1 = require("fs");
+const sax_1 = __importDefault(require("sax"));
+const parser = new fast_xml_parser_1.XMLParser();
+const parseXML = (xml, cb) => {
+    let data, er;
+    try {
+        data = parser.parse(xml);
+    }
+    catch (e) {
+        er = e;
+    }
+    finally {
+        cb(er, data);
+    }
+};
+exports.parseXML = parseXML;
+/** 解析 XML 转为对象 */
+const parseXMLToObj = (xml) => {
+    const result = { data: {}, isAppend: false };
+    try {
+        const doc = new xmldom_1.DOMParser().parseFromString(xml);
+        // eslint-disable-next-line @typescript-eslint/prefer-for-of
+        for (let i = 0; i < doc.documentElement.childNodes.length; i++) {
+            const element = doc.documentElement.childNodes[i];
+            if (element.nodeType === 1) {
+                /** @issues 这里去获取真实的标签名，因为之前工作流生成 XML 时非法字符被去掉了 */
+                const realName = element.getAttribute('realName');
+                const localName = realName || element.localName;
+                const isAppend = element.getAttribute('IsAppend') === 'true';
+                if (isAppend) {
+                    result.isAppend = true;
+                }
+                let value = element.textContent;
+                /** 同一行内同名字段勾选了 [同一字段多行合并] 开关时，勾选的字段合并前一个同名字段内容 */
+                if (isAppend && result.data[localName]) {
+                    value = `${result.data[localName].value}\r\n${element.textContent}`;
+                }
+                result.data[localName] = { value, isAppend };
+            }
+        }
+    }
+    catch (error) {
+        console.error(error);
+    }
+    return result;
+};
+exports.parseXMLToObj = parseXMLToObj;
+const sleep = (delay) => new Promise((resole) => setTimeout(resole, delay));
+exports.sleep = sleep;
+/** 获取步骤的唯一标识 */
+const getActionID = (node) => {
+    let id = node.getAttribute('Name');
+    if (!id) {
+        for (let i = 0; i < node.attributes.length; i++) {
+            const { name, value } = node.attributes[i];
+            if (name.includes(':Name')) {
+                id = value;
+                break;
+            }
+        }
+    }
+    return id;
+};
+/** 判断节点是否为分支步骤 */
+const isCondition = (node) => node && node.tagName && node.tagName.indexOf('ConditionAction') != -1;
+/** 判断节点是否为条件步骤 */
+const isBranch = (node) => node && node.tagName && node.tagName.indexOf('BranchAction') != -1;
+/** 节点递归往上去找是否包含在条件分支中 */
+const deepFindParentBranch = (node) => {
+    let branch = node.parentNode;
+    while (branch && branch.nodeType === 1 && branch.tagName && branch.tagName.indexOf('BranchAction') === -1) {
+        branch = branch.parentNode;
+    }
+    return branch;
+};
+const deepFindBranch = (list, branchID) => {
+    for (const item of list) {
+        if (item.branchs) {
+            for (const branch of item.branchs) {
+                if (branch.branchID === branchID)
+                    return branch;
+                if (branch.list) {
+                    const _branch = deepFindBranch(branch.list, branchID);
+                    if (_branch)
+                        return _branch;
+                }
+            }
+        }
+    }
+};
+const deepFindCondition = (list, conditionID) => {
+    for (const item of list) {
+        if (item.conditionID === conditionID)
+            return item;
+        if (item.branchs) {
+            for (const branch of item.branchs) {
+                if (branch.list) {
+                    const condition = deepFindCondition(branch.list, conditionID);
+                    if (condition)
+                        return condition;
+                }
+            }
+        }
+    }
+};
+/** 生成任务流程的全部【提取步骤】的序号映射表 */
+const generateExtractIndexMap = (xml) => {
+    const list = [];
+    const extractIndexMap = new Map();
+    const document = new xmldom_1.DOMParser().parseFromString(xml, 'text/xml');
+    const parseNode = (childNodes) => {
+        for (let i = 0; i < childNodes.length; i++) {
+            const node = childNodes[i];
+            if (node.nodeType === 1) {
+                switch (node.localName) {
+                    case 'ConditionAction':
+                        {
+                            const conditionID = getActionID(node);
+                            const parentBranch = deepFindParentBranch(node);
+                            if (parentBranch && parentBranch.localName === 'BranchAction') {
+                                const parentBranchID = getActionID(parentBranch);
+                                const branch = deepFindBranch(list, parentBranchID);
+                                branch.list.push({ conditionID, branchs: [] });
+                            }
+                            else {
+                                list.push({ conditionID, branchs: [] });
+                            }
+                        }
+                        break;
+                    case 'BranchAction':
+                        {
+                            const branchID = getActionID(node);
+                            const conditionID = getActionID(node.parentNode);
+                            const condition = deepFindCondition(list, conditionID);
+                            condition.branchs.push({ branchID, list: [] });
+                        }
+                        break;
+                    case 'ExtractDataAction':
+                        {
+                            const extractID = getActionID(node);
+                            const parentBranch = deepFindParentBranch(node);
+                            if (parentBranch && parentBranch.localName === 'BranchAction') {
+                                const parentBranchID = getActionID(parentBranch);
+                                const branch = deepFindBranch(list, parentBranchID);
+                                branch.list.push({ extractID });
+                            }
+                            else {
+                                list.push({ extractID });
+                            }
+                        }
+                        break;
+                }
+                parseNode(node.childNodes);
+            }
+        }
+    };
+    parseNode(document.childNodes);
+    const parseList = (list, index) => {
+        list.forEach(({ extractID, branchs }, i) => extractID ? extractIndexMap.set(extractID, index + i) : branchs.forEach(({ list }) => parseList(list, index + i)));
+    };
+    parseList(list, 0);
+    return extractIndexMap;
+};
+exports.generateExtractIndexMap = generateExtractIndexMap;
+const getGlobalCookie = (xml) => {
+    const globalCookieConfig = { globalCookie: '', isSetGlobalCookie: false };
+    const parser = sax_1.default.parser(true, { trim: true }); // strict=true 启用严格模式
+    parser.onopentag = (node) => {
+        if (node.name.includes('RootAction')) {
+            globalCookieConfig.globalCookie = node.attributes['globalCookie'] || '';
+            globalCookieConfig.isSetGlobalCookie = node.attributes['isSetGlobalCookie'] === 'true';
+        }
+    };
+    parser.onerror = (err) => {
+        console.error('XML parsing failed:', err);
+        parser.resume();
+    };
+    try {
+        parser.write(xml).close();
+    }
+    catch (e) {
+        console.error('XML processing failed:', e);
+    }
+    return globalCookieConfig;
+};
+exports.getGlobalCookie = getGlobalCookie;
+const isNullOrEmpty = (value) => {
+    return !(typeof value === 'string' && value.length > 0);
+};
+exports.isNullOrEmpty = isNullOrEmpty;
+const correctUrl = (url) => {
+    const regExp = /^https?:\/\//;
+    if (!regExp.test(url)) {
+        url = `http://${url}`;
+    }
+    return url;
+};
+exports.correctUrl = correctUrl;
+/**
+ * 在 Worker 线程中执行耗时的 JavaScript 代码，可以设置超时时间自动结束
+ * PS：这个功能目前主要给执行正则表达式时使用，因为 string.replace() 等方法是同步执行，当执行一些用户输入的不可预期的正则表达式时会卡死的当前 UI 线程
+ * @see https://skieer.feishu.cn/docs/doccncVezuTFcky9wboyAkCPLuc?sidebarOpen=1
+ * @param code 需要执行的代码
+ * @param timeout 超时时间，默认为 10s
+ */
+const executeJavaScriptInWorker = (code, timeout = 10000) => {
+    return new Promise((resolve, reject) => {
+        // 创建 Worker
+        const worker = new worker_threads_1.Worker(`
+      const { parentPort } = require('worker_threads');
+      parentPort.on('message', (code) => {
+        try {
+          const result = eval(code);
+          parentPort.postMessage(result);
+        } catch (error) {
+          parentPort.postMessage({ error: error.message });
+        }
+      });
+    `, { eval: true });
+        // 设置超时
+        const timer = setTimeout(() => {
+            worker.terminate();
+            reject(new Error('execute javascript code timeout in worker'));
+        }, timeout);
+        // 处理 Worker 返回的结果
+        worker.on('message', (result) => {
+            clearTimeout(timer);
+            worker.terminate();
+            if (result && result.error) {
+                reject(new Error(result.error));
+            }
+            else {
+                resolve(result);
+            }
+        });
+        // 处理 Worker 错误
+        worker.on('error', (error) => {
+            clearTimeout(timer);
+            worker.terminate();
+            reject(error);
+        });
+        // 发送代码到 Worker
+        worker.postMessage(code);
+    });
+};
+exports.executeJavaScriptInWorker = executeJavaScriptInWorker;
+const hashCode = (str) => {
+    let hash = 0;
+    for (let i = 0; i < str.length; i++) {
+        const character = str.charCodeAt(i);
+        hash = (hash << 5) - hash + character;
+        hash = hash & hash; // Convert to 32bit integer
+    }
+    return hash;
+};
+exports.hashCode = hashCode;
+const random = (min, max) => {
+    if (min >= max)
+        return min;
+    const range = max - min;
+    const rand = Math.random();
+    return min + Math.round(rand * range);
+};
+exports.random = random;
+/** 转换时间描述文本为标准时间对象 */
+const transformDate = (input) => {
+    if (['刚刚', '几秒前', 'just now', 'now', 'Now', 'Gerade', 'Jetzt'].includes(input)) {
+        return new Date();
+    }
+    if (['Today', 'TODAY', 'Heute'].includes(input)) {
+        return (0, moment_1.default)((0, moment_1.default)().format('YYYY-MM-DD')).toDate();
+    }
+    if (['昨天', 'Yesterday', 'Gestern'].includes(input)) {
+        return (0, moment_1.default)((0, moment_1.default)().format('YYYY-MM-DD')).subtract(1, 'd').toDate();
+    }
+    if (['前天'].includes(input)) {
+        return (0, moment_1.default)((0, moment_1.default)().format('YYYY-MM-DD')).subtract(2, 'd').toDate();
+    }
+    const yesterdayDateRegex = /(?<=^昨天\s*)(\d+:\d+)$/;
+    const yesterdayDateResult = input.match(yesterdayDateRegex);
+    if (yesterdayDateResult) {
+        return (0, moment_1.default)((0, moment_1.default)(yesterdayDateResult[0], 'HH:mm')).subtract(1, 'd').toDate();
+    }
+    const threedayDateRegex = /(?<=^前天\s*)(\d+:\d+)$/;
+    const threedayDateDateResult = input.match(threedayDateRegex);
+    if (threedayDateDateResult) {
+        return (0, moment_1.default)((0, moment_1.default)(threedayDateDateResult[0], 'HH:mm')).subtract(2, 'd').toDate();
+    }
+    const secondRegex = /^(\d+)(?=\s*秒前$|\s*seconds? ago$|\s*s$)|(?<=^hace\s*|^vor\s*|^Il y a\s*)(\d+)(?=\s*segundos?$|\s*Sekunden?$|\s*secondes?$)/i;
+    const secondResult = input.match(secondRegex);
+    if (secondResult) {
+        const x = parseInt(secondResult[0]);
+        return (0, moment_1.default)().subtract(x, 's').toDate();
+    }
+    const minuteRegex = /^(\d+)(?=\s*分钟前$|\s*分前$|\s*minutes? ago$|\s*m$|\s*Min\.$)|(?<=^hace\s*|^vor\s*|^Il y a\s*)(\d+)(?=\s*minutos?$|\s*Minuten?$|\s*minutes?$)/i;
+    const minuteResult = input.match(minuteRegex);
+    if (minuteResult) {
+        const x = parseInt(minuteResult[0]);
+        return (0, moment_1.default)().subtract(x, 'm').toDate();
+    }
+    const hourRegex = /^(\d+)(?=\s*小时前$|\s*時間前$|\s*hours? ago$|\s*h$|\s*Std\.$)|(?<=^hace\s*|^vor\s*|^Il y a\s*)(\d+)(?=\s*horas?$|\s*Stunden?$|\s*heures?$)/i;
+    const hourResult = input.match(hourRegex);
+    if (hourResult) {
+        const x = parseInt(hourResult[0]);
+        return (0, moment_1.default)().subtract(x, 'h').toDate();
+    }
+    const dayRegex = /^(\d+)(?=\s*天前$|\s*日前$|\s*days? ago$|\s*d$|\s*Tage?$)|(?<=^hace\s*|^vor\s*|^Il y a\s*)(\d+)(?=\s*días?$|\s*Tag(en)?$|\s*jours?$)/i;
+    const dayResult = input.match(dayRegex);
+    if (dayResult) {
+        const x = parseInt(dayResult[0]);
+        return (0, moment_1.default)().subtract(x, 'd').toDate();
+    }
+    const weekRegex = /^(\d+)(?=\s*周前$|\s*週間前$|\s*weeks? ago$|\s*w$|\s*Wochen?$)|(?<=^hace\s*|^vor\s*|^Il y a\s*)(\d+)(?=\s*semanas?$|\s*Wochen?$|\s*semaines?$)/i;
+    const weekResult = input.match(weekRegex);
+    if (weekResult) {
+        const x = parseInt(weekResult[0]);
+        return (0, moment_1.default)().subtract(x, 'w').toDate();
+    }
+    const monthRegex = /^(\d+)(?=\s*个月前$|\s*か月前$|\s*months? ago$|\s*mo$|\s*Monate?$)|(?<=^hace\s*|^vor\s*|^Il y a\s*)(\d+)(?=\s*mes(es)?$|\s*Monat(en)?$|\s*mois$)/i;
+    const monthResult = input.match(monthRegex);
+    if (monthResult) {
+        const x = parseInt(monthResult[0]);
+        return (0, moment_1.default)().subtract(x, 'M').toDate();
+    }
+    const yearRegex = /^(\d+)(?=\s*年前$|\s*years? ago$|\s*y$)|(?<=^hace\s*|^vor\s*|^Il y a\s*)(\d+)(?=\s*años?$|\s*Jahr(en)?$|\s*ans?$)/i;
+    const yearResult = input.match(yearRegex);
+    if (yearResult) {
+        const x = parseInt(yearResult[0]);
+        return (0, moment_1.default)().subtract(x, 'y').toDate();
+    }
+};
+exports.transformDate = transformDate;
+const repairURL = (url) => {
+    /** 先判断是否是本地路径 */
+    if ((0, fs_1.existsSync)(url)) {
+        url = `file://${url}`;
+    }
+    else {
+        const regExp = /^(http:\/\/|https:\/\/|ftp:\/\/|file:\/\/)/i;
+        if (!regExp.test(url)) {
+            url = `http://${url}`;
+        }
+    }
+    return url;
+};
+exports.repairURL = repairURL;

package/node_modules/@octopus/engine/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "@octopus/engine",
+  "productName": "octoparse-engine",
+  "version": "1.0.21",
+  "description": "run octoparse in real browser",
+  "main": "dist/index.js",
+  "module": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": "./dist/index.js",
+    "./transformer": "./dist/translator/translator.js",
+    "./enums": "./dist/enums/index.js",
+    "./browser": "./dist/browser.js",
+    "./type": "./dist/type.js",
+    "./settings": "./dist/settings.js"
+  },
+  "typesVersions": {
+    "*": {
+      "transformer": [
+        "./dist/translator/translator.d.ts"
+      ],
+      "enums": [
+        "./dist/enums/index.d.ts"
+      ],
+      "browser": [
+        "./dist/browser.d.ts"
+      ],
+      "type": [
+        "./dist/type.d.ts"
+      ],
+      "settings": [
+        "./dist/settings.d.ts"
+      ]
+    }
+  },
+  "scripts": {
+    "lint": "eslint --ext .ts,.tsx .",
+    "clean": "rimraf dist tsconfig.tsbuildinfo",
+    "build": "tsc -b --force && node ./build.mjs dist && node ./obfuscate.mjs",
+    "build:dev": "tsc -b --force && node ./build.mjs dist",
+    "pub": "pnpm build & npm publish --registry http://101.34.117.214:4873 --no-git-checks"
+  },
+  "files": [
+    "dist"
+  ],
+  "devDependencies": {
+    "@types/node": "22.9.0",
+    "@types/ws": "8.5.14",
+    "@types/xmldom": "0.1.34",
+    "glob": "11.0.3",
+    "javascript-obfuscator": "4.1.1",
+    "typescript": "catalog:"
+  },
+  "keywords": [],
+  "author": "octo",
+  "license": "MIT",
+  "dependencies": {}
+}

package/package.json

@@ -0,0 +1,79 @@
+{
+  "name": "octoparse-cli",
+  "version": "0.1.14",
+  "description": "Standalone Octoparse engine CLI that runs local extraction without the Electron client.",
+  "private": false,
+  "type": "module",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/octoparse/octoparse-cli.git"
+  },
+  "bugs": {
+    "url": "https://github.com/octoparse/octoparse-cli/issues"
+  },
+  "homepage": "https://github.com/octoparse/octoparse-cli#readme",
+  "files": [
+    "dist",
+    "schemas",
+    "examples",
+    "README.md",
+    "SECURITY.md",
+    "RUNTIME_SECURITY_NOTICE.txt"
+  ],
+  "bin": {
+    "octoparse": "dist/index.js"
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsx src/index.ts",
+    "prepack": "node scripts/prepare-engine-bundle.mjs",
+    "release": "node scripts/release.mjs",
+    "start": "node dist/index.js",
+    "test": "npm run build && node --test --test-concurrency=1 test/*.test.mjs",
+    "typecheck": "tsc --noEmit"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "dependencies": {
+    "@octopus/bpmn": "1.0.4",
+    "@octopus/engine": "1.0.21",
+    "@puppeteer/browsers": "2.8.0",
+    "@types/lodash": "^4.14.0",
+    "@types/lodash-es": "4.17.12",
+    "async": "^2.6.4",
+    "axios": "1.15.0",
+    "bpmn-moddle": "^0.14.1",
+    "debug": "^2.6.9",
+    "fast-xml-parser": "4.5.0",
+    "iconv-lite": "0.6.3",
+    "iso8601-duration": "^1.3.0",
+    "jsonpath-plus": "10.3.0",
+    "lodash": "4.17.21",
+    "lodash-es": "4.17.21",
+    "moment": "2.30.1",
+    "puppeteer-core": "24.39.1",
+    "puppeteer-extra": "3.3.6",
+    "puppeteer-extra-plugin-block-resources": "2.4.3",
+    "puppeteer-proxy": "1.0.3",
+    "rebrowser-puppeteer-core": "24.8.1",
+    "sax": "1.4.1",
+    "setimmediate": "1.0.5",
+    "uuid": "10.0.0",
+    "ws": "^8.18.3",
+    "xml2js": "0.4.23",
+    "xmlbuilder": "15.1.1",
+    "xmldom": "0.6.0",
+    "xpath": "0.0.34"
+  },
+  "bundledDependencies": [
+    "@octopus/bpmn",
+    "@octopus/engine"
+  ],
+  "devDependencies": {
+    "@types/node": "^20.0.0",
+    "@types/ws": "^8.5.14",
+    "tsx": "^4.0.0",
+    "typescript": "^5.0.0"
+  }
+}