@jayfong/x-utils 1.0.0-202407012223

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/_cjs/ai.js +80 -0
  2. package/_cjs/cli.js +51 -0
  3. package/_cjs/clipboard.js +65 -0
  4. package/_cjs/cmd.js +42 -0
  5. package/_cjs/crawler.js +341 -0
  6. package/_cjs/csv.js +97 -0
  7. package/_cjs/data_set.js +115 -0
  8. package/_cjs/env.js +63 -0
  9. package/_cjs/fs.js +35 -0
  10. package/_cjs/html.js +62 -0
  11. package/_cjs/http_server.js +56 -0
  12. package/_cjs/json.js +49 -0
  13. package/_cjs/ksb.js +217 -0
  14. package/_cjs/number.js +39 -0
  15. package/_cjs/ocr.js +74 -0
  16. package/_cjs/path.js +35 -0
  17. package/_cjs/pinyin.js +32 -0
  18. package/_cjs/puppeteer.js +271 -0
  19. package/_cjs/runner.js +88 -0
  20. package/_cjs/task.js +41 -0
  21. package/_cjs/text.js +136 -0
  22. package/_cjs/unicode/lib_unicode.js +58643 -0
  23. package/_cjs/unicode/unicode.js +441 -0
  24. package/_cjs/upload.js +100 -0
  25. package/_cjs/yaml.js +52 -0
  26. package/ai.d.ts +26 -0
  27. package/ai.js +60 -0
  28. package/cli.d.ts +14 -0
  29. package/cli.js +21 -0
  30. package/clipboard.d.ts +6 -0
  31. package/clipboard.js +35 -0
  32. package/cmd.d.ts +5 -0
  33. package/cmd.js +22 -0
  34. package/crawler.d.ts +162 -0
  35. package/crawler.js +332 -0
  36. package/csv.d.ts +14 -0
  37. package/csv.js +67 -0
  38. package/data_set.d.ts +22 -0
  39. package/data_set.js +85 -0
  40. package/env.d.ts +4 -0
  41. package/env.js +32 -0
  42. package/fs.d.ts +2 -0
  43. package/fs.js +5 -0
  44. package/html.d.ts +7 -0
  45. package/html.js +45 -0
  46. package/http_server.d.ts +10 -0
  47. package/http_server.js +26 -0
  48. package/json.d.ts +6 -0
  49. package/json.js +19 -0
  50. package/ksb.d.ts +32 -0
  51. package/ksb.js +197 -0
  52. package/number.d.ts +3 -0
  53. package/number.js +9 -0
  54. package/ocr.d.ts +10 -0
  55. package/ocr.js +54 -0
  56. package/ofd.d.ts +1 -0
  57. package/package.json +158 -0
  58. package/path.d.ts +2 -0
  59. package/path.js +5 -0
  60. package/pinyin.d.ts +3 -0
  61. package/pinyin.js +12 -0
  62. package/puppeteer.d.ts +38 -0
  63. package/puppeteer.js +241 -0
  64. package/runner.d.ts +13 -0
  65. package/runner.js +58 -0
  66. package/task.d.ts +6 -0
  67. package/task.js +21 -0
  68. package/text.d.ts +63 -0
  69. package/text.js +119 -0
  70. package/unicode/lib_unicode.d.ts +4 -0
  71. package/unicode/lib_unicode.js +58623 -0
  72. package/unicode/unicode.d.ts +347 -0
  73. package/unicode/unicode.js +426 -0
  74. package/upload.d.ts +22 -0
  75. package/upload.js +70 -0
  76. package/yaml.d.ts +6 -0
  77. package/yaml.js +22 -0
package/_cjs/ai.js ADDED
@@ -0,0 +1,80 @@
1
+ "use strict";
2
+ var __defProp = Object.defineProperty;
3
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
+ var __getOwnPropNames = Object.getOwnPropertyNames;
5
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
6
+ var __export = (target, all) => {
7
+ for (var name in all)
8
+ __defProp(target, name, { get: all[name], enumerable: true });
9
+ };
10
+ var __copyProps = (to, from, except, desc) => {
11
+ if (from && typeof from === "object" || typeof from === "function") {
12
+ for (let key of __getOwnPropNames(from))
13
+ if (!__hasOwnProp.call(to, key) && key !== except)
14
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
15
+ }
16
+ return to;
17
+ };
18
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
19
+ var ai_exports = {};
20
+ __export(ai_exports, {
21
+ CozeAiProvider: () => CozeAiProvider,
22
+ XAi: () => XAi
23
+ });
24
+ module.exports = __toCommonJS(ai_exports);
25
+ var import_vtils = require("vtils");
26
+ var import_puppeteer = require("./puppeteer.js");
27
+ class AiProvider {
28
+ }
29
+ class CozeAiProvider extends AiProvider {
30
+ constructor(options) {
31
+ super();
32
+ this.options = options;
33
+ }
34
+ xp;
35
+ page;
36
+ async prepare() {
37
+ this.xp = new import_puppeteer.XPuppeteer();
38
+ await this.xp.connectSystemBrowser();
39
+ this.page = await this.xp.openPage(this.options.botUrl);
40
+ }
41
+ async ask(question) {
42
+ const page = this.page;
43
+ await page.evaluate(
44
+ () => document.querySelector("textarea:not([placeholder])").parentElement.parentElement.previousElementSibling.firstElementChild.click()
45
+ );
46
+ await (0, import_vtils.wait)(300);
47
+ await this.page.type("textarea:not([placeholder])", question);
48
+ await (0, import_vtils.wait)(500);
49
+ await this.page.keyboard.press("Enter");
50
+ await (0, import_vtils.wait)(1e3);
51
+ await (0, import_vtils.loopUntil)(
52
+ () => page.$$eval(
53
+ "button",
54
+ (els) => !els.find((el) => el.textContent.includes("Stop Responding"))?.parentElement.getBoundingClientRect().height
55
+ ),
56
+ {
57
+ retryDelay: 300
58
+ }
59
+ );
60
+ await (0, import_vtils.wait)(300);
61
+ const code = await page.$$eval(
62
+ 'div[class^="code-area"]',
63
+ (els) => els.pop().querySelector('[class^="content"]').textContent
64
+ );
65
+ return code;
66
+ }
67
+ }
68
+ class XAi {
69
+ constructor(options) {
70
+ this.options = options;
71
+ this.provider = options.provider;
72
+ }
73
+ provider;
74
+ prepare = (0, import_vtils.memoize)(() => this.provider.prepare());
75
+ async ask(question) {
76
+ await this.prepare();
77
+ const answer = await this.provider.ask(question);
78
+ return answer;
79
+ }
80
+ }
package/_cjs/cli.js ADDED
@@ -0,0 +1,51 @@
1
+ "use strict";
2
+ var __create = Object.create;
3
+ var __defProp = Object.defineProperty;
4
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
5
+ var __getOwnPropNames = Object.getOwnPropertyNames;
6
+ var __getProtoOf = Object.getPrototypeOf;
7
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
8
+ var __export = (target, all) => {
9
+ for (var name in all)
10
+ __defProp(target, name, { get: all[name], enumerable: true });
11
+ };
12
+ var __copyProps = (to, from, except, desc) => {
13
+ if (from && typeof from === "object" || typeof from === "function") {
14
+ for (let key of __getOwnPropNames(from))
15
+ if (!__hasOwnProp.call(to, key) && key !== except)
16
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
17
+ }
18
+ return to;
19
+ };
20
+ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
21
+ // If the importer is in node compatibility mode or this is not an ESM
22
+ // file that has been converted to a CommonJS file using a Babel-
23
+ // compatible transform (i.e. "__esModule" has not been set), then set
24
+ // "default" to the CommonJS "module.exports" for node compatibility.
25
+ isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
26
+ mod
27
+ ));
28
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
29
+ var cli_exports = {};
30
+ __export(cli_exports, {
31
+ XCli: () => XCli
32
+ });
33
+ module.exports = __toCommonJS(cli_exports);
34
+ var import_cli_table3 = __toESM(require("cli-table3"), 1);
35
+ var import_inquirer = __toESM(require("inquirer"), 1);
36
+ class XCli {
37
+ static async prompt(options) {
38
+ const { x } = await import_inquirer.default.prompt({
39
+ name: "x",
40
+ message: options.message,
41
+ type: options.type,
42
+ default: options.default
43
+ });
44
+ return x;
45
+ }
46
+ static table(data) {
47
+ const table = new import_cli_table3.default();
48
+ table.push(...data);
49
+ return table.toString();
50
+ }
51
+ }
@@ -0,0 +1,65 @@
1
+ "use strict";
2
+ var __create = Object.create;
3
+ var __defProp = Object.defineProperty;
4
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
5
+ var __getOwnPropNames = Object.getOwnPropertyNames;
6
+ var __getProtoOf = Object.getPrototypeOf;
7
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
8
+ var __export = (target, all) => {
9
+ for (var name in all)
10
+ __defProp(target, name, { get: all[name], enumerable: true });
11
+ };
12
+ var __copyProps = (to, from, except, desc) => {
13
+ if (from && typeof from === "object" || typeof from === "function") {
14
+ for (let key of __getOwnPropNames(from))
15
+ if (!__hasOwnProp.call(to, key) && key !== except)
16
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
17
+ }
18
+ return to;
19
+ };
20
+ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
21
+ // If the importer is in node compatibility mode or this is not an ESM
22
+ // file that has been converted to a CommonJS file using a Babel-
23
+ // compatible transform (i.e. "__esModule" has not been set), then set
24
+ // "default" to the CommonJS "module.exports" for node compatibility.
25
+ isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
26
+ mod
27
+ ));
28
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
29
+ var clipboard_exports = {};
30
+ __export(clipboard_exports, {
31
+ XClipboard: () => XClipboard
32
+ });
33
+ module.exports = __toCommonJS(clipboard_exports);
34
+ var import_clipboardy = __toESM(require("clipboardy"), 1);
35
+ var import_cmd = require("./cmd.js");
36
+ class XClipboard {
37
+ static async readText() {
38
+ return import_clipboardy.default.read();
39
+ }
40
+ static async readHtml() {
41
+ const hex = await import_cmd.XCmd.exec(
42
+ // 读取剪切板为 HTML
43
+ // https://stackoverflow.com/questions/2545289/getting-rtf-data-out-of-mac-os-x-pasteboard-clipboard
44
+ // https://stackoverflow.com/a/17217886
45
+ `osascript -e "the clipboard as \xABclass HTML\xBB"`
46
+ );
47
+ const html = Buffer.from(
48
+ hex.substring(10, hex.length - 1),
49
+ "hex"
50
+ ).toString();
51
+ return html;
52
+ }
53
+ static async writeText(text) {
54
+ await import_clipboardy.default.write(text);
55
+ }
56
+ static async writeHtml(html) {
57
+ const hex = Buffer.from(html).toString("hex");
58
+ await import_cmd.XCmd.exec(
59
+ // 设置剪切板为 HTML
60
+ // https://stackoverflow.com/questions/11085654/apple-script-how-can-i-copy-html-content-to-the-clipboard
61
+ // https://stackoverflow.com/a/11089226
62
+ `osascript -e "set the clipboard to \xABdata HTML${hex}\xBB"`
63
+ );
64
+ }
65
+ }
package/_cjs/cmd.js ADDED
@@ -0,0 +1,42 @@
1
+ "use strict";
2
+ var __defProp = Object.defineProperty;
3
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
+ var __getOwnPropNames = Object.getOwnPropertyNames;
5
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
6
+ var __export = (target, all) => {
7
+ for (var name in all)
8
+ __defProp(target, name, { get: all[name], enumerable: true });
9
+ };
10
+ var __copyProps = (to, from, except, desc) => {
11
+ if (from && typeof from === "object" || typeof from === "function") {
12
+ for (let key of __getOwnPropNames(from))
13
+ if (!__hasOwnProp.call(to, key) && key !== except)
14
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
15
+ }
16
+ return to;
17
+ };
18
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
19
+ var cmd_exports = {};
20
+ __export(cmd_exports, {
21
+ XCmd: () => XCmd
22
+ });
23
+ module.exports = __toCommonJS(cmd_exports);
24
+ var import_execa = require("execa");
25
+ class XCmd {
26
+ static execDivider = "clrue91c60000e6rl7w03b40p";
27
+ static async exec(command, options) {
28
+ const { stdout } = await (0, import_execa.execa)(
29
+ "zsh",
30
+ [
31
+ "-c",
32
+ `
33
+ echo ${XCmd.execDivider}
34
+ ${command}
35
+ echo ${XCmd.execDivider}
36
+ `
37
+ ],
38
+ options
39
+ );
40
+ return stdout.split(XCmd.execDivider)[1].trim();
41
+ }
42
+ }
@@ -0,0 +1,341 @@
1
+ "use strict";
2
+ var __defProp = Object.defineProperty;
3
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
+ var __getOwnPropNames = Object.getOwnPropertyNames;
5
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
6
+ var __export = (target, all) => {
7
+ for (var name in all)
8
+ __defProp(target, name, { get: all[name], enumerable: true });
9
+ };
10
+ var __copyProps = (to, from, except, desc) => {
11
+ if (from && typeof from === "object" || typeof from === "function") {
12
+ for (let key of __getOwnPropNames(from))
13
+ if (!__hasOwnProp.call(to, key) && key !== except)
14
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
15
+ }
16
+ return to;
17
+ };
18
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
19
+ var crawler_exports = {};
20
+ __export(crawler_exports, {
21
+ XCrawler: () => XCrawler
22
+ });
23
+ module.exports = __toCommonJS(crawler_exports);
24
+ var import_x_request = require("@jayfong/x-request");
25
+ var import_node_html_parser = require("node-html-parser");
26
+ var import_vtils = require("vtils");
27
+ var import_vae = require("vtils/vae");
28
+ var import_ocr = require("./ocr.js");
29
+ var import_puppeteer = require("./puppeteer.js");
30
+ class XCrawler extends import_vtils.EventBus {
31
+ constructor(config) {
32
+ super();
33
+ this.config = config;
34
+ this.schemaMap = config.schema(import_vae.v);
35
+ if (config.listen) {
36
+ this.on("newData", (newData) => {
37
+ config.listen(newData);
38
+ });
39
+ }
40
+ }
41
+ schemaMap;
42
+ id = Date.now().toString(36);
43
+ normalizeUrl(pageUrl, url) {
44
+ url = url.trim();
45
+ return !url ? "" : (0, import_vtils.isUrl)(url) ? url : new URL(
46
+ url.startsWith(".") || url.startsWith("/") ? url : `/${url}`,
47
+ pageUrl
48
+ ).toString();
49
+ }
50
+ async fetchUrlHeadless(pageUrl, url, options) {
51
+ const text = await import_puppeteer.XPuppeteer.getPageHtml(this.normalizeUrl(pageUrl, url), {
52
+ headful: false,
53
+ proxy: this.config.proxy,
54
+ blockAssets: true,
55
+ instanceId: this.id,
56
+ ...options
57
+ });
58
+ return text;
59
+ }
60
+ async fetchUrlHeadful(pageUrl, url, options) {
61
+ const text = await import_puppeteer.XPuppeteer.getPageHtml(this.normalizeUrl(pageUrl, url), {
62
+ headful: true,
63
+ proxy: this.config.proxy,
64
+ blockAssets: true,
65
+ instanceId: this.id,
66
+ ...options
67
+ });
68
+ return text;
69
+ }
70
+ async fetchUrlRequest(pageUrl, url) {
71
+ const absoluteUrl = this.normalizeUrl(pageUrl, url);
72
+ const { data: text } = await import_x_request.xr.getText({
73
+ url: absoluteUrl,
74
+ autoReferer: true,
75
+ proxyUrl: this.config.proxy
76
+ });
77
+ return text;
78
+ }
79
+ async fetchUrl(pageUrl, url, engine, options) {
80
+ const text = engine === "headless" ? await this.fetchUrlHeadless(pageUrl, url, options) : engine === "headful" ? await this.fetchUrlHeadful(pageUrl, url, options) : await this.fetchUrlRequest(pageUrl, url);
81
+ return text;
82
+ }
83
+ async fetchUrlAsHTMLElement(pageUrl, url, engine, options) {
84
+ const text = await this.fetchUrl(pageUrl, url, engine, options);
85
+ return (0, import_node_html_parser.parse)(text);
86
+ }
87
+ async fetchUrlAsJSON(pageUrl, url, engine) {
88
+ const text = await this.fetchUrl(pageUrl, url, engine);
89
+ return JSON.parse(text);
90
+ }
91
+ async ocr(pageUrl, el) {
92
+ const imgDocs = el.querySelectorAll("img");
93
+ for (const imgDoc of imgDocs) {
94
+ const imgUrl = imgDoc.getAttribute("src");
95
+ if (imgUrl) {
96
+ const ocrText = await import_ocr.XOcr.recognize(
97
+ this.normalizeUrl(pageUrl, imgUrl)
98
+ );
99
+ imgDoc.replaceWith(`<div>${(0, import_vtils.escape)(ocrText)} </div>`);
100
+ }
101
+ }
102
+ }
103
+ findNextById = (0, import_vtils.memoize)((id) => {
104
+ const findNextById = (nexts) => {
105
+ for (const next of (0, import_vtils.castArray)(nexts)) {
106
+ if (typeof next !== "string") {
107
+ if (next.id === id) {
108
+ return next;
109
+ }
110
+ if (next.next) {
111
+ const _next = findNextById(next.next);
112
+ if (_next) {
113
+ return _next;
114
+ }
115
+ }
116
+ }
117
+ }
118
+ };
119
+ return this.config.next && findNextById(this.config.next);
120
+ });
121
+ pageUrlCache = /* @__PURE__ */ new Set();
122
+ async next(pageUrl, contentType, engine, elOrField, data, next, index, rootElement) {
123
+ if (next.filter && !await next.filter(data)) {
124
+ return;
125
+ }
126
+ contentType = next.contentType || contentType;
127
+ engine = next.engine || engine;
128
+ if (contentType === "json") {
129
+ const field = elOrField;
130
+ const nextFields = next.url ? await Promise.all(
131
+ (0, import_vtils.castArray)(
132
+ typeof next.url === "function" ? await next.url(data) : data[next.url]
133
+ ).map((url) => this.fetchUrlAsJSON(pageUrl, url, engine))
134
+ ) : !next.field || next.field === "." ? [field] : typeof next.field === "string" ? (0, import_vtils.get)(field, next.field) : (0, import_vtils.castArray)(await next.field(field));
135
+ for (const nextField of nextFields) {
136
+ const newData = (0, import_vtils.cloneDeepFast)(data);
137
+ if (next.extract) {
138
+ const fieldKeys = (0, import_vtils.keysStrict)(next.extract);
139
+ for (const fieldKey of fieldKeys) {
140
+ const fieldConfig = typeof next.extract[fieldKey] === "string" || typeof next.extract[fieldKey] === "function" || Array.isArray(next.extract[fieldKey]) ? { select: next.extract[fieldKey] } : next.extract[fieldKey];
141
+ let fieldValue = !fieldConfig.select || fieldConfig.select === "." ? nextField : typeof fieldConfig.select === "string" ? (0, import_vtils.get)(nextField, fieldConfig.select) : Array.isArray(fieldConfig.select) ? "TODO" : "TODO";
142
+ if (fieldValue != null) {
143
+ fieldValue = fieldConfig.transform ? await fieldConfig.transform(fieldValue) : fieldValue;
144
+ const fieldSchema = this.schemaMap[fieldKey];
145
+ fieldValue = fieldSchema.cast(fieldValue);
146
+ newData[fieldKey] = fieldValue;
147
+ }
148
+ }
149
+ }
150
+ if (next.next) {
151
+ for (const nextItem of (0, import_vtils.castArray)(next.next)) {
152
+ await this.next(
153
+ pageUrl,
154
+ contentType,
155
+ engine,
156
+ nextField,
157
+ newData,
158
+ typeof nextItem === "string" ? this.findNextById(nextItem) : nextItem,
159
+ 0,
160
+ rootElement
161
+ );
162
+ }
163
+ } else {
164
+ this.emit("newData", newData);
165
+ }
166
+ }
167
+ } else {
168
+ const el = elOrField;
169
+ const nextEls = next.url ? await Promise.all(
170
+ next._url.map(
171
+ (fullUrl) => this.fetchUrlAsHTMLElement(
172
+ pageUrl,
173
+ fullUrl,
174
+ engine,
175
+ next.engineOptions
176
+ )
177
+ )
178
+ ) : !next.element || next.element === "." ? [el] : typeof next.element === "string" ? el.querySelectorAll(next.element) : (0, import_vtils.castArray)(
179
+ await next.element(el, {
180
+ index,
181
+ root: rootElement
182
+ })
183
+ );
184
+ const nextParamsList = [];
185
+ let nextElIndex = 0;
186
+ for (const nextEl of nextEls) {
187
+ const newData = (0, import_vtils.cloneDeepFast)(data);
188
+ if (next.extract) {
189
+ const fieldKeys = (0, import_vtils.keysStrict)(next.extract);
190
+ for (const fieldKey of fieldKeys) {
191
+ const fieldConfig = typeof next.extract[fieldKey] === "string" || typeof next.extract[fieldKey] === "function" || Array.isArray(next.extract[fieldKey]) ? { select: next.extract[fieldKey] } : next.extract[fieldKey];
192
+ const fieldEl = !fieldConfig.select || fieldConfig.select === "." ? nextEl : typeof fieldConfig.select === "string" ? nextEl.querySelector(fieldConfig.select) : Array.isArray(fieldConfig.select) ? nextEl.querySelectorAll(fieldConfig.select[0]).find((item) => {
193
+ const filter = fieldConfig.select[1];
194
+ return typeof filter === "string" ? item.text.includes(filter) : (0, import_vtils.isRegExp)(filter) ? filter.test(item.text) : filter(item);
195
+ }) : await fieldConfig.select(nextEl);
196
+ if (fieldEl != null) {
197
+ let fieldText = "";
198
+ if (typeof fieldEl === "string") {
199
+ fieldText = fieldEl.trim();
200
+ } else {
201
+ if (fieldConfig.ocr) {
202
+ await this.ocr(pageUrl, fieldEl);
203
+ }
204
+ switch (fieldConfig.extract) {
205
+ case "getHTML":
206
+ fieldText = fieldEl.outerHTML;
207
+ break;
208
+ case "getFirstImageUrl":
209
+ fieldText = this.normalizeUrl(
210
+ pageUrl,
211
+ fieldEl.getAttribute("src") || fieldEl.querySelector("img")?.getAttribute("src") || ""
212
+ );
213
+ break;
214
+ case "getFirstLinkUrl":
215
+ fieldText = this.normalizeUrl(
216
+ pageUrl,
217
+ fieldEl.getAttribute("href") || fieldEl.querySelector("a")?.getAttribute("href") || ""
218
+ );
219
+ break;
220
+ default:
221
+ fieldText = fieldEl.text.trim();
222
+ break;
223
+ }
224
+ }
225
+ let fieldValue = fieldConfig.transform ? await fieldConfig.transform(fieldText) : fieldText;
226
+ const fieldSchema = this.schemaMap[fieldKey];
227
+ if (fieldSchema.options.type !== "array") {
228
+ fieldValue = fieldSchema.cast(fieldValue);
229
+ } else {
230
+ fieldValue = fieldSchema.cast([
231
+ ...newData[fieldKey] || [],
232
+ fieldValue
233
+ ]);
234
+ }
235
+ newData[fieldKey] = fieldValue;
236
+ }
237
+ }
238
+ }
239
+ if (next.next) {
240
+ const nexts = await Promise.all(
241
+ (0, import_vtils.castArray)(next.next).map(async (nextItem) => {
242
+ nextItem = (0, import_vtils.cloneDeepFast)(
243
+ typeof nextItem === "string" ? this.findNextById(nextItem) : nextItem
244
+ );
245
+ if (nextItem.url) {
246
+ nextItem._url = (0, import_vtils.castArray)(
247
+ typeof nextItem.url === "function" ? await nextItem.url({
248
+ data: newData,
249
+ el: nextEl
250
+ }) : newData[nextItem.url]
251
+ ).map((url) => this.normalizeUrl(pageUrl, url)).filter((fullUrl) => !this.pageUrlCache.has(fullUrl)).map((fullUrl) => {
252
+ this.pageUrlCache.add(fullUrl);
253
+ return fullUrl;
254
+ });
255
+ }
256
+ return nextItem;
257
+ })
258
+ );
259
+ let i = 0;
260
+ for (const nextItem of nexts) {
261
+ nextParamsList.push([
262
+ nextItem.url && nextItem._url[i] || pageUrl,
263
+ contentType,
264
+ engine,
265
+ nextEl,
266
+ newData,
267
+ nextItem,
268
+ nextElIndex,
269
+ rootElement
270
+ ]);
271
+ i++;
272
+ }
273
+ } else {
274
+ this.emit("newData", newData);
275
+ }
276
+ nextElIndex++;
277
+ }
278
+ for (const nextParams of nextParamsList) {
279
+ await this.next(...nextParams);
280
+ if (next.nextPageUrl) {
281
+ const nextPageUrl = await next.nextPageUrl({
282
+ data: nextParams[4],
283
+ el: nextParams[3],
284
+ url: nextParams[0]
285
+ });
286
+ if (nextPageUrl) {
287
+ const nextPageAbsoluteUrl = this.normalizeUrl(
288
+ nextParams[0],
289
+ nextPageUrl
290
+ );
291
+ if (nextPageAbsoluteUrl !== nextParams[0]) {
292
+ const nextPageElement = await this.fetchUrlAsHTMLElement(
293
+ nextPageAbsoluteUrl,
294
+ nextPageAbsoluteUrl,
295
+ engine,
296
+ this.config.engineOptions
297
+ );
298
+ await this.next(
299
+ nextPageAbsoluteUrl,
300
+ contentType,
301
+ engine,
302
+ nextPageElement,
303
+ nextParams[4],
304
+ (0, import_vtils.omitStrict)(next, ["url"]),
305
+ index,
306
+ rootElement
307
+ );
308
+ }
309
+ }
310
+ }
311
+ }
312
+ }
313
+ }
314
+ async start() {
315
+ const initialContentType = this.config.contentType || "html";
316
+ const initialEngine = this.config.engine || "request";
317
+ const rootElement = initialContentType === "json" ? await this.fetchUrlAsJSON(
318
+ this.config.entry,
319
+ this.config.entry,
320
+ initialEngine
321
+ ) : await this.fetchUrlAsHTMLElement(
322
+ this.config.entry,
323
+ this.config.entry,
324
+ initialEngine,
325
+ this.config.engineOptions
326
+ );
327
+ await this.next(
328
+ this.config.entry,
329
+ initialContentType,
330
+ initialEngine,
331
+ rootElement,
332
+ {},
333
+ {
334
+ extract: this.config.extract,
335
+ next: this.config.next
336
+ },
337
+ 0,
338
+ rootElement
339
+ );
340
+ }
341
+ }
package/_cjs/csv.js ADDED
@@ -0,0 +1,97 @@
1
+ "use strict";
2
+ var __create = Object.create;
3
+ var __defProp = Object.defineProperty;
4
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
5
+ var __getOwnPropNames = Object.getOwnPropertyNames;
6
+ var __getProtoOf = Object.getPrototypeOf;
7
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
8
+ var __export = (target, all) => {
9
+ for (var name in all)
10
+ __defProp(target, name, { get: all[name], enumerable: true });
11
+ };
12
+ var __copyProps = (to, from, except, desc) => {
13
+ if (from && typeof from === "object" || typeof from === "function") {
14
+ for (let key of __getOwnPropNames(from))
15
+ if (!__hasOwnProp.call(to, key) && key !== except)
16
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
17
+ }
18
+ return to;
19
+ };
20
+ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
21
+ // If the importer is in node compatibility mode or this is not an ESM
22
+ // file that has been converted to a CommonJS file using a Babel-
23
+ // compatible transform (i.e. "__esModule" has not been set), then set
24
+ // "default" to the CommonJS "module.exports" for node compatibility.
25
+ isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
26
+ mod
27
+ ));
28
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
29
+ var csv_exports = {};
30
+ __export(csv_exports, {
31
+ XCsv: () => XCsv
32
+ });
33
+ module.exports = __toCommonJS(csv_exports);
34
+ var csv = __toESM(require("csv"), 1);
35
+ var fs = __toESM(require("node:fs"), 1);
36
+ var import_vtils = require("vtils");
37
+ var import_date = require("vtils/date");
38
+ class XCsv {
39
+ static async parse(value, options = {}) {
40
+ const res = [];
41
+ for await (const item of csv.parse(value, {
42
+ columns: true
43
+ })) {
44
+ res.push(
45
+ options.objectFieldNames?.length || options.booleanFieldNames?.length || options.dateFieldNames?.length || options.arrayFieldNames?.length && options.arrayDelimiter ? (0, import_vtils.mapValues)(item, (v, k) => {
46
+ if (options.arrayFieldNames.includes(k)) {
47
+ return v.split(options.arrayDelimiter).filter(Boolean);
48
+ }
49
+ if (options.objectFieldNames?.includes(k)) {
50
+ return v ? JSON.parse(v) : {};
51
+ }
52
+ if (options.booleanFieldNames?.includes(k)) {
53
+ return v ? v === 1 : void 0;
54
+ }
55
+ if (options.dateFieldNames?.includes(k)) {
56
+ return v ? new Date(v) : void 0;
57
+ }
58
+ return v;
59
+ }) : item
60
+ );
61
+ }
62
+ return res;
63
+ }
64
+ static async stringify(value, options = {}) {
65
+ let res = "";
66
+ for await (const line of csv.stringify(
67
+ options.objectFieldNames?.length || options.booleanFieldNames?.length || options.dateFieldNames?.length || options.arrayFieldNames?.length && options.arrayDelimiter ? value.map(
68
+ (item) => (0, import_vtils.mapValues)(item, (v, k) => {
69
+ if (options.arrayFieldNames.includes(k)) {
70
+ return v ? v.join(options.arrayDelimiter) : "";
71
+ }
72
+ if (options.objectFieldNames?.includes(k)) {
73
+ return JSON.stringify(v || {});
74
+ }
75
+ if (options.booleanFieldNames?.includes(k)) {
76
+ return v ? "1" : "0";
77
+ }
78
+ if (options.dateFieldNames?.includes(k)) {
79
+ return v ? (0, import_date.formatDate)(v, "yyyy-mm-dd hh:ii:ss") : v;
80
+ }
81
+ return v;
82
+ })
83
+ ) : value,
84
+ { header: true }
85
+ )) {
86
+ res += line;
87
+ }
88
+ return res;
89
+ }
90
+ static async read(file, options = {}) {
91
+ const value = await fs.promises.readFile(file, "utf8");
92
+ return XCsv.parse(value, options);
93
+ }
94
+ static async write(file, value, options = {}) {
95
+ await fs.promises.writeFile(file, await XCsv.stringify(value, options));
96
+ }
97
+ }