@jayfong/x-utils 1.0.0-202510061015 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/case.d.mts +184 -0
- package/lib/case.mjs +120 -0
- package/lib/chinese.d.mts +39 -0
- package/lib/chinese.mjs +52 -0
- package/lib/cli.d.mts +64 -0
- package/lib/cli.mjs +59 -0
- package/lib/clipboard.d.mts +50 -0
- package/lib/clipboard.mjs +122 -0
- package/lib/cmd.d.mts +38 -0
- package/lib/cmd.mjs +52 -0
- package/lib/crawler.d.mts +330 -0
- package/lib/crawler.mjs +313 -0
- package/lib/csv.d.mts +67 -0
- package/lib/csv.mjs +76 -0
- package/lib/data_set.d.mts +76 -0
- package/lib/data_set.mjs +104 -0
- package/lib/fs.d.mts +49 -0
- package/lib/fs.mjs +65 -0
- package/lib/json.d.mts +50 -0
- package/lib/json.mjs +74 -0
- package/lib/puppeteer.d.mts +88 -0
- package/lib/puppeteer.mjs +219 -0
- package/lib/runner.d.mts +52 -0
- package/lib/runner.mjs +50 -0
- package/lib/task.d.mts +37 -0
- package/lib/task.mjs +43 -0
- package/lib/text.d.mts +84 -0
- package/lib/text.mjs +141 -0
- package/lib/yaml.d.mts +51 -0
- package/lib/yaml.mjs +64 -0
- package/package.json +50 -164
- package/_cjs/ai.js +0 -80
- package/_cjs/alipan.js +0 -173
- package/_cjs/cli.js +0 -51
- package/_cjs/clipboard.js +0 -65
- package/_cjs/cmd.js +0 -42
- package/_cjs/compressor.js +0 -47
- package/_cjs/crawler.js +0 -341
- package/_cjs/csv.js +0 -97
- package/_cjs/data_set.js +0 -115
- package/_cjs/env.js +0 -82
- package/_cjs/fs.js +0 -86
- package/_cjs/html.js +0 -62
- package/_cjs/http_server.js +0 -79
- package/_cjs/json.js +0 -49
- package/_cjs/ksb.js +0 -218
- package/_cjs/number.js +0 -39
- package/_cjs/ocr.js +0 -74
- package/_cjs/package.json +0 -3
- package/_cjs/path.js +0 -35
- package/_cjs/pinyin.js +0 -37
- package/_cjs/puppeteer.js +0 -271
- package/_cjs/runner.js +0 -88
- package/_cjs/task.js +0 -41
- package/_cjs/text.js +0 -136
- package/_cjs/unicode/lib_unicode.js +0 -58643
- package/_cjs/unicode/unicode.js +0 -441
- package/_cjs/upload.js +0 -101
- package/_cjs/yaml.js +0 -52
- package/ai.d.ts +0 -26
- package/ai.js +0 -60
- package/alipan.d.ts +0 -209
- package/alipan.js +0 -153
- package/cli.d.ts +0 -14
- package/cli.js +0 -21
- package/clipboard.d.ts +0 -6
- package/clipboard.js +0 -35
- package/cmd.d.ts +0 -5
- package/cmd.js +0 -22
- package/compressor.d.ts +0 -5
- package/compressor.js +0 -17
- package/crawler.d.ts +0 -162
- package/crawler.js +0 -332
- package/csv.d.ts +0 -14
- package/csv.js +0 -67
- package/data_set.d.ts +0 -22
- package/data_set.js +0 -85
- package/env.d.ts +0 -6
- package/env.js +0 -52
- package/fs.d.ts +0 -23
- package/fs.js +0 -56
- package/html.d.ts +0 -7
- package/html.js +0 -45
- package/http_server.d.ts +0 -13
- package/http_server.js +0 -49
- package/json.d.ts +0 -6
- package/json.js +0 -19
- package/ksb.d.ts +0 -37
- package/ksb.js +0 -198
- package/number.d.ts +0 -3
- package/number.js +0 -9
- package/ocr.d.ts +0 -17
- package/ocr.js +0 -54
- package/path.d.ts +0 -2
- package/path.js +0 -5
- package/pinyin.d.ts +0 -8
- package/pinyin.js +0 -17
- package/puppeteer.d.ts +0 -38
- package/puppeteer.js +0 -241
- package/runner.d.ts +0 -13
- package/runner.js +0 -58
- package/task.d.ts +0 -6
- package/task.js +0 -21
- package/text.d.ts +0 -63
- package/text.js +0 -119
- package/unicode/lib_unicode.d.ts +0 -4
- package/unicode/lib_unicode.js +0 -58623
- package/unicode/unicode.d.ts +0 -347
- package/unicode/unicode.js +0 -426
- package/upload.d.ts +0 -34
- package/upload.js +0 -71
- package/yaml.d.ts +0 -6
- package/yaml.js +0 -22
package/_cjs/crawler.js
DELETED
|
@@ -1,341 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __defProp = Object.defineProperty;
|
|
3
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
-
var __export = (target, all) => {
|
|
7
|
-
for (var name in all)
|
|
8
|
-
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
-
};
|
|
10
|
-
var __copyProps = (to, from, except, desc) => {
|
|
11
|
-
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
-
for (let key of __getOwnPropNames(from))
|
|
13
|
-
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
-
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
-
}
|
|
16
|
-
return to;
|
|
17
|
-
};
|
|
18
|
-
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
-
var crawler_exports = {};
|
|
20
|
-
__export(crawler_exports, {
|
|
21
|
-
XCrawler: () => XCrawler
|
|
22
|
-
});
|
|
23
|
-
module.exports = __toCommonJS(crawler_exports);
|
|
24
|
-
var import_x_request = require("@jayfong/x-request");
|
|
25
|
-
var import_node_html_parser = require("node-html-parser");
|
|
26
|
-
var import_vtils = require("vtils");
|
|
27
|
-
var import_vae = require("vtils/vae");
|
|
28
|
-
var import_ocr = require("./ocr.js");
|
|
29
|
-
var import_puppeteer = require("./puppeteer.js");
|
|
30
|
-
class XCrawler extends import_vtils.EventBus {
|
|
31
|
-
constructor(config) {
|
|
32
|
-
super();
|
|
33
|
-
this.config = config;
|
|
34
|
-
this.schemaMap = config.schema(import_vae.v);
|
|
35
|
-
if (config.listen) {
|
|
36
|
-
this.on("newData", (newData) => {
|
|
37
|
-
config.listen(newData);
|
|
38
|
-
});
|
|
39
|
-
}
|
|
40
|
-
}
|
|
41
|
-
schemaMap;
|
|
42
|
-
id = Date.now().toString(36);
|
|
43
|
-
normalizeUrl(pageUrl, url) {
|
|
44
|
-
url = url.trim();
|
|
45
|
-
return !url ? "" : (0, import_vtils.isUrl)(url) ? url : new URL(
|
|
46
|
-
url.startsWith(".") || url.startsWith("/") ? url : `/${url}`,
|
|
47
|
-
pageUrl
|
|
48
|
-
).toString();
|
|
49
|
-
}
|
|
50
|
-
async fetchUrlHeadless(pageUrl, url, options) {
|
|
51
|
-
const text = await import_puppeteer.XPuppeteer.getPageHtml(this.normalizeUrl(pageUrl, url), {
|
|
52
|
-
headful: false,
|
|
53
|
-
proxy: this.config.proxy,
|
|
54
|
-
blockAssets: true,
|
|
55
|
-
instanceId: this.id,
|
|
56
|
-
...options
|
|
57
|
-
});
|
|
58
|
-
return text;
|
|
59
|
-
}
|
|
60
|
-
async fetchUrlHeadful(pageUrl, url, options) {
|
|
61
|
-
const text = await import_puppeteer.XPuppeteer.getPageHtml(this.normalizeUrl(pageUrl, url), {
|
|
62
|
-
headful: true,
|
|
63
|
-
proxy: this.config.proxy,
|
|
64
|
-
blockAssets: true,
|
|
65
|
-
instanceId: this.id,
|
|
66
|
-
...options
|
|
67
|
-
});
|
|
68
|
-
return text;
|
|
69
|
-
}
|
|
70
|
-
async fetchUrlRequest(pageUrl, url) {
|
|
71
|
-
const absoluteUrl = this.normalizeUrl(pageUrl, url);
|
|
72
|
-
const { data: text } = await import_x_request.xr.getText({
|
|
73
|
-
url: absoluteUrl,
|
|
74
|
-
autoReferer: true,
|
|
75
|
-
proxyUrl: this.config.proxy
|
|
76
|
-
});
|
|
77
|
-
return text;
|
|
78
|
-
}
|
|
79
|
-
async fetchUrl(pageUrl, url, engine, options) {
|
|
80
|
-
const text = engine === "headless" ? await this.fetchUrlHeadless(pageUrl, url, options) : engine === "headful" ? await this.fetchUrlHeadful(pageUrl, url, options) : await this.fetchUrlRequest(pageUrl, url);
|
|
81
|
-
return text;
|
|
82
|
-
}
|
|
83
|
-
async fetchUrlAsHTMLElement(pageUrl, url, engine, options) {
|
|
84
|
-
const text = await this.fetchUrl(pageUrl, url, engine, options);
|
|
85
|
-
return (0, import_node_html_parser.parse)(text);
|
|
86
|
-
}
|
|
87
|
-
async fetchUrlAsJSON(pageUrl, url, engine) {
|
|
88
|
-
const text = await this.fetchUrl(pageUrl, url, engine);
|
|
89
|
-
return JSON.parse(text);
|
|
90
|
-
}
|
|
91
|
-
async ocr(pageUrl, el) {
|
|
92
|
-
const imgDocs = el.querySelectorAll("img");
|
|
93
|
-
for (const imgDoc of imgDocs) {
|
|
94
|
-
const imgUrl = imgDoc.getAttribute("src");
|
|
95
|
-
if (imgUrl) {
|
|
96
|
-
const ocrText = await import_ocr.XOcr.recognize(
|
|
97
|
-
this.normalizeUrl(pageUrl, imgUrl)
|
|
98
|
-
);
|
|
99
|
-
imgDoc.replaceWith(`<div>${(0, import_vtils.escape)(ocrText)} </div>`);
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
findNextById = (0, import_vtils.memoize)((id) => {
|
|
104
|
-
const findNextById = (nexts) => {
|
|
105
|
-
for (const next of (0, import_vtils.castArray)(nexts)) {
|
|
106
|
-
if (typeof next !== "string") {
|
|
107
|
-
if (next.id === id) {
|
|
108
|
-
return next;
|
|
109
|
-
}
|
|
110
|
-
if (next.next) {
|
|
111
|
-
const _next = findNextById(next.next);
|
|
112
|
-
if (_next) {
|
|
113
|
-
return _next;
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
};
|
|
119
|
-
return this.config.next && findNextById(this.config.next);
|
|
120
|
-
});
|
|
121
|
-
pageUrlCache = /* @__PURE__ */ new Set();
|
|
122
|
-
async next(pageUrl, contentType, engine, elOrField, data, next, index, rootElement) {
|
|
123
|
-
if (next.filter && !await next.filter(data)) {
|
|
124
|
-
return;
|
|
125
|
-
}
|
|
126
|
-
contentType = next.contentType || contentType;
|
|
127
|
-
engine = next.engine || engine;
|
|
128
|
-
if (contentType === "json") {
|
|
129
|
-
const field = elOrField;
|
|
130
|
-
const nextFields = next.url ? await Promise.all(
|
|
131
|
-
(0, import_vtils.castArray)(
|
|
132
|
-
typeof next.url === "function" ? await next.url(data) : data[next.url]
|
|
133
|
-
).map((url) => this.fetchUrlAsJSON(pageUrl, url, engine))
|
|
134
|
-
) : !next.field || next.field === "." ? [field] : typeof next.field === "string" ? (0, import_vtils.get)(field, next.field) : (0, import_vtils.castArray)(await next.field(field));
|
|
135
|
-
for (const nextField of nextFields) {
|
|
136
|
-
const newData = (0, import_vtils.cloneDeepFast)(data);
|
|
137
|
-
if (next.extract) {
|
|
138
|
-
const fieldKeys = (0, import_vtils.keysStrict)(next.extract);
|
|
139
|
-
for (const fieldKey of fieldKeys) {
|
|
140
|
-
const fieldConfig = typeof next.extract[fieldKey] === "string" || typeof next.extract[fieldKey] === "function" || Array.isArray(next.extract[fieldKey]) ? { select: next.extract[fieldKey] } : next.extract[fieldKey];
|
|
141
|
-
let fieldValue = !fieldConfig.select || fieldConfig.select === "." ? nextField : typeof fieldConfig.select === "string" ? (0, import_vtils.get)(nextField, fieldConfig.select) : Array.isArray(fieldConfig.select) ? "TODO" : "TODO";
|
|
142
|
-
if (fieldValue != null) {
|
|
143
|
-
fieldValue = fieldConfig.transform ? await fieldConfig.transform(fieldValue) : fieldValue;
|
|
144
|
-
const fieldSchema = this.schemaMap[fieldKey];
|
|
145
|
-
fieldValue = fieldSchema.cast(fieldValue);
|
|
146
|
-
newData[fieldKey] = fieldValue;
|
|
147
|
-
}
|
|
148
|
-
}
|
|
149
|
-
}
|
|
150
|
-
if (next.next) {
|
|
151
|
-
for (const nextItem of (0, import_vtils.castArray)(next.next)) {
|
|
152
|
-
await this.next(
|
|
153
|
-
pageUrl,
|
|
154
|
-
contentType,
|
|
155
|
-
engine,
|
|
156
|
-
nextField,
|
|
157
|
-
newData,
|
|
158
|
-
typeof nextItem === "string" ? this.findNextById(nextItem) : nextItem,
|
|
159
|
-
0,
|
|
160
|
-
rootElement
|
|
161
|
-
);
|
|
162
|
-
}
|
|
163
|
-
} else {
|
|
164
|
-
this.emit("newData", newData);
|
|
165
|
-
}
|
|
166
|
-
}
|
|
167
|
-
} else {
|
|
168
|
-
const el = elOrField;
|
|
169
|
-
const nextEls = next.url ? await Promise.all(
|
|
170
|
-
next._url.map(
|
|
171
|
-
(fullUrl) => this.fetchUrlAsHTMLElement(
|
|
172
|
-
pageUrl,
|
|
173
|
-
fullUrl,
|
|
174
|
-
engine,
|
|
175
|
-
next.engineOptions
|
|
176
|
-
)
|
|
177
|
-
)
|
|
178
|
-
) : !next.element || next.element === "." ? [el] : typeof next.element === "string" ? el.querySelectorAll(next.element) : (0, import_vtils.castArray)(
|
|
179
|
-
await next.element(el, {
|
|
180
|
-
index,
|
|
181
|
-
root: rootElement
|
|
182
|
-
})
|
|
183
|
-
);
|
|
184
|
-
const nextParamsList = [];
|
|
185
|
-
let nextElIndex = 0;
|
|
186
|
-
for (const nextEl of nextEls) {
|
|
187
|
-
const newData = (0, import_vtils.cloneDeepFast)(data);
|
|
188
|
-
if (next.extract) {
|
|
189
|
-
const fieldKeys = (0, import_vtils.keysStrict)(next.extract);
|
|
190
|
-
for (const fieldKey of fieldKeys) {
|
|
191
|
-
const fieldConfig = typeof next.extract[fieldKey] === "string" || typeof next.extract[fieldKey] === "function" || Array.isArray(next.extract[fieldKey]) ? { select: next.extract[fieldKey] } : next.extract[fieldKey];
|
|
192
|
-
const fieldEl = !fieldConfig.select || fieldConfig.select === "." ? nextEl : typeof fieldConfig.select === "string" ? nextEl.querySelector(fieldConfig.select) : Array.isArray(fieldConfig.select) ? nextEl.querySelectorAll(fieldConfig.select[0]).find((item) => {
|
|
193
|
-
const filter = fieldConfig.select[1];
|
|
194
|
-
return typeof filter === "string" ? item.text.includes(filter) : (0, import_vtils.isRegExp)(filter) ? filter.test(item.text) : filter(item);
|
|
195
|
-
}) : await fieldConfig.select(nextEl);
|
|
196
|
-
if (fieldEl != null) {
|
|
197
|
-
let fieldText = "";
|
|
198
|
-
if (typeof fieldEl === "string") {
|
|
199
|
-
fieldText = fieldEl.trim();
|
|
200
|
-
} else {
|
|
201
|
-
if (fieldConfig.ocr) {
|
|
202
|
-
await this.ocr(pageUrl, fieldEl);
|
|
203
|
-
}
|
|
204
|
-
switch (fieldConfig.extract) {
|
|
205
|
-
case "getHTML":
|
|
206
|
-
fieldText = fieldEl.outerHTML;
|
|
207
|
-
break;
|
|
208
|
-
case "getFirstImageUrl":
|
|
209
|
-
fieldText = this.normalizeUrl(
|
|
210
|
-
pageUrl,
|
|
211
|
-
fieldEl.getAttribute("src") || fieldEl.querySelector("img")?.getAttribute("src") || ""
|
|
212
|
-
);
|
|
213
|
-
break;
|
|
214
|
-
case "getFirstLinkUrl":
|
|
215
|
-
fieldText = this.normalizeUrl(
|
|
216
|
-
pageUrl,
|
|
217
|
-
fieldEl.getAttribute("href") || fieldEl.querySelector("a")?.getAttribute("href") || ""
|
|
218
|
-
);
|
|
219
|
-
break;
|
|
220
|
-
default:
|
|
221
|
-
fieldText = fieldEl.text.trim();
|
|
222
|
-
break;
|
|
223
|
-
}
|
|
224
|
-
}
|
|
225
|
-
let fieldValue = fieldConfig.transform ? await fieldConfig.transform(fieldText) : fieldText;
|
|
226
|
-
const fieldSchema = this.schemaMap[fieldKey];
|
|
227
|
-
if (fieldSchema.options.type !== "array") {
|
|
228
|
-
fieldValue = fieldSchema.cast(fieldValue);
|
|
229
|
-
} else {
|
|
230
|
-
fieldValue = fieldSchema.cast([
|
|
231
|
-
...newData[fieldKey] || [],
|
|
232
|
-
fieldValue
|
|
233
|
-
]);
|
|
234
|
-
}
|
|
235
|
-
newData[fieldKey] = fieldValue;
|
|
236
|
-
}
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
if (next.next) {
|
|
240
|
-
const nexts = await Promise.all(
|
|
241
|
-
(0, import_vtils.castArray)(next.next).map(async (nextItem) => {
|
|
242
|
-
nextItem = (0, import_vtils.cloneDeepFast)(
|
|
243
|
-
typeof nextItem === "string" ? this.findNextById(nextItem) : nextItem
|
|
244
|
-
);
|
|
245
|
-
if (nextItem.url) {
|
|
246
|
-
nextItem._url = (0, import_vtils.castArray)(
|
|
247
|
-
typeof nextItem.url === "function" ? await nextItem.url({
|
|
248
|
-
data: newData,
|
|
249
|
-
el: nextEl
|
|
250
|
-
}) : newData[nextItem.url]
|
|
251
|
-
).map((url) => this.normalizeUrl(pageUrl, url)).filter((fullUrl) => !this.pageUrlCache.has(fullUrl)).map((fullUrl) => {
|
|
252
|
-
this.pageUrlCache.add(fullUrl);
|
|
253
|
-
return fullUrl;
|
|
254
|
-
});
|
|
255
|
-
}
|
|
256
|
-
return nextItem;
|
|
257
|
-
})
|
|
258
|
-
);
|
|
259
|
-
let i = 0;
|
|
260
|
-
for (const nextItem of nexts) {
|
|
261
|
-
nextParamsList.push([
|
|
262
|
-
nextItem.url && nextItem._url[i] || pageUrl,
|
|
263
|
-
contentType,
|
|
264
|
-
engine,
|
|
265
|
-
nextEl,
|
|
266
|
-
newData,
|
|
267
|
-
nextItem,
|
|
268
|
-
nextElIndex,
|
|
269
|
-
rootElement
|
|
270
|
-
]);
|
|
271
|
-
i++;
|
|
272
|
-
}
|
|
273
|
-
} else {
|
|
274
|
-
this.emit("newData", newData);
|
|
275
|
-
}
|
|
276
|
-
nextElIndex++;
|
|
277
|
-
}
|
|
278
|
-
for (const nextParams of nextParamsList) {
|
|
279
|
-
await this.next(...nextParams);
|
|
280
|
-
if (next.nextPageUrl) {
|
|
281
|
-
const nextPageUrl = await next.nextPageUrl({
|
|
282
|
-
data: nextParams[4],
|
|
283
|
-
el: nextParams[3],
|
|
284
|
-
url: nextParams[0]
|
|
285
|
-
});
|
|
286
|
-
if (nextPageUrl) {
|
|
287
|
-
const nextPageAbsoluteUrl = this.normalizeUrl(
|
|
288
|
-
nextParams[0],
|
|
289
|
-
nextPageUrl
|
|
290
|
-
);
|
|
291
|
-
if (nextPageAbsoluteUrl !== nextParams[0]) {
|
|
292
|
-
const nextPageElement = await this.fetchUrlAsHTMLElement(
|
|
293
|
-
nextPageAbsoluteUrl,
|
|
294
|
-
nextPageAbsoluteUrl,
|
|
295
|
-
engine,
|
|
296
|
-
this.config.engineOptions
|
|
297
|
-
);
|
|
298
|
-
await this.next(
|
|
299
|
-
nextPageAbsoluteUrl,
|
|
300
|
-
contentType,
|
|
301
|
-
engine,
|
|
302
|
-
nextPageElement,
|
|
303
|
-
nextParams[4],
|
|
304
|
-
(0, import_vtils.omitStrict)(next, ["url"]),
|
|
305
|
-
index,
|
|
306
|
-
rootElement
|
|
307
|
-
);
|
|
308
|
-
}
|
|
309
|
-
}
|
|
310
|
-
}
|
|
311
|
-
}
|
|
312
|
-
}
|
|
313
|
-
}
|
|
314
|
-
async start() {
|
|
315
|
-
const initialContentType = this.config.contentType || "html";
|
|
316
|
-
const initialEngine = this.config.engine || "request";
|
|
317
|
-
const rootElement = initialContentType === "json" ? await this.fetchUrlAsJSON(
|
|
318
|
-
this.config.entry,
|
|
319
|
-
this.config.entry,
|
|
320
|
-
initialEngine
|
|
321
|
-
) : await this.fetchUrlAsHTMLElement(
|
|
322
|
-
this.config.entry,
|
|
323
|
-
this.config.entry,
|
|
324
|
-
initialEngine,
|
|
325
|
-
this.config.engineOptions
|
|
326
|
-
);
|
|
327
|
-
await this.next(
|
|
328
|
-
this.config.entry,
|
|
329
|
-
initialContentType,
|
|
330
|
-
initialEngine,
|
|
331
|
-
rootElement,
|
|
332
|
-
{},
|
|
333
|
-
{
|
|
334
|
-
extract: this.config.extract,
|
|
335
|
-
next: this.config.next
|
|
336
|
-
},
|
|
337
|
-
0,
|
|
338
|
-
rootElement
|
|
339
|
-
);
|
|
340
|
-
}
|
|
341
|
-
}
|
package/_cjs/csv.js
DELETED
|
@@ -1,97 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __create = Object.create;
|
|
3
|
-
var __defProp = Object.defineProperty;
|
|
4
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
-
var __export = (target, all) => {
|
|
9
|
-
for (var name in all)
|
|
10
|
-
__defProp(target, name, { get: all[name], enumerable: true });
|
|
11
|
-
};
|
|
12
|
-
var __copyProps = (to, from, except, desc) => {
|
|
13
|
-
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
|
-
for (let key of __getOwnPropNames(from))
|
|
15
|
-
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
16
|
-
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
17
|
-
}
|
|
18
|
-
return to;
|
|
19
|
-
};
|
|
20
|
-
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
-
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
-
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
-
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
-
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
-
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
-
mod
|
|
27
|
-
));
|
|
28
|
-
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
|
-
var csv_exports = {};
|
|
30
|
-
__export(csv_exports, {
|
|
31
|
-
XCsv: () => XCsv
|
|
32
|
-
});
|
|
33
|
-
module.exports = __toCommonJS(csv_exports);
|
|
34
|
-
var csv = __toESM(require("csv"), 1);
|
|
35
|
-
var fs = __toESM(require("node:fs"), 1);
|
|
36
|
-
var import_vtils = require("vtils");
|
|
37
|
-
var import_date = require("vtils/date");
|
|
38
|
-
class XCsv {
|
|
39
|
-
static async parse(value, options = {}) {
|
|
40
|
-
const res = [];
|
|
41
|
-
for await (const item of csv.parse(value, {
|
|
42
|
-
columns: true
|
|
43
|
-
})) {
|
|
44
|
-
res.push(
|
|
45
|
-
options.objectFieldNames?.length || options.booleanFieldNames?.length || options.dateFieldNames?.length || options.arrayFieldNames?.length && options.arrayDelimiter ? (0, import_vtils.mapValues)(item, (v, k) => {
|
|
46
|
-
if (options.arrayFieldNames.includes(k)) {
|
|
47
|
-
return v.split(options.arrayDelimiter).filter(Boolean);
|
|
48
|
-
}
|
|
49
|
-
if (options.objectFieldNames?.includes(k)) {
|
|
50
|
-
return v ? JSON.parse(v) : {};
|
|
51
|
-
}
|
|
52
|
-
if (options.booleanFieldNames?.includes(k)) {
|
|
53
|
-
return v ? v === 1 : void 0;
|
|
54
|
-
}
|
|
55
|
-
if (options.dateFieldNames?.includes(k)) {
|
|
56
|
-
return v ? new Date(v) : void 0;
|
|
57
|
-
}
|
|
58
|
-
return v;
|
|
59
|
-
}) : item
|
|
60
|
-
);
|
|
61
|
-
}
|
|
62
|
-
return res;
|
|
63
|
-
}
|
|
64
|
-
static async stringify(value, options = {}) {
|
|
65
|
-
let res = "";
|
|
66
|
-
for await (const line of csv.stringify(
|
|
67
|
-
options.objectFieldNames?.length || options.booleanFieldNames?.length || options.dateFieldNames?.length || options.arrayFieldNames?.length && options.arrayDelimiter ? value.map(
|
|
68
|
-
(item) => (0, import_vtils.mapValues)(item, (v, k) => {
|
|
69
|
-
if (options.arrayFieldNames.includes(k)) {
|
|
70
|
-
return v ? v.join(options.arrayDelimiter) : "";
|
|
71
|
-
}
|
|
72
|
-
if (options.objectFieldNames?.includes(k)) {
|
|
73
|
-
return JSON.stringify(v || {});
|
|
74
|
-
}
|
|
75
|
-
if (options.booleanFieldNames?.includes(k)) {
|
|
76
|
-
return v ? "1" : "0";
|
|
77
|
-
}
|
|
78
|
-
if (options.dateFieldNames?.includes(k)) {
|
|
79
|
-
return v ? (0, import_date.formatDate)(v, "yyyy-mm-dd hh:ii:ss") : v;
|
|
80
|
-
}
|
|
81
|
-
return v;
|
|
82
|
-
})
|
|
83
|
-
) : value,
|
|
84
|
-
{ header: true }
|
|
85
|
-
)) {
|
|
86
|
-
res += line;
|
|
87
|
-
}
|
|
88
|
-
return res;
|
|
89
|
-
}
|
|
90
|
-
static async read(file, options = {}) {
|
|
91
|
-
const value = await fs.promises.readFile(file, "utf8");
|
|
92
|
-
return XCsv.parse(value, options);
|
|
93
|
-
}
|
|
94
|
-
static async write(file, value, options = {}) {
|
|
95
|
-
await fs.promises.writeFile(file, await XCsv.stringify(value, options));
|
|
96
|
-
}
|
|
97
|
-
}
|
package/_cjs/data_set.js
DELETED
|
@@ -1,115 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __create = Object.create;
|
|
3
|
-
var __defProp = Object.defineProperty;
|
|
4
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
-
var __export = (target, all) => {
|
|
9
|
-
for (var name in all)
|
|
10
|
-
__defProp(target, name, { get: all[name], enumerable: true });
|
|
11
|
-
};
|
|
12
|
-
var __copyProps = (to, from, except, desc) => {
|
|
13
|
-
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
|
-
for (let key of __getOwnPropNames(from))
|
|
15
|
-
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
16
|
-
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
17
|
-
}
|
|
18
|
-
return to;
|
|
19
|
-
};
|
|
20
|
-
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
-
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
-
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
-
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
-
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
-
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
-
mod
|
|
27
|
-
));
|
|
28
|
-
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
|
-
var data_set_exports = {};
|
|
30
|
-
__export(data_set_exports, {
|
|
31
|
-
XDataSet: () => XDataSet
|
|
32
|
-
});
|
|
33
|
-
module.exports = __toCommonJS(data_set_exports);
|
|
34
|
-
var import_exit_hook = require("exit-hook");
|
|
35
|
-
var import_fs_extra = __toESM(require("fs-extra"), 1);
|
|
36
|
-
var path = __toESM(require("node:path"), 1);
|
|
37
|
-
var import_vtils = require("vtils");
|
|
38
|
-
var import_vae = require("vtils/vae");
|
|
39
|
-
var import_csv = require("./csv.js");
|
|
40
|
-
var import_json = require("./json.js");
|
|
41
|
-
var import_yaml = require("./yaml.js");
|
|
42
|
-
class XDataSet {
|
|
43
|
-
constructor(options) {
|
|
44
|
-
this.options = options;
|
|
45
|
-
this.storeType = options.storeFile.endsWith(".csv") ? "csv" : options.storeFile.endsWith(".yaml") || options.storeFile.endsWith(".yml") ? "yaml" : "json";
|
|
46
|
-
this.dataShapeSchema = options.dataSchema(import_vae.v);
|
|
47
|
-
this.dataSchema = import_vae.v.object(this.dataShapeSchema);
|
|
48
|
-
this.csvOptions = {
|
|
49
|
-
arrayDelimiter: options.csvArrayDelimiter,
|
|
50
|
-
arrayFieldNames: (0, import_vtils.keysStrict)(this.dataShapeSchema).filter(
|
|
51
|
-
(key) => this.dataShapeSchema[key].options.type === "array"
|
|
52
|
-
),
|
|
53
|
-
objectFieldNames: (0, import_vtils.keysStrict)(this.dataShapeSchema).filter(
|
|
54
|
-
(key) => this.dataShapeSchema[key].options.type === "object"
|
|
55
|
-
),
|
|
56
|
-
booleanFieldNames: (0, import_vtils.keysStrict)(this.dataShapeSchema).filter(
|
|
57
|
-
(key) => this.dataShapeSchema[key].options.type === "boolean"
|
|
58
|
-
),
|
|
59
|
-
dateFieldNames: (0, import_vtils.keysStrict)(this.dataShapeSchema).filter(
|
|
60
|
-
(key) => this.dataShapeSchema[key].options.type === "date"
|
|
61
|
-
)
|
|
62
|
-
};
|
|
63
|
-
if (!options.noAutoWrite) {
|
|
64
|
-
const timer = setInterval(() => {
|
|
65
|
-
this.write();
|
|
66
|
-
}, options.autoWriteInterval || 5e3);
|
|
67
|
-
(0, import_exit_hook.asyncExitHook)(
|
|
68
|
-
async () => {
|
|
69
|
-
clearInterval(timer);
|
|
70
|
-
await this.write();
|
|
71
|
-
},
|
|
72
|
-
{
|
|
73
|
-
wait: 1e4
|
|
74
|
-
}
|
|
75
|
-
);
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
storeType;
|
|
79
|
-
dataShapeSchema;
|
|
80
|
-
dataSchema;
|
|
81
|
-
csvOptions;
|
|
82
|
-
currentMap;
|
|
83
|
-
get current() {
|
|
84
|
-
const orderedDataFieldNames = (0, import_vtils.keysStrict)(this.dataShapeSchema);
|
|
85
|
-
const dataSet = (0, import_vtils.keysStrict)(this.currentMap).map((dataKeyValue) => {
|
|
86
|
-
const data = {
|
|
87
|
-
...this.currentMap[dataKeyValue],
|
|
88
|
-
[this.options.dataKey]: dataKeyValue
|
|
89
|
-
};
|
|
90
|
-
return this.dataSchema.cast(
|
|
91
|
-
(0, import_vtils.pickStrict)(data, orderedDataFieldNames)
|
|
92
|
-
);
|
|
93
|
-
});
|
|
94
|
-
return dataSet;
|
|
95
|
-
}
|
|
96
|
-
async read() {
|
|
97
|
-
let dataSet = [];
|
|
98
|
-
try {
|
|
99
|
-
dataSet = await (this.storeType === "csv" ? import_csv.XCsv.read(this.options.storeFile, this.csvOptions) : this.storeType === "yaml" ? import_yaml.XYaml.read(this.options.storeFile) : import_json.XJson.read(this.options.storeFile));
|
|
100
|
-
} catch {
|
|
101
|
-
}
|
|
102
|
-
dataSet = dataSet.map((data) => this.dataSchema.cast(data));
|
|
103
|
-
this.currentMap = (0, import_vtils.keyBy)(
|
|
104
|
-
dataSet,
|
|
105
|
-
(data) => data[this.options.dataKey]
|
|
106
|
-
);
|
|
107
|
-
return dataSet;
|
|
108
|
-
}
|
|
109
|
-
async write(dataSet = this.current) {
|
|
110
|
-
console.log(`\u{1F4A5} \u5199\u5165 ${this.options.storeFile}`);
|
|
111
|
-
await import_fs_extra.default.ensureDir(path.dirname(this.options.storeFile));
|
|
112
|
-
await (this.storeType === "csv" ? import_csv.XCsv.write(this.options.storeFile, dataSet, this.csvOptions) : this.storeType === "yaml" ? import_yaml.XYaml.write(this.options.storeFile, dataSet) : import_json.XJson.write(this.options.storeFile, dataSet));
|
|
113
|
-
console.log(`\u2705 \u5199\u5165 ${this.options.storeFile}`);
|
|
114
|
-
}
|
|
115
|
-
}
|
package/_cjs/env.js
DELETED
|
@@ -1,82 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __create = Object.create;
|
|
3
|
-
var __defProp = Object.defineProperty;
|
|
4
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
-
var __export = (target, all) => {
|
|
9
|
-
for (var name in all)
|
|
10
|
-
__defProp(target, name, { get: all[name], enumerable: true });
|
|
11
|
-
};
|
|
12
|
-
var __copyProps = (to, from, except, desc) => {
|
|
13
|
-
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
|
-
for (let key of __getOwnPropNames(from))
|
|
15
|
-
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
16
|
-
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
17
|
-
}
|
|
18
|
-
return to;
|
|
19
|
-
};
|
|
20
|
-
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
-
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
-
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
-
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
-
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
-
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
-
mod
|
|
27
|
-
));
|
|
28
|
-
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
|
-
var env_exports = {};
|
|
30
|
-
__export(env_exports, {
|
|
31
|
-
XEnv: () => XEnv
|
|
32
|
-
});
|
|
33
|
-
module.exports = __toCommonJS(env_exports);
|
|
34
|
-
var import_dotenv = __toESM(require("dotenv"), 1);
|
|
35
|
-
var import_vtils = require("vtils");
|
|
36
|
-
var import_fs = require("./fs.js");
|
|
37
|
-
var import_path = require("./path.js");
|
|
38
|
-
var import_yaml = require("./yaml.js");
|
|
39
|
-
class XEnv {
|
|
40
|
-
static getEnvVarsFromFile = (0, import_vtils.memoize)(
|
|
41
|
-
async function get(mode = "", local = false) {
|
|
42
|
-
const envVars = mode ? await XEnv.getEnvVarsFromFile("") : {};
|
|
43
|
-
const baseName = local ? `.env${mode ? `.${mode}` : ""}.local` : `.env${mode ? `.${mode}` : ""}`;
|
|
44
|
-
const envYmlFile = import_path.XPath.join(process.cwd(), `${baseName}.yml`);
|
|
45
|
-
const envYamlFile = import_path.XPath.join(process.cwd(), `${baseName}.yaml`);
|
|
46
|
-
const dotenvFile = import_path.XPath.join(process.cwd(), baseName);
|
|
47
|
-
if (await import_fs.XFs.pathExists(envYmlFile)) {
|
|
48
|
-
Object.assign(envVars, await import_yaml.XYaml.read(envYmlFile));
|
|
49
|
-
} else if (await import_fs.XFs.pathExists(envYamlFile)) {
|
|
50
|
-
Object.assign(envVars, await import_yaml.XYaml.read(envYamlFile));
|
|
51
|
-
} else if (await import_fs.XFs.pathExists(dotenvFile)) {
|
|
52
|
-
const dotenvContent = await import_fs.XFs.readFile(dotenvFile, "utf-8");
|
|
53
|
-
Object.assign(envVars, await import_dotenv.default.parse(dotenvContent));
|
|
54
|
-
}
|
|
55
|
-
if (!local) {
|
|
56
|
-
const localEnvVars = await XEnv.getEnvVarsFromFile(mode, true);
|
|
57
|
-
Object.assign(envVars, localEnvVars);
|
|
58
|
-
}
|
|
59
|
-
return envVars;
|
|
60
|
-
},
|
|
61
|
-
(mode, local) => `${mode}_${local}`
|
|
62
|
-
);
|
|
63
|
-
static async getEnvVar(name, mode = process.env.X_ENV_MODE || "") {
|
|
64
|
-
if (process.env[name] != null) {
|
|
65
|
-
return process.env[name];
|
|
66
|
-
}
|
|
67
|
-
const envVars = await XEnv.getEnvVarsFromFile(mode);
|
|
68
|
-
return (0, import_vtils.isPlainObject)(envVars) ? envVars[name] ?? void 0 : void 0;
|
|
69
|
-
}
|
|
70
|
-
static async getEnvVars(mode = process.env.X_ENV_MODE || "") {
|
|
71
|
-
const envVars = await XEnv.getEnvVarsFromFile(mode);
|
|
72
|
-
return (0, import_vtils.isPlainObject)(envVars) ? envVars : {};
|
|
73
|
-
}
|
|
74
|
-
static async extendProcessEnv(mode = process.env.X_ENV_MODE || "") {
|
|
75
|
-
const envVars = await XEnv.getEnvVarsFromFile(mode);
|
|
76
|
-
if ((0, import_vtils.isPlainObject)(envVars)) {
|
|
77
|
-
Object.keys(envVars).forEach((env) => {
|
|
78
|
-
process.env[env] = envVars[env];
|
|
79
|
-
});
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
}
|