@jayfong/x-utils 1.0.0-202510061015 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/ai.d.mts +103 -0
- package/lib/ai.mjs +75 -0
- package/lib/cache.d.mts +464 -0
- package/lib/cache.mjs +531 -0
- package/lib/captcha.d.mts +259 -0
- package/lib/captcha.mjs +174 -0
- package/lib/case.d.mts +184 -0
- package/lib/case.mjs +120 -0
- package/lib/chinese.d.mts +57 -0
- package/lib/chinese.mjs +79 -0
- package/lib/cli.d.mts +52 -0
- package/lib/cli.mjs +62 -0
- package/lib/clipboard.d.mts +50 -0
- package/lib/clipboard.mjs +122 -0
- package/lib/cmd.d.mts +38 -0
- package/lib/cmd.mjs +52 -0
- package/lib/crawler.d.mts +330 -0
- package/lib/crawler.mjs +313 -0
- package/lib/crypto.d.mts +158 -0
- package/lib/crypto.mjs +213 -0
- package/lib/csv.d.mts +67 -0
- package/lib/csv.mjs +76 -0
- package/lib/data_set.d.mts +76 -0
- package/lib/data_set.mjs +104 -0
- package/lib/db_value.d.mts +213 -0
- package/lib/db_value.mjs +208 -0
- package/lib/dispose.d.mts +98 -0
- package/lib/dispose.mjs +107 -0
- package/lib/file_store.d.mts +167 -0
- package/lib/file_store.mjs +120 -0
- package/lib/fs.d.mts +111 -0
- package/lib/fs.mjs +127 -0
- package/lib/html.d.mts +103 -0
- package/lib/html.mjs +112 -0
- package/lib/json.d.mts +50 -0
- package/lib/json.mjs +74 -0
- package/lib/jwt.d.mts +128 -0
- package/lib/jwt.mjs +104 -0
- package/lib/mail.d.mts +125 -0
- package/lib/mail.mjs +108 -0
- package/lib/pay.d.mts +394 -0
- package/lib/pay.mjs +343 -0
- package/lib/puppeteer.d.mts +88 -0
- package/lib/puppeteer.mjs +219 -0
- package/lib/random.d.mts +71 -0
- package/lib/random.mjs +92 -0
- package/lib/redis.d.mts +32 -0
- package/lib/redis.mjs +24 -0
- package/lib/runner.d.mts +80 -0
- package/lib/runner.mjs +66 -0
- package/lib/sensitive_words.d.mts +82 -0
- package/lib/sensitive_words.mjs +73 -0
- package/lib/sms.d.mts +133 -0
- package/lib/sms.mjs +100 -0
- package/lib/task.d.mts +37 -0
- package/lib/task.mjs +43 -0
- package/lib/text.d.mts +84 -0
- package/lib/text.mjs +141 -0
- package/lib/yaml.d.mts +51 -0
- package/lib/yaml.mjs +64 -0
- package/package.json +82 -164
- package/_cjs/ai.js +0 -80
- package/_cjs/alipan.js +0 -173
- package/_cjs/cli.js +0 -51
- package/_cjs/clipboard.js +0 -65
- package/_cjs/cmd.js +0 -42
- package/_cjs/compressor.js +0 -47
- package/_cjs/crawler.js +0 -341
- package/_cjs/csv.js +0 -97
- package/_cjs/data_set.js +0 -115
- package/_cjs/env.js +0 -82
- package/_cjs/fs.js +0 -86
- package/_cjs/html.js +0 -62
- package/_cjs/http_server.js +0 -79
- package/_cjs/json.js +0 -49
- package/_cjs/ksb.js +0 -218
- package/_cjs/number.js +0 -39
- package/_cjs/ocr.js +0 -74
- package/_cjs/package.json +0 -3
- package/_cjs/path.js +0 -35
- package/_cjs/pinyin.js +0 -37
- package/_cjs/puppeteer.js +0 -271
- package/_cjs/runner.js +0 -88
- package/_cjs/task.js +0 -41
- package/_cjs/text.js +0 -136
- package/_cjs/unicode/lib_unicode.js +0 -58643
- package/_cjs/unicode/unicode.js +0 -441
- package/_cjs/upload.js +0 -101
- package/_cjs/yaml.js +0 -52
- package/ai.d.ts +0 -26
- package/ai.js +0 -60
- package/alipan.d.ts +0 -209
- package/alipan.js +0 -153
- package/cli.d.ts +0 -14
- package/cli.js +0 -21
- package/clipboard.d.ts +0 -6
- package/clipboard.js +0 -35
- package/cmd.d.ts +0 -5
- package/cmd.js +0 -22
- package/compressor.d.ts +0 -5
- package/compressor.js +0 -17
- package/crawler.d.ts +0 -162
- package/crawler.js +0 -332
- package/csv.d.ts +0 -14
- package/csv.js +0 -67
- package/data_set.d.ts +0 -22
- package/data_set.js +0 -85
- package/env.d.ts +0 -6
- package/env.js +0 -52
- package/fs.d.ts +0 -23
- package/fs.js +0 -56
- package/html.d.ts +0 -7
- package/html.js +0 -45
- package/http_server.d.ts +0 -13
- package/http_server.js +0 -49
- package/json.d.ts +0 -6
- package/json.js +0 -19
- package/ksb.d.ts +0 -37
- package/ksb.js +0 -198
- package/number.d.ts +0 -3
- package/number.js +0 -9
- package/ocr.d.ts +0 -17
- package/ocr.js +0 -54
- package/path.d.ts +0 -2
- package/path.js +0 -5
- package/pinyin.d.ts +0 -8
- package/pinyin.js +0 -17
- package/puppeteer.d.ts +0 -38
- package/puppeteer.js +0 -241
- package/runner.d.ts +0 -13
- package/runner.js +0 -58
- package/task.d.ts +0 -6
- package/task.js +0 -21
- package/text.d.ts +0 -63
- package/text.js +0 -119
- package/unicode/lib_unicode.d.ts +0 -4
- package/unicode/lib_unicode.js +0 -58623
- package/unicode/unicode.d.ts +0 -347
- package/unicode/unicode.js +0 -426
- package/upload.d.ts +0 -34
- package/upload.js +0 -71
- package/yaml.d.ts +0 -6
- package/yaml.js +0 -22
package/crawler.js
DELETED
|
@@ -1,332 +0,0 @@
|
|
|
1
|
-
import { xr } from "@jayfong/x-request";
|
|
2
|
-
import { parse as parseHTML } from "node-html-parser";
|
|
3
|
-
import {
|
|
4
|
-
EventBus,
|
|
5
|
-
castArray,
|
|
6
|
-
cloneDeepFast,
|
|
7
|
-
escape,
|
|
8
|
-
get,
|
|
9
|
-
isRegExp,
|
|
10
|
-
isUrl,
|
|
11
|
-
keysStrict,
|
|
12
|
-
memoize,
|
|
13
|
-
omitStrict
|
|
14
|
-
} from "vtils";
|
|
15
|
-
import { v } from "vtils/vae";
|
|
16
|
-
import { XOcr } from "./ocr.js";
|
|
17
|
-
import { XPuppeteer } from "./puppeteer.js";
|
|
18
|
-
class XCrawler extends EventBus {
|
|
19
|
-
constructor(config) {
|
|
20
|
-
super();
|
|
21
|
-
this.config = config;
|
|
22
|
-
this.schemaMap = config.schema(v);
|
|
23
|
-
if (config.listen) {
|
|
24
|
-
this.on("newData", (newData) => {
|
|
25
|
-
config.listen(newData);
|
|
26
|
-
});
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
schemaMap;
|
|
30
|
-
id = Date.now().toString(36);
|
|
31
|
-
normalizeUrl(pageUrl, url) {
|
|
32
|
-
url = url.trim();
|
|
33
|
-
return !url ? "" : isUrl(url) ? url : new URL(
|
|
34
|
-
url.startsWith(".") || url.startsWith("/") ? url : `/${url}`,
|
|
35
|
-
pageUrl
|
|
36
|
-
).toString();
|
|
37
|
-
}
|
|
38
|
-
async fetchUrlHeadless(pageUrl, url, options) {
|
|
39
|
-
const text = await XPuppeteer.getPageHtml(this.normalizeUrl(pageUrl, url), {
|
|
40
|
-
headful: false,
|
|
41
|
-
proxy: this.config.proxy,
|
|
42
|
-
blockAssets: true,
|
|
43
|
-
instanceId: this.id,
|
|
44
|
-
...options
|
|
45
|
-
});
|
|
46
|
-
return text;
|
|
47
|
-
}
|
|
48
|
-
async fetchUrlHeadful(pageUrl, url, options) {
|
|
49
|
-
const text = await XPuppeteer.getPageHtml(this.normalizeUrl(pageUrl, url), {
|
|
50
|
-
headful: true,
|
|
51
|
-
proxy: this.config.proxy,
|
|
52
|
-
blockAssets: true,
|
|
53
|
-
instanceId: this.id,
|
|
54
|
-
...options
|
|
55
|
-
});
|
|
56
|
-
return text;
|
|
57
|
-
}
|
|
58
|
-
async fetchUrlRequest(pageUrl, url) {
|
|
59
|
-
const absoluteUrl = this.normalizeUrl(pageUrl, url);
|
|
60
|
-
const { data: text } = await xr.getText({
|
|
61
|
-
url: absoluteUrl,
|
|
62
|
-
autoReferer: true,
|
|
63
|
-
proxyUrl: this.config.proxy
|
|
64
|
-
});
|
|
65
|
-
return text;
|
|
66
|
-
}
|
|
67
|
-
async fetchUrl(pageUrl, url, engine, options) {
|
|
68
|
-
const text = engine === "headless" ? await this.fetchUrlHeadless(pageUrl, url, options) : engine === "headful" ? await this.fetchUrlHeadful(pageUrl, url, options) : await this.fetchUrlRequest(pageUrl, url);
|
|
69
|
-
return text;
|
|
70
|
-
}
|
|
71
|
-
async fetchUrlAsHTMLElement(pageUrl, url, engine, options) {
|
|
72
|
-
const text = await this.fetchUrl(pageUrl, url, engine, options);
|
|
73
|
-
return parseHTML(text);
|
|
74
|
-
}
|
|
75
|
-
async fetchUrlAsJSON(pageUrl, url, engine) {
|
|
76
|
-
const text = await this.fetchUrl(pageUrl, url, engine);
|
|
77
|
-
return JSON.parse(text);
|
|
78
|
-
}
|
|
79
|
-
async ocr(pageUrl, el) {
|
|
80
|
-
const imgDocs = el.querySelectorAll("img");
|
|
81
|
-
for (const imgDoc of imgDocs) {
|
|
82
|
-
const imgUrl = imgDoc.getAttribute("src");
|
|
83
|
-
if (imgUrl) {
|
|
84
|
-
const ocrText = await XOcr.recognize(
|
|
85
|
-
this.normalizeUrl(pageUrl, imgUrl)
|
|
86
|
-
);
|
|
87
|
-
imgDoc.replaceWith(`<div>${escape(ocrText)} </div>`);
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
findNextById = memoize((id) => {
|
|
92
|
-
const findNextById = (nexts) => {
|
|
93
|
-
for (const next of castArray(nexts)) {
|
|
94
|
-
if (typeof next !== "string") {
|
|
95
|
-
if (next.id === id) {
|
|
96
|
-
return next;
|
|
97
|
-
}
|
|
98
|
-
if (next.next) {
|
|
99
|
-
const _next = findNextById(next.next);
|
|
100
|
-
if (_next) {
|
|
101
|
-
return _next;
|
|
102
|
-
}
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
};
|
|
107
|
-
return this.config.next && findNextById(this.config.next);
|
|
108
|
-
});
|
|
109
|
-
pageUrlCache = /* @__PURE__ */ new Set();
|
|
110
|
-
async next(pageUrl, contentType, engine, elOrField, data, next, index, rootElement) {
|
|
111
|
-
if (next.filter && !await next.filter(data)) {
|
|
112
|
-
return;
|
|
113
|
-
}
|
|
114
|
-
contentType = next.contentType || contentType;
|
|
115
|
-
engine = next.engine || engine;
|
|
116
|
-
if (contentType === "json") {
|
|
117
|
-
const field = elOrField;
|
|
118
|
-
const nextFields = next.url ? await Promise.all(
|
|
119
|
-
castArray(
|
|
120
|
-
typeof next.url === "function" ? await next.url(data) : data[next.url]
|
|
121
|
-
).map((url) => this.fetchUrlAsJSON(pageUrl, url, engine))
|
|
122
|
-
) : !next.field || next.field === "." ? [field] : typeof next.field === "string" ? get(field, next.field) : castArray(await next.field(field));
|
|
123
|
-
for (const nextField of nextFields) {
|
|
124
|
-
const newData = cloneDeepFast(data);
|
|
125
|
-
if (next.extract) {
|
|
126
|
-
const fieldKeys = keysStrict(next.extract);
|
|
127
|
-
for (const fieldKey of fieldKeys) {
|
|
128
|
-
const fieldConfig = typeof next.extract[fieldKey] === "string" || typeof next.extract[fieldKey] === "function" || Array.isArray(next.extract[fieldKey]) ? { select: next.extract[fieldKey] } : next.extract[fieldKey];
|
|
129
|
-
let fieldValue = !fieldConfig.select || fieldConfig.select === "." ? nextField : typeof fieldConfig.select === "string" ? get(nextField, fieldConfig.select) : Array.isArray(fieldConfig.select) ? "TODO" : "TODO";
|
|
130
|
-
if (fieldValue != null) {
|
|
131
|
-
fieldValue = fieldConfig.transform ? await fieldConfig.transform(fieldValue) : fieldValue;
|
|
132
|
-
const fieldSchema = this.schemaMap[fieldKey];
|
|
133
|
-
fieldValue = fieldSchema.cast(fieldValue);
|
|
134
|
-
newData[fieldKey] = fieldValue;
|
|
135
|
-
}
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
if (next.next) {
|
|
139
|
-
for (const nextItem of castArray(next.next)) {
|
|
140
|
-
await this.next(
|
|
141
|
-
pageUrl,
|
|
142
|
-
contentType,
|
|
143
|
-
engine,
|
|
144
|
-
nextField,
|
|
145
|
-
newData,
|
|
146
|
-
typeof nextItem === "string" ? this.findNextById(nextItem) : nextItem,
|
|
147
|
-
0,
|
|
148
|
-
rootElement
|
|
149
|
-
);
|
|
150
|
-
}
|
|
151
|
-
} else {
|
|
152
|
-
this.emit("newData", newData);
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
|
-
} else {
|
|
156
|
-
const el = elOrField;
|
|
157
|
-
const nextEls = next.url ? await Promise.all(
|
|
158
|
-
next._url.map(
|
|
159
|
-
(fullUrl) => this.fetchUrlAsHTMLElement(
|
|
160
|
-
pageUrl,
|
|
161
|
-
fullUrl,
|
|
162
|
-
engine,
|
|
163
|
-
next.engineOptions
|
|
164
|
-
)
|
|
165
|
-
)
|
|
166
|
-
) : !next.element || next.element === "." ? [el] : typeof next.element === "string" ? el.querySelectorAll(next.element) : castArray(
|
|
167
|
-
await next.element(el, {
|
|
168
|
-
index,
|
|
169
|
-
root: rootElement
|
|
170
|
-
})
|
|
171
|
-
);
|
|
172
|
-
const nextParamsList = [];
|
|
173
|
-
let nextElIndex = 0;
|
|
174
|
-
for (const nextEl of nextEls) {
|
|
175
|
-
const newData = cloneDeepFast(data);
|
|
176
|
-
if (next.extract) {
|
|
177
|
-
const fieldKeys = keysStrict(next.extract);
|
|
178
|
-
for (const fieldKey of fieldKeys) {
|
|
179
|
-
const fieldConfig = typeof next.extract[fieldKey] === "string" || typeof next.extract[fieldKey] === "function" || Array.isArray(next.extract[fieldKey]) ? { select: next.extract[fieldKey] } : next.extract[fieldKey];
|
|
180
|
-
const fieldEl = !fieldConfig.select || fieldConfig.select === "." ? nextEl : typeof fieldConfig.select === "string" ? nextEl.querySelector(fieldConfig.select) : Array.isArray(fieldConfig.select) ? nextEl.querySelectorAll(fieldConfig.select[0]).find((item) => {
|
|
181
|
-
const filter = fieldConfig.select[1];
|
|
182
|
-
return typeof filter === "string" ? item.text.includes(filter) : isRegExp(filter) ? filter.test(item.text) : filter(item);
|
|
183
|
-
}) : await fieldConfig.select(nextEl);
|
|
184
|
-
if (fieldEl != null) {
|
|
185
|
-
let fieldText = "";
|
|
186
|
-
if (typeof fieldEl === "string") {
|
|
187
|
-
fieldText = fieldEl.trim();
|
|
188
|
-
} else {
|
|
189
|
-
if (fieldConfig.ocr) {
|
|
190
|
-
await this.ocr(pageUrl, fieldEl);
|
|
191
|
-
}
|
|
192
|
-
switch (fieldConfig.extract) {
|
|
193
|
-
case "getHTML":
|
|
194
|
-
fieldText = fieldEl.outerHTML;
|
|
195
|
-
break;
|
|
196
|
-
case "getFirstImageUrl":
|
|
197
|
-
fieldText = this.normalizeUrl(
|
|
198
|
-
pageUrl,
|
|
199
|
-
fieldEl.getAttribute("src") || fieldEl.querySelector("img")?.getAttribute("src") || ""
|
|
200
|
-
);
|
|
201
|
-
break;
|
|
202
|
-
case "getFirstLinkUrl":
|
|
203
|
-
fieldText = this.normalizeUrl(
|
|
204
|
-
pageUrl,
|
|
205
|
-
fieldEl.getAttribute("href") || fieldEl.querySelector("a")?.getAttribute("href") || ""
|
|
206
|
-
);
|
|
207
|
-
break;
|
|
208
|
-
default:
|
|
209
|
-
fieldText = fieldEl.text.trim();
|
|
210
|
-
break;
|
|
211
|
-
}
|
|
212
|
-
}
|
|
213
|
-
let fieldValue = fieldConfig.transform ? await fieldConfig.transform(fieldText) : fieldText;
|
|
214
|
-
const fieldSchema = this.schemaMap[fieldKey];
|
|
215
|
-
if (fieldSchema.options.type !== "array") {
|
|
216
|
-
fieldValue = fieldSchema.cast(fieldValue);
|
|
217
|
-
} else {
|
|
218
|
-
fieldValue = fieldSchema.cast([
|
|
219
|
-
...newData[fieldKey] || [],
|
|
220
|
-
fieldValue
|
|
221
|
-
]);
|
|
222
|
-
}
|
|
223
|
-
newData[fieldKey] = fieldValue;
|
|
224
|
-
}
|
|
225
|
-
}
|
|
226
|
-
}
|
|
227
|
-
if (next.next) {
|
|
228
|
-
const nexts = await Promise.all(
|
|
229
|
-
castArray(next.next).map(async (nextItem) => {
|
|
230
|
-
nextItem = cloneDeepFast(
|
|
231
|
-
typeof nextItem === "string" ? this.findNextById(nextItem) : nextItem
|
|
232
|
-
);
|
|
233
|
-
if (nextItem.url) {
|
|
234
|
-
nextItem._url = castArray(
|
|
235
|
-
typeof nextItem.url === "function" ? await nextItem.url({
|
|
236
|
-
data: newData,
|
|
237
|
-
el: nextEl
|
|
238
|
-
}) : newData[nextItem.url]
|
|
239
|
-
).map((url) => this.normalizeUrl(pageUrl, url)).filter((fullUrl) => !this.pageUrlCache.has(fullUrl)).map((fullUrl) => {
|
|
240
|
-
this.pageUrlCache.add(fullUrl);
|
|
241
|
-
return fullUrl;
|
|
242
|
-
});
|
|
243
|
-
}
|
|
244
|
-
return nextItem;
|
|
245
|
-
})
|
|
246
|
-
);
|
|
247
|
-
let i = 0;
|
|
248
|
-
for (const nextItem of nexts) {
|
|
249
|
-
nextParamsList.push([
|
|
250
|
-
nextItem.url && nextItem._url[i] || pageUrl,
|
|
251
|
-
contentType,
|
|
252
|
-
engine,
|
|
253
|
-
nextEl,
|
|
254
|
-
newData,
|
|
255
|
-
nextItem,
|
|
256
|
-
nextElIndex,
|
|
257
|
-
rootElement
|
|
258
|
-
]);
|
|
259
|
-
i++;
|
|
260
|
-
}
|
|
261
|
-
} else {
|
|
262
|
-
this.emit("newData", newData);
|
|
263
|
-
}
|
|
264
|
-
nextElIndex++;
|
|
265
|
-
}
|
|
266
|
-
for (const nextParams of nextParamsList) {
|
|
267
|
-
await this.next(...nextParams);
|
|
268
|
-
if (next.nextPageUrl) {
|
|
269
|
-
const nextPageUrl = await next.nextPageUrl({
|
|
270
|
-
data: nextParams[4],
|
|
271
|
-
el: nextParams[3],
|
|
272
|
-
url: nextParams[0]
|
|
273
|
-
});
|
|
274
|
-
if (nextPageUrl) {
|
|
275
|
-
const nextPageAbsoluteUrl = this.normalizeUrl(
|
|
276
|
-
nextParams[0],
|
|
277
|
-
nextPageUrl
|
|
278
|
-
);
|
|
279
|
-
if (nextPageAbsoluteUrl !== nextParams[0]) {
|
|
280
|
-
const nextPageElement = await this.fetchUrlAsHTMLElement(
|
|
281
|
-
nextPageAbsoluteUrl,
|
|
282
|
-
nextPageAbsoluteUrl,
|
|
283
|
-
engine,
|
|
284
|
-
this.config.engineOptions
|
|
285
|
-
);
|
|
286
|
-
await this.next(
|
|
287
|
-
nextPageAbsoluteUrl,
|
|
288
|
-
contentType,
|
|
289
|
-
engine,
|
|
290
|
-
nextPageElement,
|
|
291
|
-
nextParams[4],
|
|
292
|
-
omitStrict(next, ["url"]),
|
|
293
|
-
index,
|
|
294
|
-
rootElement
|
|
295
|
-
);
|
|
296
|
-
}
|
|
297
|
-
}
|
|
298
|
-
}
|
|
299
|
-
}
|
|
300
|
-
}
|
|
301
|
-
}
|
|
302
|
-
async start() {
|
|
303
|
-
const initialContentType = this.config.contentType || "html";
|
|
304
|
-
const initialEngine = this.config.engine || "request";
|
|
305
|
-
const rootElement = initialContentType === "json" ? await this.fetchUrlAsJSON(
|
|
306
|
-
this.config.entry,
|
|
307
|
-
this.config.entry,
|
|
308
|
-
initialEngine
|
|
309
|
-
) : await this.fetchUrlAsHTMLElement(
|
|
310
|
-
this.config.entry,
|
|
311
|
-
this.config.entry,
|
|
312
|
-
initialEngine,
|
|
313
|
-
this.config.engineOptions
|
|
314
|
-
);
|
|
315
|
-
await this.next(
|
|
316
|
-
this.config.entry,
|
|
317
|
-
initialContentType,
|
|
318
|
-
initialEngine,
|
|
319
|
-
rootElement,
|
|
320
|
-
{},
|
|
321
|
-
{
|
|
322
|
-
extract: this.config.extract,
|
|
323
|
-
next: this.config.next
|
|
324
|
-
},
|
|
325
|
-
0,
|
|
326
|
-
rootElement
|
|
327
|
-
);
|
|
328
|
-
}
|
|
329
|
-
}
|
|
330
|
-
export {
|
|
331
|
-
XCrawler
|
|
332
|
-
};
|
package/csv.d.ts
DELETED
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
export type XCsvParseOptions = {
|
|
2
|
-
arrayFieldNames?: string[];
|
|
3
|
-
arrayDelimiter?: string;
|
|
4
|
-
objectFieldNames?: string[];
|
|
5
|
-
booleanFieldNames?: string[];
|
|
6
|
-
dateFieldNames?: string[];
|
|
7
|
-
};
|
|
8
|
-
export type XCsvStringifyOptions = XCsvParseOptions;
|
|
9
|
-
export declare class XCsv {
|
|
10
|
-
static parse<T>(value: string, options?: XCsvParseOptions): Promise<T[]>;
|
|
11
|
-
static stringify(value: any[], options?: XCsvStringifyOptions): Promise<string>;
|
|
12
|
-
static read<T>(file: string, options?: XCsvParseOptions): Promise<T[]>;
|
|
13
|
-
static write(file: string, value: any, options?: XCsvStringifyOptions): Promise<void>;
|
|
14
|
-
}
|
package/csv.js
DELETED
|
@@ -1,67 +0,0 @@
|
|
|
1
|
-
import * as csv from "csv";
|
|
2
|
-
import * as fs from "node:fs";
|
|
3
|
-
import { mapValues } from "vtils";
|
|
4
|
-
import { formatDate } from "vtils/date";
|
|
5
|
-
class XCsv {
|
|
6
|
-
static async parse(value, options = {}) {
|
|
7
|
-
const res = [];
|
|
8
|
-
for await (const item of csv.parse(value, {
|
|
9
|
-
columns: true
|
|
10
|
-
})) {
|
|
11
|
-
res.push(
|
|
12
|
-
options.objectFieldNames?.length || options.booleanFieldNames?.length || options.dateFieldNames?.length || options.arrayFieldNames?.length && options.arrayDelimiter ? mapValues(item, (v, k) => {
|
|
13
|
-
if (options.arrayFieldNames.includes(k)) {
|
|
14
|
-
return v.split(options.arrayDelimiter).filter(Boolean);
|
|
15
|
-
}
|
|
16
|
-
if (options.objectFieldNames?.includes(k)) {
|
|
17
|
-
return v ? JSON.parse(v) : {};
|
|
18
|
-
}
|
|
19
|
-
if (options.booleanFieldNames?.includes(k)) {
|
|
20
|
-
return v ? v === 1 : void 0;
|
|
21
|
-
}
|
|
22
|
-
if (options.dateFieldNames?.includes(k)) {
|
|
23
|
-
return v ? new Date(v) : void 0;
|
|
24
|
-
}
|
|
25
|
-
return v;
|
|
26
|
-
}) : item
|
|
27
|
-
);
|
|
28
|
-
}
|
|
29
|
-
return res;
|
|
30
|
-
}
|
|
31
|
-
static async stringify(value, options = {}) {
|
|
32
|
-
let res = "";
|
|
33
|
-
for await (const line of csv.stringify(
|
|
34
|
-
options.objectFieldNames?.length || options.booleanFieldNames?.length || options.dateFieldNames?.length || options.arrayFieldNames?.length && options.arrayDelimiter ? value.map(
|
|
35
|
-
(item) => mapValues(item, (v, k) => {
|
|
36
|
-
if (options.arrayFieldNames.includes(k)) {
|
|
37
|
-
return v ? v.join(options.arrayDelimiter) : "";
|
|
38
|
-
}
|
|
39
|
-
if (options.objectFieldNames?.includes(k)) {
|
|
40
|
-
return JSON.stringify(v || {});
|
|
41
|
-
}
|
|
42
|
-
if (options.booleanFieldNames?.includes(k)) {
|
|
43
|
-
return v ? "1" : "0";
|
|
44
|
-
}
|
|
45
|
-
if (options.dateFieldNames?.includes(k)) {
|
|
46
|
-
return v ? formatDate(v, "yyyy-mm-dd hh:ii:ss") : v;
|
|
47
|
-
}
|
|
48
|
-
return v;
|
|
49
|
-
})
|
|
50
|
-
) : value,
|
|
51
|
-
{ header: true }
|
|
52
|
-
)) {
|
|
53
|
-
res += line;
|
|
54
|
-
}
|
|
55
|
-
return res;
|
|
56
|
-
}
|
|
57
|
-
static async read(file, options = {}) {
|
|
58
|
-
const value = await fs.promises.readFile(file, "utf8");
|
|
59
|
-
return XCsv.parse(value, options);
|
|
60
|
-
}
|
|
61
|
-
static async write(file, value, options = {}) {
|
|
62
|
-
await fs.promises.writeFile(file, await XCsv.stringify(value, options));
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
export {
|
|
66
|
-
XCsv
|
|
67
|
-
};
|
package/data_set.d.ts
DELETED
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
import { OmitStrict } from 'vtils/types';
|
|
2
|
-
import { VaeObjectSchemaShapeOf, v } from 'vtils/vae';
|
|
3
|
-
export type XDataSetOptions<T extends Record<string, any>, K extends keyof T> = {
|
|
4
|
-
storeFile: string;
|
|
5
|
-
dataSchema: (_: typeof v) => VaeObjectSchemaShapeOf<T>;
|
|
6
|
-
dataKey: K;
|
|
7
|
-
csvArrayDelimiter?: string;
|
|
8
|
-
autoWriteInterval?: number;
|
|
9
|
-
noAutoWrite?: boolean;
|
|
10
|
-
};
|
|
11
|
-
export declare class XDataSet<T extends Record<string, any>, K extends keyof T> {
|
|
12
|
-
private options;
|
|
13
|
-
private storeType;
|
|
14
|
-
private dataShapeSchema;
|
|
15
|
-
private dataSchema;
|
|
16
|
-
private csvOptions;
|
|
17
|
-
currentMap: Record<T[K], OmitStrict<T, K>>;
|
|
18
|
-
get current(): T[];
|
|
19
|
-
constructor(options: XDataSetOptions<T, K>);
|
|
20
|
-
read(): Promise<T[]>;
|
|
21
|
-
write(dataSet?: T[]): Promise<void>;
|
|
22
|
-
}
|
package/data_set.js
DELETED
|
@@ -1,85 +0,0 @@
|
|
|
1
|
-
import { asyncExitHook } from "exit-hook";
|
|
2
|
-
import fs from "fs-extra";
|
|
3
|
-
import * as path from "node:path";
|
|
4
|
-
import { keyBy, keysStrict, pickStrict } from "vtils";
|
|
5
|
-
import { v } from "vtils/vae";
|
|
6
|
-
import { XCsv } from "./csv.js";
|
|
7
|
-
import { XJson } from "./json.js";
|
|
8
|
-
import { XYaml } from "./yaml.js";
|
|
9
|
-
class XDataSet {
|
|
10
|
-
constructor(options) {
|
|
11
|
-
this.options = options;
|
|
12
|
-
this.storeType = options.storeFile.endsWith(".csv") ? "csv" : options.storeFile.endsWith(".yaml") || options.storeFile.endsWith(".yml") ? "yaml" : "json";
|
|
13
|
-
this.dataShapeSchema = options.dataSchema(v);
|
|
14
|
-
this.dataSchema = v.object(this.dataShapeSchema);
|
|
15
|
-
this.csvOptions = {
|
|
16
|
-
arrayDelimiter: options.csvArrayDelimiter,
|
|
17
|
-
arrayFieldNames: keysStrict(this.dataShapeSchema).filter(
|
|
18
|
-
(key) => this.dataShapeSchema[key].options.type === "array"
|
|
19
|
-
),
|
|
20
|
-
objectFieldNames: keysStrict(this.dataShapeSchema).filter(
|
|
21
|
-
(key) => this.dataShapeSchema[key].options.type === "object"
|
|
22
|
-
),
|
|
23
|
-
booleanFieldNames: keysStrict(this.dataShapeSchema).filter(
|
|
24
|
-
(key) => this.dataShapeSchema[key].options.type === "boolean"
|
|
25
|
-
),
|
|
26
|
-
dateFieldNames: keysStrict(this.dataShapeSchema).filter(
|
|
27
|
-
(key) => this.dataShapeSchema[key].options.type === "date"
|
|
28
|
-
)
|
|
29
|
-
};
|
|
30
|
-
if (!options.noAutoWrite) {
|
|
31
|
-
const timer = setInterval(() => {
|
|
32
|
-
this.write();
|
|
33
|
-
}, options.autoWriteInterval || 5e3);
|
|
34
|
-
asyncExitHook(
|
|
35
|
-
async () => {
|
|
36
|
-
clearInterval(timer);
|
|
37
|
-
await this.write();
|
|
38
|
-
},
|
|
39
|
-
{
|
|
40
|
-
wait: 1e4
|
|
41
|
-
}
|
|
42
|
-
);
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
storeType;
|
|
46
|
-
dataShapeSchema;
|
|
47
|
-
dataSchema;
|
|
48
|
-
csvOptions;
|
|
49
|
-
currentMap;
|
|
50
|
-
get current() {
|
|
51
|
-
const orderedDataFieldNames = keysStrict(this.dataShapeSchema);
|
|
52
|
-
const dataSet = keysStrict(this.currentMap).map((dataKeyValue) => {
|
|
53
|
-
const data = {
|
|
54
|
-
...this.currentMap[dataKeyValue],
|
|
55
|
-
[this.options.dataKey]: dataKeyValue
|
|
56
|
-
};
|
|
57
|
-
return this.dataSchema.cast(
|
|
58
|
-
pickStrict(data, orderedDataFieldNames)
|
|
59
|
-
);
|
|
60
|
-
});
|
|
61
|
-
return dataSet;
|
|
62
|
-
}
|
|
63
|
-
async read() {
|
|
64
|
-
let dataSet = [];
|
|
65
|
-
try {
|
|
66
|
-
dataSet = await (this.storeType === "csv" ? XCsv.read(this.options.storeFile, this.csvOptions) : this.storeType === "yaml" ? XYaml.read(this.options.storeFile) : XJson.read(this.options.storeFile));
|
|
67
|
-
} catch {
|
|
68
|
-
}
|
|
69
|
-
dataSet = dataSet.map((data) => this.dataSchema.cast(data));
|
|
70
|
-
this.currentMap = keyBy(
|
|
71
|
-
dataSet,
|
|
72
|
-
(data) => data[this.options.dataKey]
|
|
73
|
-
);
|
|
74
|
-
return dataSet;
|
|
75
|
-
}
|
|
76
|
-
async write(dataSet = this.current) {
|
|
77
|
-
console.log(`\u{1F4A5} \u5199\u5165 ${this.options.storeFile}`);
|
|
78
|
-
await fs.ensureDir(path.dirname(this.options.storeFile));
|
|
79
|
-
await (this.storeType === "csv" ? XCsv.write(this.options.storeFile, dataSet, this.csvOptions) : this.storeType === "yaml" ? XYaml.write(this.options.storeFile, dataSet) : XJson.write(this.options.storeFile, dataSet));
|
|
80
|
-
console.log(`\u2705 \u5199\u5165 ${this.options.storeFile}`);
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
export {
|
|
84
|
-
XDataSet
|
|
85
|
-
};
|
package/env.d.ts
DELETED
package/env.js
DELETED
|
@@ -1,52 +0,0 @@
|
|
|
1
|
-
import dotenv from "dotenv";
|
|
2
|
-
import { isPlainObject, memoize } from "vtils";
|
|
3
|
-
import { XFs } from "./fs.js";
|
|
4
|
-
import { XPath } from "./path.js";
|
|
5
|
-
import { XYaml } from "./yaml.js";
|
|
6
|
-
class XEnv {
|
|
7
|
-
static getEnvVarsFromFile = memoize(
|
|
8
|
-
async function get(mode = "", local = false) {
|
|
9
|
-
const envVars = mode ? await XEnv.getEnvVarsFromFile("") : {};
|
|
10
|
-
const baseName = local ? `.env${mode ? `.${mode}` : ""}.local` : `.env${mode ? `.${mode}` : ""}`;
|
|
11
|
-
const envYmlFile = XPath.join(process.cwd(), `${baseName}.yml`);
|
|
12
|
-
const envYamlFile = XPath.join(process.cwd(), `${baseName}.yaml`);
|
|
13
|
-
const dotenvFile = XPath.join(process.cwd(), baseName);
|
|
14
|
-
if (await XFs.pathExists(envYmlFile)) {
|
|
15
|
-
Object.assign(envVars, await XYaml.read(envYmlFile));
|
|
16
|
-
} else if (await XFs.pathExists(envYamlFile)) {
|
|
17
|
-
Object.assign(envVars, await XYaml.read(envYamlFile));
|
|
18
|
-
} else if (await XFs.pathExists(dotenvFile)) {
|
|
19
|
-
const dotenvContent = await XFs.readFile(dotenvFile, "utf-8");
|
|
20
|
-
Object.assign(envVars, await dotenv.parse(dotenvContent));
|
|
21
|
-
}
|
|
22
|
-
if (!local) {
|
|
23
|
-
const localEnvVars = await XEnv.getEnvVarsFromFile(mode, true);
|
|
24
|
-
Object.assign(envVars, localEnvVars);
|
|
25
|
-
}
|
|
26
|
-
return envVars;
|
|
27
|
-
},
|
|
28
|
-
(mode, local) => `${mode}_${local}`
|
|
29
|
-
);
|
|
30
|
-
static async getEnvVar(name, mode = process.env.X_ENV_MODE || "") {
|
|
31
|
-
if (process.env[name] != null) {
|
|
32
|
-
return process.env[name];
|
|
33
|
-
}
|
|
34
|
-
const envVars = await XEnv.getEnvVarsFromFile(mode);
|
|
35
|
-
return isPlainObject(envVars) ? envVars[name] ?? void 0 : void 0;
|
|
36
|
-
}
|
|
37
|
-
static async getEnvVars(mode = process.env.X_ENV_MODE || "") {
|
|
38
|
-
const envVars = await XEnv.getEnvVarsFromFile(mode);
|
|
39
|
-
return isPlainObject(envVars) ? envVars : {};
|
|
40
|
-
}
|
|
41
|
-
static async extendProcessEnv(mode = process.env.X_ENV_MODE || "") {
|
|
42
|
-
const envVars = await XEnv.getEnvVarsFromFile(mode);
|
|
43
|
-
if (isPlainObject(envVars)) {
|
|
44
|
-
Object.keys(envVars).forEach((env) => {
|
|
45
|
-
process.env[env] = envVars[env];
|
|
46
|
-
});
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
export {
|
|
51
|
-
XEnv
|
|
52
|
-
};
|
package/fs.d.ts
DELETED
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
|
-
/// <reference types="node" />
|
|
3
|
-
import fs from 'fs-extra';
|
|
4
|
-
import { NeedleOptions } from 'needle';
|
|
5
|
-
import { Readable } from 'node:stream';
|
|
6
|
-
export type XFsExtraImageInfo = {
|
|
7
|
-
/** 图片宽度(单位: 像素) */
|
|
8
|
-
width: number;
|
|
9
|
-
/** 图片高度(单位: 像素) */
|
|
10
|
-
height: number;
|
|
11
|
-
/** 图片类型: jpg, png... */
|
|
12
|
-
type: string;
|
|
13
|
-
/** 图片 Mime 类型: image/jpg, image/png... */
|
|
14
|
-
mime: string;
|
|
15
|
-
};
|
|
16
|
-
export declare const XFs: typeof fs & {
|
|
17
|
-
getMimeType(extOrName: string): string;
|
|
18
|
-
getExtension(typeOrName: string): string;
|
|
19
|
-
toBuffer(file: Buffer | Readable): Promise<Buffer>;
|
|
20
|
-
toText(file: Buffer | Readable): Promise<string>;
|
|
21
|
-
toMaskImage(file: Buffer | Readable): Promise<Buffer>;
|
|
22
|
-
getImageInfo(urlOrStreamOrBuffer: string | Buffer | Readable, options?: NeedleOptions): Promise<XFsExtraImageInfo | null>;
|
|
23
|
-
};
|
package/fs.js
DELETED
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
import fs from "fs-extra";
|
|
2
|
-
import mime from "mime";
|
|
3
|
-
import probeImageSize from "probe-image-size";
|
|
4
|
-
const XFsExtra = {
|
|
5
|
-
getMimeType(extOrName) {
|
|
6
|
-
return mime.getType(extOrName) || "application/octet-stream";
|
|
7
|
-
},
|
|
8
|
-
getExtension(typeOrName) {
|
|
9
|
-
if (typeOrName.includes(".")) {
|
|
10
|
-
return typeOrName.split(".").pop();
|
|
11
|
-
}
|
|
12
|
-
return mime.getExtension(typeOrName) || "";
|
|
13
|
-
},
|
|
14
|
-
async toBuffer(file) {
|
|
15
|
-
if (Buffer.isBuffer(file)) {
|
|
16
|
-
return file;
|
|
17
|
-
}
|
|
18
|
-
const chunks = [];
|
|
19
|
-
for await (const chunk of file) {
|
|
20
|
-
chunks.push(chunk);
|
|
21
|
-
}
|
|
22
|
-
const buffer = Buffer.concat(chunks);
|
|
23
|
-
return buffer;
|
|
24
|
-
},
|
|
25
|
-
async toText(file) {
|
|
26
|
-
const buffer = Buffer.isBuffer(file) ? file : await XFsExtra.toBuffer(file);
|
|
27
|
-
return buffer.toString("utf-8");
|
|
28
|
-
},
|
|
29
|
-
async toMaskImage(file) {
|
|
30
|
-
const buffers = [
|
|
31
|
-
// PNG 文件头
|
|
32
|
-
// https://zh.wikipedia.org/wiki/PNG
|
|
33
|
-
Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]),
|
|
34
|
-
await XFsExtra.toBuffer(file)
|
|
35
|
-
];
|
|
36
|
-
return Buffer.concat(buffers);
|
|
37
|
-
},
|
|
38
|
-
async getImageInfo(urlOrStreamOrBuffer, options) {
|
|
39
|
-
return typeof urlOrStreamOrBuffer === "string" ? probeImageSize(urlOrStreamOrBuffer, {
|
|
40
|
-
rejectUnauthorized: false,
|
|
41
|
-
...options,
|
|
42
|
-
headers: {
|
|
43
|
-
"User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 MicroMessenger/7.0.17(0x17001126) NetType/WIFI Language/zh_CN",
|
|
44
|
-
Referer: urlOrStreamOrBuffer,
|
|
45
|
-
...options?.headers
|
|
46
|
-
}
|
|
47
|
-
}) : Buffer.isBuffer(urlOrStreamOrBuffer) ? probeImageSize.sync(urlOrStreamOrBuffer) : probeImageSize(urlOrStreamOrBuffer, true);
|
|
48
|
-
}
|
|
49
|
-
};
|
|
50
|
-
const XFs = {
|
|
51
|
-
...fs,
|
|
52
|
-
...XFsExtra
|
|
53
|
-
};
|
|
54
|
-
export {
|
|
55
|
-
XFs
|
|
56
|
-
};
|
package/html.d.ts
DELETED
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
|
-
import { ParagraphChild } from 'docx';
|
|
3
|
-
export { HTMLElement as XHtmlElement, NodeType as XHtmlNodeType, } from 'node-html-parser';
|
|
4
|
-
export declare class XHtml {
|
|
5
|
-
static parse(html: string): import("node-html-parser").HTMLElement;
|
|
6
|
-
static toDocxParagraphChildren(html: string, fetchImage: (url: string) => Promise<Buffer>): Promise<ParagraphChild[]>;
|
|
7
|
-
}
|