@linxiraos/pi-utils 1.1.27 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -6
- package/README.md +1 -1
- package/dist/types/ar/index.d.ts +1 -0
- package/dist/types/ar/zip-package.d.ts +21 -0
- package/dist/types/color.d.ts +2 -0
- package/dist/types/dirs.d.ts +31 -12
- package/dist/types/docx/converter.d.ts +2 -0
- package/dist/types/dom/core.d.ts +15 -4
- package/dist/types/dom/parser.d.ts +5 -3
- package/dist/types/dom/selector.d.ts +7 -1
- package/dist/types/index.d.ts +1 -0
- package/dist/types/logger/rotating-file.d.ts +4 -2
- package/dist/types/loop-phase.d.ts +12 -7
- package/dist/types/marked/core.d.ts +7 -6
- package/dist/types/marked-list.d.ts +38 -0
- package/dist/types/math-delimiters.d.ts +10 -0
- package/dist/types/readability/types.d.ts +2 -0
- package/dist/types/release-assets.d.ts +22 -0
- package/dist/types/temp.d.ts +7 -0
- package/dist/types/turndown/types.d.ts +7 -0
- package/dist/types/vterm/buffer.d.ts +16 -5
- package/package.json +3 -3
- package/src/ar/index.ts +1 -0
- package/src/ar/open.ts +29 -8
- package/src/ar/zip-package.ts +57 -0
- package/src/chalk.ts +66 -23
- package/src/color.ts +6 -0
- package/src/dirs.ts +84 -24
- package/src/docx/converter.ts +26 -12
- package/src/dom/core.ts +354 -111
- package/src/dom/parser.ts +42 -19
- package/src/dom/selector.ts +328 -95
- package/src/fetch-retry.ts +2 -0
- package/src/format.ts +5 -1
- package/src/index.ts +1 -0
- package/src/json-lexer.ts +54 -0
- package/src/json-parse.ts +21 -6
- package/src/logger/rotating-file.ts +5 -3
- package/src/logger.ts +39 -23
- package/src/loop-phase.ts +66 -21
- package/src/marked/core.ts +42 -18
- package/src/marked-list.ts +73 -0
- package/src/math-delimiters.ts +16 -0
- package/src/module-timer.ts +5 -5
- package/src/postmortem.ts +1 -1
- package/src/process-name.ts +2 -2
- package/src/prompt.ts +3 -1
- package/src/ptree.ts +28 -35
- package/src/readability/readability.ts +349 -39
- package/src/readability/types.ts +2 -0
- package/src/release-assets.ts +40 -0
- package/src/runtime-install.ts +2 -2
- package/src/stream.ts +34 -13
- package/src/temp.ts +7 -0
- package/src/template.ts +7 -4
- package/src/timing-buffer.ts +1 -1
- package/src/tls-fetch.ts +1 -1
- package/src/turndown/gfm.ts +16 -5
- package/src/turndown/html.ts +28 -8
- package/src/turndown/service.ts +43 -15
- package/src/turndown/types.ts +7 -0
- package/src/vterm/buffer.ts +31 -11
- package/src/vterm/terminal.ts +213 -100
package/src/ptree.ts
CHANGED
|
@@ -183,10 +183,12 @@ export class ChildProcess<In extends InMask = InMask> {
|
|
|
183
183
|
#stderrDone: Promise<void>;
|
|
184
184
|
#exited: Promise<number>;
|
|
185
185
|
#openPipeReaders = 1;
|
|
186
|
-
//
|
|
187
|
-
//
|
|
188
|
-
|
|
189
|
-
#
|
|
186
|
+
// Set by attachTimeout() at the command deadline; untimed commands keep
|
|
187
|
+
// complete EOF-based capture. Active pipe readers are cancelled at cutoff so
|
|
188
|
+
// their pending read() settles as done and partial output is kept.
|
|
189
|
+
#cutoff = false;
|
|
190
|
+
/** Readers of piped stdio; the timeout cancels them so pending reads settle. */
|
|
191
|
+
#pipeReaders = new Set<{ cancel(reason?: unknown): Promise<void> }>();
|
|
190
192
|
#timeoutTimer?: NodeJS.Timeout;
|
|
191
193
|
#stderrStream?: ReadableStream<Uint8Array>;
|
|
192
194
|
// Termination in flight after kill(); aborted exits await it before reporting.
|
|
@@ -224,31 +226,24 @@ export class ChildProcess<In extends InMask = InMask> {
|
|
|
224
226
|
// Normalize Bun's exited promise into our exitReason / exitedCleanly model.
|
|
225
227
|
const { promise, resolve, reject } = Promise.withResolvers<number>();
|
|
226
228
|
this.#exited = promise;
|
|
227
|
-
const drainCutoff = Promise.withResolvers<void>();
|
|
228
|
-
this.#drainCutoff = drainCutoff.promise;
|
|
229
|
-
this.#resolveDrainCutoff = drainCutoff.resolve;
|
|
230
|
-
// The cutoff remains pending for untimed commands, preserving complete
|
|
231
|
-
// EOF-based capture. attachTimeout() resolves it at the command deadline.
|
|
232
229
|
|
|
233
|
-
const pipeCutoff = this.#drainCutoff;
|
|
234
230
|
this.#stderrDone = (async () => {
|
|
235
231
|
const reader = stderrStream.getReader();
|
|
232
|
+
this.#pipeReaders.add(reader);
|
|
236
233
|
try {
|
|
237
234
|
for (;;) {
|
|
238
|
-
|
|
239
|
-
reader.read().then(r => ({ cutoff: false as const, r })),
|
|
240
|
-
pipeCutoff.then(() => ({ cutoff: true as const })),
|
|
241
|
-
]);
|
|
242
|
-
if (chunk.cutoff) {
|
|
235
|
+
if (this.#cutoff) {
|
|
243
236
|
await reader.cancel().catch(() => {});
|
|
244
237
|
break;
|
|
245
238
|
}
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
this.#
|
|
239
|
+
const r = await reader.read();
|
|
240
|
+
if (r.done) break;
|
|
241
|
+
this.#stderrChunks?.push(r.value);
|
|
242
|
+
this.#stderrTail += dec.decode(r.value, { stream: true });
|
|
249
243
|
trim();
|
|
250
244
|
}
|
|
251
245
|
} catch {}
|
|
246
|
+
this.#pipeReaders.delete(reader);
|
|
252
247
|
this.#openPipeReaders--;
|
|
253
248
|
this.#stderrTail += dec.decode();
|
|
254
249
|
trim();
|
|
@@ -440,24 +435,23 @@ export class ChildProcess<In extends InMask = InMask> {
|
|
|
440
435
|
async #readStream(stream: ReadableStream<Uint8Array>): Promise<string> {
|
|
441
436
|
this.#openPipeReaders++;
|
|
442
437
|
const reader = stream.getReader();
|
|
438
|
+
this.#pipeReaders.add(reader);
|
|
443
439
|
const dec = new TextDecoder();
|
|
444
440
|
let out = "";
|
|
445
441
|
try {
|
|
446
442
|
for (;;) {
|
|
447
|
-
|
|
448
|
-
reader.read().then(r => ({ cutoff: false as const, r })),
|
|
449
|
-
this.#drainCutoff.then(() => ({ cutoff: true as const })),
|
|
450
|
-
]);
|
|
451
|
-
if (chunk.cutoff) {
|
|
443
|
+
if (this.#cutoff) {
|
|
452
444
|
await reader.cancel().catch(() => {});
|
|
453
445
|
break;
|
|
454
446
|
}
|
|
455
|
-
|
|
456
|
-
|
|
447
|
+
const r = await reader.read();
|
|
448
|
+
if (r.done) break;
|
|
449
|
+
out += dec.decode(r.value, { stream: true });
|
|
457
450
|
}
|
|
458
451
|
} catch {
|
|
459
452
|
// A cancelled or failed read keeps whatever was already collected.
|
|
460
453
|
}
|
|
454
|
+
this.#pipeReaders.delete(reader);
|
|
461
455
|
this.#openPipeReaders--;
|
|
462
456
|
return out + dec.decode();
|
|
463
457
|
}
|
|
@@ -465,29 +459,27 @@ export class ChildProcess<In extends InMask = InMask> {
|
|
|
465
459
|
async #readBytes(): Promise<Uint8Array<ArrayBuffer>> {
|
|
466
460
|
const reader = this.proc.stdout.getReader();
|
|
467
461
|
this.#openPipeReaders++;
|
|
462
|
+
this.#pipeReaders.add(reader);
|
|
468
463
|
const chunks: Uint8Array[] = [];
|
|
469
464
|
let length = 0;
|
|
470
465
|
try {
|
|
471
466
|
for (;;) {
|
|
472
|
-
|
|
473
|
-
reader.read().then(r => ({ cutoff: false as const, r })),
|
|
474
|
-
this.#drainCutoff.then(() => ({ cutoff: true as const })),
|
|
475
|
-
]);
|
|
476
|
-
if (chunk.cutoff) {
|
|
467
|
+
if (this.#cutoff) {
|
|
477
468
|
await reader.cancel().catch(() => {});
|
|
478
469
|
break;
|
|
479
470
|
}
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
471
|
+
const r = await reader.read();
|
|
472
|
+
if (r.done) break;
|
|
473
|
+
chunks.push(r.value);
|
|
474
|
+
length += r.value.byteLength;
|
|
483
475
|
}
|
|
484
476
|
} catch {
|
|
485
477
|
// A cancelled or failed read keeps whatever was already collected.
|
|
486
478
|
} finally {
|
|
479
|
+
this.#pipeReaders.delete(reader);
|
|
487
480
|
this.#openPipeReaders--;
|
|
488
481
|
reader.releaseLock();
|
|
489
482
|
}
|
|
490
|
-
|
|
491
483
|
const bytes = new Uint8Array(length);
|
|
492
484
|
let offset = 0;
|
|
493
485
|
for (const chunk of chunks) {
|
|
@@ -597,7 +589,8 @@ export class ChildProcess<In extends InMask = InMask> {
|
|
|
597
589
|
) {
|
|
598
590
|
this.kill(new TimeoutError(ms, this.#stderrTail), -1);
|
|
599
591
|
}
|
|
600
|
-
this.#
|
|
592
|
+
this.#cutoff = true;
|
|
593
|
+
for (const reader of this.#pipeReaders) reader.cancel().catch(() => {});
|
|
601
594
|
}, ms);
|
|
602
595
|
timer.unref?.();
|
|
603
596
|
this.#timeoutTimer = timer;
|
|
@@ -34,6 +34,9 @@ const UNLIKELY_ROLES = new Set(["menu", "menubar", "complementary", "navigation"
|
|
|
34
34
|
const ARTICLE_TYPES =
|
|
35
35
|
/^(?:Article|AdvertiserContentArticle|NewsArticle|AnalysisNewsArticle|OpinionNewsArticle|ReportageNewsArticle|ReviewNewsArticle|Report|ScholarlyArticle|MedicalScholarlyArticle|SocialMediaPosting|BlogPosting|LiveBlogPosting|DiscussionForumPosting|TechArticle|APIReference)$/;
|
|
36
36
|
const NORMALIZE = /\s{2,}/g;
|
|
37
|
+
const ELEMENT_NODE = 1;
|
|
38
|
+
const TEXT_NODE = 3;
|
|
39
|
+
const CDATA_SECTION_NODE = 4;
|
|
37
40
|
|
|
38
41
|
type Metadata = {
|
|
39
42
|
title?: string;
|
|
@@ -119,6 +122,273 @@ function linkDensity(node: ReadabilityElement): number {
|
|
|
119
122
|
return linked / total;
|
|
120
123
|
}
|
|
121
124
|
|
|
125
|
+
/** Whether a UTF-16 code unit is whitespace to `String#trim` and `\s`. */
|
|
126
|
+
function isSpace(code: number): boolean {
|
|
127
|
+
if (code <= 0x20) return code === 0x20 || (code >= 0x09 && code <= 0x0d);
|
|
128
|
+
if (code < 0xa0) return false;
|
|
129
|
+
return (
|
|
130
|
+
code === 0xa0 ||
|
|
131
|
+
code === 0x1680 ||
|
|
132
|
+
(code >= 0x2000 && code <= 0x200a) ||
|
|
133
|
+
code === 0x2028 ||
|
|
134
|
+
code === 0x2029 ||
|
|
135
|
+
code === 0x202f ||
|
|
136
|
+
code === 0x205f ||
|
|
137
|
+
code === 0x3000 ||
|
|
138
|
+
code === 0xfeff
|
|
139
|
+
);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** Separators `#scoreParagraph` counts clauses by. */
|
|
143
|
+
function isSeparator(code: number): boolean {
|
|
144
|
+
return (
|
|
145
|
+
code === 0x2c ||
|
|
146
|
+
code === 0x060c ||
|
|
147
|
+
code === 0xfe50 ||
|
|
148
|
+
code === 0xfe10 ||
|
|
149
|
+
code === 0xfe11 ||
|
|
150
|
+
code === 0x2e41 ||
|
|
151
|
+
code === 0x2e34 ||
|
|
152
|
+
code === 0x2e32 ||
|
|
153
|
+
code === 0xff0c
|
|
154
|
+
);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
const SEPARATORS = /[\u002c\u060c\ufe50\ufe10\ufe11\u2e41\u2e34\u2e32\uff0c]/;
|
|
158
|
+
|
|
159
|
+
/** {@link TextRun} flags: no non-whitespace text; starts with whitespace; ends with whitespace. */
|
|
160
|
+
const BLANK = 1;
|
|
161
|
+
const LEADING_SPACE = 2;
|
|
162
|
+
const TRAILING_SPACE = 4;
|
|
163
|
+
|
|
164
|
+
/** Measures `text()` of concatenated pieces without building the string. */
|
|
165
|
+
class TextRun {
|
|
166
|
+
length = 0;
|
|
167
|
+
/** A blank run that contains whitespace carries both space flags. */
|
|
168
|
+
flags = BLANK;
|
|
169
|
+
commas = 0;
|
|
170
|
+
separators = 0;
|
|
171
|
+
|
|
172
|
+
reset(): void {
|
|
173
|
+
this.length = 0;
|
|
174
|
+
this.flags = BLANK;
|
|
175
|
+
this.commas = 0;
|
|
176
|
+
this.separators = 0;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** Append a run measured earlier. */
|
|
180
|
+
append(length: number, flags: number, commas: number, separators: number): void {
|
|
181
|
+
this.commas += commas;
|
|
182
|
+
this.separators += separators;
|
|
183
|
+
if (flags & BLANK) {
|
|
184
|
+
if (flags & LEADING_SPACE) this.flags |= this.flags & BLANK ? LEADING_SPACE | TRAILING_SPACE : TRAILING_SPACE;
|
|
185
|
+
return;
|
|
186
|
+
}
|
|
187
|
+
if (this.flags & BLANK) {
|
|
188
|
+
this.length = length;
|
|
189
|
+
this.flags = (this.flags & LEADING_SPACE) | flags;
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
192
|
+
// Whitespace between two non-blank runs collapses to a single character.
|
|
193
|
+
if (this.flags & TRAILING_SPACE || flags & LEADING_SPACE) this.length++;
|
|
194
|
+
this.length += length;
|
|
195
|
+
this.flags = (this.flags & LEADING_SPACE) | (flags & TRAILING_SPACE);
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
appendText(value: string): void {
|
|
199
|
+
if (!value) return;
|
|
200
|
+
let length = 0;
|
|
201
|
+
let commas = 0;
|
|
202
|
+
let separators = 0;
|
|
203
|
+
let seen = false;
|
|
204
|
+
let gap = false;
|
|
205
|
+
for (let index = 0; index < value.length; index++) {
|
|
206
|
+
const code = value.charCodeAt(index);
|
|
207
|
+
if (isSpace(code)) {
|
|
208
|
+
if (seen) gap = true;
|
|
209
|
+
continue;
|
|
210
|
+
}
|
|
211
|
+
if (gap) {
|
|
212
|
+
length++;
|
|
213
|
+
gap = false;
|
|
214
|
+
}
|
|
215
|
+
length++;
|
|
216
|
+
seen = true;
|
|
217
|
+
if (isSeparator(code)) {
|
|
218
|
+
separators++;
|
|
219
|
+
if (code === 0x2c) commas++;
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
const flags = seen
|
|
223
|
+
? (isSpace(value.charCodeAt(0)) ? LEADING_SPACE : 0) | (gap ? TRAILING_SPACE : 0)
|
|
224
|
+
: BLANK | LEADING_SPACE | TRAILING_SPACE;
|
|
225
|
+
this.append(length, flags, commas, separators);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
function isTag(node: ReadabilityElement, upper: string, lower: string): boolean {
|
|
230
|
+
return node.tagName === upper || node.tagName === lower;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* `text()` lengths, comma counts, descendant tag counts and link density of every element in a tree, from one
|
|
235
|
+
* bottom-up pass over the whole tree holding the first element asked about. Answers describe the tree as
|
|
236
|
+
* measured, so only ask about elements whose subtree has not changed since.
|
|
237
|
+
*/
|
|
238
|
+
class SubtreeStats {
|
|
239
|
+
readonly #countsAsText: (node: ReadabilityNode) => boolean;
|
|
240
|
+
readonly #index = new Map<ReadabilityNode, number>();
|
|
241
|
+
readonly #length: number[] = [];
|
|
242
|
+
readonly #flags: number[] = [];
|
|
243
|
+
readonly #commas: number[] = [];
|
|
244
|
+
readonly #separators: number[] = [];
|
|
245
|
+
readonly #paragraphs: number[] = [];
|
|
246
|
+
readonly #images: number[] = [];
|
|
247
|
+
readonly #inputs: number[] = [];
|
|
248
|
+
/** Range of an element's descendant anchors in {@link #links}. */
|
|
249
|
+
readonly #linkStart: number[] = [];
|
|
250
|
+
readonly #linkEnd: number[] = [];
|
|
251
|
+
/** `linkDensity` term of each anchor, in document order. */
|
|
252
|
+
readonly #links: number[] = [];
|
|
253
|
+
|
|
254
|
+
/** `countsAsText` tells whether a non-element child's `textContent` is part of its parent's. */
|
|
255
|
+
constructor(countsAsText: (node: ReadabilityNode) => boolean) {
|
|
256
|
+
this.#countsAsText = countsAsText;
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/** `text(node).length`. */
|
|
260
|
+
length(node: ReadabilityElement): number {
|
|
261
|
+
const index = this.#at(node);
|
|
262
|
+
return index < 0 ? text(node).length : this.#length[index];
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/** `!text(node)`. */
|
|
266
|
+
blank(node: ReadabilityElement): boolean {
|
|
267
|
+
const index = this.#at(node);
|
|
268
|
+
return index < 0 ? !text(node) : (this.#flags[index] & BLANK) !== 0;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/** `text(node).split(",").length - 1`. */
|
|
272
|
+
commas(node: ReadabilityElement): number {
|
|
273
|
+
const index = this.#at(node);
|
|
274
|
+
return index < 0 ? text(node).split(",").length - 1 : this.#commas[index];
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/** `text(node).split(SEPARATORS).length - 1`. */
|
|
278
|
+
separators(node: ReadabilityElement): number {
|
|
279
|
+
const index = this.#at(node);
|
|
280
|
+
return index < 0 ? text(node).split(SEPARATORS).length - 1 : this.#separators[index];
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/** `node.getElementsByTagName("p").length`. */
|
|
284
|
+
paragraphs(node: ReadabilityElement): number {
|
|
285
|
+
const index = this.#at(node);
|
|
286
|
+
return index < 0 ? node.getElementsByTagName("p").length : this.#paragraphs[index];
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
/** `node.getElementsByTagName("img").length`. */
|
|
290
|
+
images(node: ReadabilityElement): number {
|
|
291
|
+
const index = this.#at(node);
|
|
292
|
+
return index < 0 ? node.getElementsByTagName("img").length : this.#images[index];
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/** `node.getElementsByTagName("input").length`. */
|
|
296
|
+
inputs(node: ReadabilityElement): number {
|
|
297
|
+
const index = this.#at(node);
|
|
298
|
+
return index < 0 ? node.getElementsByTagName("input").length : this.#inputs[index];
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
/** `linkDensity(node)`, summed in the same order so the result is bit-identical. */
|
|
302
|
+
linkDensity(node: ReadabilityElement): number {
|
|
303
|
+
const index = this.#at(node);
|
|
304
|
+
if (index < 0) return linkDensity(node);
|
|
305
|
+
const total = this.#length[index];
|
|
306
|
+
if (!total) return 0;
|
|
307
|
+
let linked = 0;
|
|
308
|
+
for (let link = this.#linkStart[index]; link < this.#linkEnd[index]; link++) linked += this.#links[link];
|
|
309
|
+
return linked / total;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
#at(node: ReadabilityElement): number {
|
|
313
|
+
const index = this.#index.get(node);
|
|
314
|
+
if (index !== undefined) return index;
|
|
315
|
+
if (node.nodeType !== ELEMENT_NODE) return -1;
|
|
316
|
+
let root: ReadabilityNode = node;
|
|
317
|
+
while (root.parentNode?.nodeType === ELEMENT_NODE) root = root.parentNode;
|
|
318
|
+
this.#measure(root as ReadabilityElement);
|
|
319
|
+
return this.#index.get(node) ?? -1;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
#measure(root: ReadabilityElement): void {
|
|
323
|
+
const first = this.#length.length;
|
|
324
|
+
const order: ReadabilityElement[] = [];
|
|
325
|
+
const anchors: ReadabilityElement[] = [];
|
|
326
|
+
const pending = [root];
|
|
327
|
+
while (pending.length) {
|
|
328
|
+
const node = pending.pop();
|
|
329
|
+
if (!node) continue;
|
|
330
|
+
this.#index.set(node, first + order.length);
|
|
331
|
+
order.push(node);
|
|
332
|
+
// An anchor counts for its ancestors only, so its own range starts after it.
|
|
333
|
+
if (isTag(node, "A", "a")) anchors.push(node);
|
|
334
|
+
this.#linkStart.push(this.#links.length + anchors.length);
|
|
335
|
+
this.#linkEnd.push(0);
|
|
336
|
+
this.#length.push(0);
|
|
337
|
+
this.#flags.push(0);
|
|
338
|
+
this.#commas.push(0);
|
|
339
|
+
this.#separators.push(0);
|
|
340
|
+
this.#paragraphs.push(0);
|
|
341
|
+
this.#images.push(0);
|
|
342
|
+
this.#inputs.push(0);
|
|
343
|
+
const children = node.childNodes;
|
|
344
|
+
for (let child = children.length - 1; child >= 0; child--) {
|
|
345
|
+
if (children[child].nodeType === ELEMENT_NODE) pending.push(children[child] as ReadabilityElement);
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
const run = new TextRun();
|
|
349
|
+
// Reverse document order reaches every element after all of its descendants.
|
|
350
|
+
for (let offset = order.length - 1; offset >= 0; offset--) {
|
|
351
|
+
const node = order[offset];
|
|
352
|
+
const index = first + offset;
|
|
353
|
+
const children = node.childNodes;
|
|
354
|
+
// A template's parsed children are not part of its text in every DOM; ask it directly.
|
|
355
|
+
const leaf = !children.length || isTag(node, "TEMPLATE", "template");
|
|
356
|
+
run.reset();
|
|
357
|
+
if (leaf) run.appendText(node.textContent ?? "");
|
|
358
|
+
let paragraphs = 0;
|
|
359
|
+
let images = 0;
|
|
360
|
+
let inputs = 0;
|
|
361
|
+
let linkEnd = this.#linkStart[index];
|
|
362
|
+
for (let child = 0; child < children.length; child++) {
|
|
363
|
+
const childNode = children[child];
|
|
364
|
+
if (childNode.nodeType !== ELEMENT_NODE) {
|
|
365
|
+
if (!leaf && this.#countsAsText(childNode)) run.appendText(childNode.textContent ?? "");
|
|
366
|
+
continue;
|
|
367
|
+
}
|
|
368
|
+
const element = childNode as ReadabilityElement;
|
|
369
|
+
const at = this.#index.get(element) ?? -1;
|
|
370
|
+
if (!leaf) run.append(this.#length[at], this.#flags[at], this.#commas[at], this.#separators[at]);
|
|
371
|
+
paragraphs += this.#paragraphs[at] + (isTag(element, "P", "p") ? 1 : 0);
|
|
372
|
+
images += this.#images[at] + (isTag(element, "IMG", "img") ? 1 : 0);
|
|
373
|
+
inputs += this.#inputs[at] + (isTag(element, "INPUT", "input") ? 1 : 0);
|
|
374
|
+
linkEnd = this.#linkEnd[at];
|
|
375
|
+
}
|
|
376
|
+
this.#length[index] = run.length;
|
|
377
|
+
this.#flags[index] = run.flags;
|
|
378
|
+
this.#commas[index] = run.commas;
|
|
379
|
+
this.#separators[index] = run.separators;
|
|
380
|
+
this.#paragraphs[index] = paragraphs;
|
|
381
|
+
this.#images[index] = images;
|
|
382
|
+
this.#inputs[index] = inputs;
|
|
383
|
+
this.#linkEnd[index] = linkEnd;
|
|
384
|
+
}
|
|
385
|
+
for (const anchor of anchors) {
|
|
386
|
+
const length = this.#length[this.#index.get(anchor) ?? -1];
|
|
387
|
+
this.#links.push(length * ((anchor.getAttribute("href") ?? "").startsWith("#") ? 0.3 : 1));
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
|
|
122
392
|
function visible(node: ReadabilityElement): boolean {
|
|
123
393
|
const style = node.getAttribute("style")?.toLowerCase() ?? "";
|
|
124
394
|
return (
|
|
@@ -135,6 +405,24 @@ function removeAll(root: ReadabilityNode, tags: readonly string[]): void {
|
|
|
135
405
|
}
|
|
136
406
|
}
|
|
137
407
|
|
|
408
|
+
/** Replace `parent`'s children, detaching the old ones in one step where the DOM supports it. */
|
|
409
|
+
function replaceChildren(parent: ReadabilityNode, children: readonly ReadabilityNode[]): void {
|
|
410
|
+
if (parent.replaceChildren) parent.replaceChildren();
|
|
411
|
+
else while (parent.firstChild) parent.firstChild.remove();
|
|
412
|
+
for (const child of children) parent.appendChild(child);
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
/** Move all of `from`'s children to the end of `to` without detaching them one by one. */
|
|
416
|
+
function moveChildren(from: ReadabilityNode, to: ReadabilityNode): void {
|
|
417
|
+
if (!from.replaceChildren) {
|
|
418
|
+
while (from.firstChild) to.appendChild(from.firstChild);
|
|
419
|
+
return;
|
|
420
|
+
}
|
|
421
|
+
const children = Array.from(from.childNodes);
|
|
422
|
+
from.replaceChildren();
|
|
423
|
+
for (const child of children) to.appendChild(child);
|
|
424
|
+
}
|
|
425
|
+
|
|
138
426
|
function entityDecode(value: string | undefined | null): string | undefined | null {
|
|
139
427
|
if (!value) return value;
|
|
140
428
|
const named: Record<string, string> = { quot: '"', amp: "&", apos: "'", lt: "<", gt: ">" };
|
|
@@ -283,6 +571,8 @@ export class Readability<T = string> {
|
|
|
283
571
|
readonly #scores = new Map<ReadabilityElement, number>();
|
|
284
572
|
#byline: string | undefined;
|
|
285
573
|
#lang: string | null = null;
|
|
574
|
+
/** Per non-text node type, whether this DOM counts its `textContent` in a parent's. */
|
|
575
|
+
readonly #textByNodeType = new Map<number, boolean>();
|
|
286
576
|
|
|
287
577
|
constructor(document: ReadabilityDocument, options: ReadabilityOptions<T> = {}) {
|
|
288
578
|
this.#document = document;
|
|
@@ -303,10 +593,11 @@ export class Readability<T = string> {
|
|
|
303
593
|
removeAll(this.#document, ["script", "style"]);
|
|
304
594
|
const body = this.#document.body;
|
|
305
595
|
if (!body) return null;
|
|
306
|
-
|
|
596
|
+
// Retries start from the body as it was before the first attempt rearranged it.
|
|
597
|
+
const pristine = Array.from(body.childNodes, node => node.cloneNode(true));
|
|
307
598
|
const attempts: Attempt[] = [];
|
|
308
599
|
for (const mode of [0, 1, 2, 3]) {
|
|
309
|
-
if (mode) body.
|
|
600
|
+
if (mode) replaceChildren(body, mode === 3 ? pristine : pristine.map(node => node.cloneNode(true)));
|
|
310
601
|
this.#scores.clear();
|
|
311
602
|
this.#byline = undefined;
|
|
312
603
|
const attempt = this.#extract(body, documentElement, metadata.title ?? "", mode);
|
|
@@ -356,10 +647,13 @@ export class Readability<T = string> {
|
|
|
356
647
|
node.remove();
|
|
357
648
|
continue;
|
|
358
649
|
}
|
|
359
|
-
if (!this.#byline
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
650
|
+
if (!this.#byline) {
|
|
651
|
+
const byline = this.#bylineText(node, label);
|
|
652
|
+
if (byline) {
|
|
653
|
+
this.#byline = byline;
|
|
654
|
+
node.remove();
|
|
655
|
+
continue;
|
|
656
|
+
}
|
|
363
657
|
}
|
|
364
658
|
if (!titleRemoved && /^(?:H1|H2)$/.test(node.tagName) && this.#similar(articleTitle, text(node)) > 0.75) {
|
|
365
659
|
titleRemoved = true;
|
|
@@ -375,12 +669,14 @@ export class Readability<T = string> {
|
|
|
375
669
|
}
|
|
376
670
|
if (SCORE_TAGS.has(node.tagName)) scored.push(node);
|
|
377
671
|
}
|
|
378
|
-
|
|
672
|
+
// Scoring and sibling selection below only read the tree, so one measurement serves them all.
|
|
673
|
+
const stats = new SubtreeStats(this.#countsAsText);
|
|
674
|
+
for (const paragraph of scored) this.#scoreParagraph(paragraph, weightClasses, stats);
|
|
379
675
|
let top: ReadabilityElement | undefined;
|
|
380
676
|
let topScore = Number.NEGATIVE_INFINITY;
|
|
381
677
|
for (const [candidate, raw] of this.#scores) {
|
|
382
678
|
if (candidate.tagName === "BODY" || candidate.tagName === "HTML") continue;
|
|
383
|
-
const score = raw * (1 - linkDensity(candidate));
|
|
679
|
+
const score = raw * (1 - stats.linkDensity(candidate));
|
|
384
680
|
this.#scores.set(candidate, score);
|
|
385
681
|
if (score > topScore) {
|
|
386
682
|
top = candidate;
|
|
@@ -399,18 +695,19 @@ export class Readability<T = string> {
|
|
|
399
695
|
const siblings = parent ? elements(parent.children) : [top];
|
|
400
696
|
const threshold = Math.max(10, (this.#scores.get(top) ?? topScore) * 0.2);
|
|
401
697
|
for (const sibling of siblings) {
|
|
402
|
-
|
|
698
|
+
// Moving earlier siblings into the article leaves this sibling's subtree, and so its stats, intact.
|
|
699
|
+
const siblingLength = stats.length(sibling);
|
|
403
700
|
const sameClassBonus =
|
|
404
701
|
sibling.className && sibling.className === top.className ? (this.#scores.get(top) ?? 0) * 0.2 : 0;
|
|
405
702
|
const include =
|
|
406
703
|
sibling === top ||
|
|
407
704
|
(this.#scores.get(sibling) ?? 0) + sameClassBonus >= threshold ||
|
|
408
705
|
(sibling.tagName === "P" &&
|
|
409
|
-
((
|
|
410
|
-
(
|
|
411
|
-
|
|
412
|
-
linkDensity(sibling) === 0 &&
|
|
413
|
-
/\.(?: |$)/.test(
|
|
706
|
+
((siblingLength > 80 && stats.linkDensity(sibling) < 0.25) ||
|
|
707
|
+
(siblingLength > 0 &&
|
|
708
|
+
siblingLength < 80 &&
|
|
709
|
+
stats.linkDensity(sibling) === 0 &&
|
|
710
|
+
/\.(?: |$)/.test(text(sibling)))));
|
|
414
711
|
if (!include) continue;
|
|
415
712
|
if (["DIV", "ARTICLE", "SECTION", "P", "OL", "UL"].includes(sibling.tagName)) {
|
|
416
713
|
article.appendChild(sibling);
|
|
@@ -419,14 +716,14 @@ export class Readability<T = string> {
|
|
|
419
716
|
const replacement = this.#document.createElement("DIV");
|
|
420
717
|
for (const attribute of Array.from(sibling.attributes))
|
|
421
718
|
replacement.setAttribute(attribute.name, attribute.value);
|
|
422
|
-
|
|
719
|
+
moveChildren(sibling, replacement);
|
|
423
720
|
article.appendChild(replacement);
|
|
424
721
|
}
|
|
425
722
|
this.#clean(article, mode < 3);
|
|
426
723
|
const page = this.#document.createElement("DIV");
|
|
427
724
|
page.id = "readability-page-1";
|
|
428
725
|
page.className = "page";
|
|
429
|
-
|
|
726
|
+
moveChildren(article, page);
|
|
430
727
|
article.appendChild(page);
|
|
431
728
|
const content = text(article);
|
|
432
729
|
let dir: string | null | undefined;
|
|
@@ -441,13 +738,10 @@ export class Readability<T = string> {
|
|
|
441
738
|
return { element: article, length: content.length, dir };
|
|
442
739
|
}
|
|
443
740
|
|
|
444
|
-
#scoreParagraph(node: ReadabilityElement, weightClasses: boolean): void {
|
|
445
|
-
const
|
|
446
|
-
if (
|
|
447
|
-
const score =
|
|
448
|
-
1 +
|
|
449
|
-
content.split(/[\u002c\u060c\ufe50\ufe10\ufe11\u2e41\u2e34\u2e32\uff0c]/).length +
|
|
450
|
-
Math.min(Math.floor(content.length / 100), 3);
|
|
741
|
+
#scoreParagraph(node: ReadabilityElement, weightClasses: boolean, stats: SubtreeStats): void {
|
|
742
|
+
const length = stats.length(node);
|
|
743
|
+
if (length < 25) return;
|
|
744
|
+
const score = 1 + (stats.separators(node) + 1) + Math.min(Math.floor(length / 100), 3);
|
|
451
745
|
let ancestor = node.parentNode;
|
|
452
746
|
for (let level = 0; ancestor && level < 5; level++, ancestor = ancestor.parentNode) {
|
|
453
747
|
const element = ancestor as ReadabilityElement;
|
|
@@ -465,19 +759,19 @@ export class Readability<T = string> {
|
|
|
465
759
|
if (classWeight(heading) < 0 || linkDensity(heading) > 0.33) heading.remove();
|
|
466
760
|
}
|
|
467
761
|
if (conditional) {
|
|
762
|
+
// Nodes come in document order, so removing one only changes ancestors that were already judged.
|
|
763
|
+
const stats = new SubtreeStats(this.#countsAsText);
|
|
468
764
|
for (const node of elements(root.querySelectorAll("table, ul, div"))) {
|
|
469
765
|
if (node === root) continue;
|
|
470
|
-
const
|
|
471
|
-
const
|
|
472
|
-
const images = node.getElementsByTagName("img").length;
|
|
473
|
-
const inputs = node.getElementsByTagName("input").length;
|
|
766
|
+
const paragraphs = stats.paragraphs(node);
|
|
767
|
+
const images = stats.images(node);
|
|
474
768
|
if (
|
|
475
769
|
classWeight(node) < 0 ||
|
|
476
|
-
(
|
|
477
|
-
(
|
|
770
|
+
(stats.blank(node) && !images) ||
|
|
771
|
+
(stats.commas(node) + 1 < 10 &&
|
|
478
772
|
((images > paragraphs && paragraphs > 0) ||
|
|
479
|
-
inputs > Math.floor(paragraphs / 3) ||
|
|
480
|
-
linkDensity(node) > 0.5))
|
|
773
|
+
stats.inputs(node) > Math.floor(paragraphs / 3) ||
|
|
774
|
+
stats.linkDensity(node) > 0.5))
|
|
481
775
|
)
|
|
482
776
|
node.remove();
|
|
483
777
|
}
|
|
@@ -512,17 +806,33 @@ export class Readability<T = string> {
|
|
|
512
806
|
}
|
|
513
807
|
}
|
|
514
808
|
|
|
515
|
-
|
|
809
|
+
/** Byline text when `node` is marked as one; the attribute checks run first so most nodes never build text. */
|
|
810
|
+
#bylineText(node: ReadabilityElement, label: string): string | undefined {
|
|
811
|
+
if (
|
|
812
|
+
node.getAttribute("rel") !== "author" &&
|
|
813
|
+
!(node.getAttribute("itemprop") ?? "").includes("author") &&
|
|
814
|
+
!BYLINE.test(label)
|
|
815
|
+
)
|
|
816
|
+
return undefined;
|
|
516
817
|
const value = text(node);
|
|
517
|
-
return
|
|
518
|
-
value.length > 0 &&
|
|
519
|
-
value.length < 100 &&
|
|
520
|
-
(node.getAttribute("rel") === "author" ||
|
|
521
|
-
(node.getAttribute("itemprop") ?? "").includes("author") ||
|
|
522
|
-
BYLINE.test(label))
|
|
523
|
-
);
|
|
818
|
+
return value.length > 0 && value.length < 100 ? value : undefined;
|
|
524
819
|
}
|
|
525
820
|
|
|
821
|
+
/** Whether a non-element child's `textContent` is part of its parent's (text is; comments are in some DOMs). */
|
|
822
|
+
readonly #countsAsText = (node: ReadabilityNode): boolean => {
|
|
823
|
+
if (node.nodeType === TEXT_NODE || node.nodeType === CDATA_SECTION_NODE) return true;
|
|
824
|
+
const value = node.textContent;
|
|
825
|
+
if (!value) return false;
|
|
826
|
+
let counts = this.#textByNodeType.get(node.nodeType);
|
|
827
|
+
if (counts === undefined) {
|
|
828
|
+
const probe = this.#document.createElement("DIV");
|
|
829
|
+
probe.appendChild(node.cloneNode(false));
|
|
830
|
+
counts = probe.textContent === value;
|
|
831
|
+
this.#textByNodeType.set(node.nodeType, counts);
|
|
832
|
+
}
|
|
833
|
+
return counts;
|
|
834
|
+
};
|
|
835
|
+
|
|
526
836
|
#similar(left: string, right: string): number {
|
|
527
837
|
const leftTokens = left.toLowerCase().split(/\W+/).filter(Boolean);
|
|
528
838
|
const rightTokens = right.toLowerCase().split(/\W+/).filter(Boolean);
|
package/src/readability/types.ts
CHANGED
|
@@ -14,6 +14,8 @@ export interface ReadabilityNode {
|
|
|
14
14
|
appendChild(node: ReadabilityNode): ReadabilityNode;
|
|
15
15
|
cloneNode(deep?: boolean): ReadabilityNode;
|
|
16
16
|
remove(): void;
|
|
17
|
+
/** Detach all children; used when present to move or reset children in one step. */
|
|
18
|
+
replaceChildren?(): void;
|
|
17
19
|
}
|
|
18
20
|
|
|
19
21
|
/** Minimal standards-shaped DOM element accepted by the readability extractor. */
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Published CLI release-asset naming — the one contract shared by the release
|
|
3
|
+
* build (`scripts/ci-release-build-binaries.ts` outfiles), the CI upload matrix
|
|
4
|
+
* (`.github/workflows/ci.yml` `release_binary.binary_path`), the installers
|
|
5
|
+
* (`scripts/install.sh` / `install.ps1`) and the updater's binary fallback.
|
|
6
|
+
*
|
|
7
|
+
* Assets are `zeta-cli-<os>-<arch>[.exe]`. `zeta-cli` is the *file-name* face
|
|
8
|
+
* of the package and deliberately does NOT follow {@link CLI_BIN_NAME}
|
|
9
|
+
* (`zetacode`, the user-visible command face): the brand cutover moved the
|
|
10
|
+
* command name, and the updater's asset name drifted with it, so every binary
|
|
11
|
+
* fallback asked GitHub for `zetacode-*` assets no release carries (v1.1.27 /
|
|
12
|
+
* main). `scripts/check-release-asset-names.ts` pins both sides together.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Leading token of every published CLI binary asset, separator included:
|
|
17
|
+
* assets are `<prefix><os>[-musl]-<arch>[.exe]`.
|
|
18
|
+
*/
|
|
19
|
+
export const RELEASE_CLI_ASSET_PREFIX = "zeta-cli-";
|
|
20
|
+
|
|
21
|
+
/** Asset-name OS token per platform; Linux additionally has a musl build. */
|
|
22
|
+
const RELEASE_OS: Partial<Record<NodeJS.Platform, string>> = {
|
|
23
|
+
darwin: "darwin",
|
|
24
|
+
linux: "linux",
|
|
25
|
+
win32: "windows",
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
const RELEASE_ARCH: Record<string, string> = { x64: "x64", arm64: "arm64" };
|
|
29
|
+
|
|
30
|
+
/** Release asset for a CLI binary, e.g. `zeta-cli-windows-x64.exe`. */
|
|
31
|
+
export function releaseCliAssetName(platform: NodeJS.Platform, arch: string, options: { musl?: boolean } = {}): string {
|
|
32
|
+
const os = RELEASE_OS[platform];
|
|
33
|
+
if (!os) throw new Error(`Unsupported platform: ${platform}`);
|
|
34
|
+
if (options.musl && platform !== "linux") throw new Error(`musl builds exist on linux only: ${platform}`);
|
|
35
|
+
const archName = RELEASE_ARCH[arch];
|
|
36
|
+
if (!archName) throw new Error(`Unsupported architecture: ${arch}`);
|
|
37
|
+
const libc = options.musl ? "-musl" : "";
|
|
38
|
+
const suffix = platform === "win32" ? ".exe" : "";
|
|
39
|
+
return `${RELEASE_CLI_ASSET_PREFIX}${os}${libc}-${archName}${suffix}`;
|
|
40
|
+
}
|