onionskin-cli 0.0.2 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +111 -0
- package/INSTALL.md +5 -1
- package/NOTICE +34 -0
- package/README.md +101 -33
- package/docs/contracts/batch.schema.json +27 -4
- package/docs/contracts/ops.schema.json +117 -5
- package/package.json +4 -2
- package/skills/onionskin/SKILL.md +9 -6
- package/skills/onionskin/reference/batch.md +5 -3
- package/skills/onionskin/reference/formats.md +5 -1
- package/skills/onionskin/reference/ops.md +47 -3
- package/skills/onionskin/reference/troubleshooting.md +31 -0
- package/src/agent-detect.js +28 -2
- package/src/agent-prompt.js +12 -1
- package/src/agent-runner.js +77 -19
- package/src/anchor-work.js +154 -0
- package/src/anchor-worker.js +38 -0
- package/src/anchors.js +61 -11
- package/src/apply.js +119 -47
- package/src/batch.js +32 -8
- package/src/check-worker.js +8 -5
- package/src/cli.js +91 -25
- package/src/constants.js +34 -1
- package/src/diff.js +3 -1
- package/src/docx.js +368 -58
- package/src/edit-hash.js +19 -0
- package/src/fsutil.js +1 -1
- package/src/history-worker.js +8 -8
- package/src/history.js +135 -31
- package/src/html-active.js +384 -0
- package/src/html-envelope.js +51 -0
- package/src/html-parser.js +8 -0
- package/src/html-render-worker.js +25 -0
- package/src/html-render.js +880 -0
- package/src/html-utf8.js +65 -0
- package/src/html-write-worker.js +14 -0
- package/src/html-write.js +380 -0
- package/src/html.js +1257 -0
- package/src/local.js +68 -4
- package/src/merge-package.js +1110 -0
- package/src/merge-pptx.js +98 -38
- package/src/merge-proof.js +1019 -0
- package/src/merge-worker.js +60 -0
- package/src/merge-xlsx.js +84 -29
- package/src/merge.js +286 -83
- package/src/nfc.js +46 -0
- package/src/notes.js +122 -0
- package/src/offload.js +400 -1
- package/src/open-worker.js +27 -0
- package/src/ops/common.js +2 -1
- package/src/ops/docx.js +58 -29
- package/src/ops/html.js +394 -0
- package/src/ops/pptx.js +74 -21
- package/src/ops/run.js +23 -3
- package/src/ops/runs.js +215 -13
- package/src/ops/schema.js +8 -1
- package/src/ops/xlsx.js +12 -7
- package/src/ops-worker.js +17 -0
- package/src/package.js +26 -2
- package/src/person.js +8 -1
- package/src/pptx.js +85 -21
- package/src/quick-plan.js +13 -6
- package/src/redline-worker.js +10 -14
- package/src/reply.js +46 -19
- package/src/retain.js +223 -0
- package/src/review.js +32 -18
- package/src/rewrite-check.js +2 -2
- package/src/sanitize.js +16 -3
- package/src/server.js +88 -37
- package/src/session.js +1143 -208
- package/src/stop-gate.js +34 -0
- package/src/suggest-reply.js +42 -7
- package/src/suggest.js +14 -29
- package/src/text.js +161 -11
- package/src/update.js +288 -0
- package/src/voice-guide.js +4 -3
- package/src/voice-learn.js +4 -3
- package/src/voice-sample-worker.js +7 -6
- package/src/voice-worker.js +4 -6
- package/src/watch.js +188 -0
- package/src/worker-host.js +23 -0
- package/src/xlsx.js +355 -94
- package/src/xml-tree.js +6 -3
- package/src/xml.js +82 -34
- package/src/zip-write.js +2 -1
- package/src/zip.js +9 -5
- package/vendor/parse5/LICENSE.txt +32 -0
- package/vendor/parse5/parse5.mjs +7822 -0
- package/vendor/parse5.lock.json +23 -0
- package/web/dist/assets/index-DkNx0LSH.css +1 -0
- package/web/dist/assets/{index-BUtofdtr.js → index-DytKkMdi.js} +117 -107
- package/web/dist/index.html +2 -2
- package/web/dist/assets/index-Dfizlg0i.css +0 -1
package/src/stop-gate.js
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
// Stopping a worker thread while it is inside zlib aborts the whole server process on Node 26: terminating it between the creation of a
|
|
2
|
+
// zlib handle and its init leaves a handle the worker's cleanup closes uninitialized ("CompressionStream::Close() Assertion failed:
|
|
3
|
+
// init_done_ && \"close before init\"", SIGABRT). src/offload.js startWorker therefore never stops a worker inside a zlib call: the
|
|
4
|
+
// worker marks each call in a word it shares with its parent, and the parent, before it terminates the worker, marks that it is stopping
|
|
5
|
+
// it and waits for the call in progress to end. The worker never starts another call once its parent is stopping it.
|
|
6
|
+
// The word: bit 1 = the worker is inside a call, bit 2 = the parent is stopping it. Imports nothing, so src/worker-host.js stays
|
|
7
|
+
// synchronous to evaluate. On the server's own thread no word is set and gated(fn) just runs fn.
|
|
8
|
+
|
|
9
|
+
const INSIDE = 1;
|
|
10
|
+
export const STOPPING = 2;
|
|
11
|
+
let word = null;
|
|
12
|
+
let depth = 0;
|
|
13
|
+
|
|
14
|
+
/** Called once by src/worker-host.js with the worker's shared word (an Int32Array over a SharedArrayBuffer, index 1). */
|
|
15
|
+
export function setStopGate(arr) {
|
|
16
|
+
word = arr;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** Runs fn (a zlib call) where the worker cannot be stopped; when the parent is stopping it, waits here until it is stopped. */
|
|
20
|
+
export function gated(fn) {
|
|
21
|
+
if (!word) return fn();
|
|
22
|
+
if (depth === 0 && Atomics.compareExchange(word, 1, 0, INSIDE) !== 0) {
|
|
23
|
+
for (;;) Atomics.wait(word, 1, STOPPING); // the parent is stopping this worker: never enter zlib again, terminate() ends this wait
|
|
24
|
+
}
|
|
25
|
+
depth++;
|
|
26
|
+
try {
|
|
27
|
+
return fn();
|
|
28
|
+
} finally {
|
|
29
|
+
if (--depth === 0) {
|
|
30
|
+
Atomics.and(word, 1, ~INSIDE);
|
|
31
|
+
Atomics.notify(word, 1);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
}
|
package/src/suggest-reply.js
CHANGED
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
// Quick fixes (idea 10 option A), the pure half of "Use this": where in the file a suggestion may be written, and the one reply the
|
|
2
|
-
// local runner sends for it. No I/O and no session state: src/
|
|
2
|
+
// local runner sends for it, and where a selection sits before the model is asked. No I/O and no session state: src/anchor-work.js runs
|
|
3
|
+
// them (in the server's anchor worker, or on the caller's thread for the synchronous Sessions methods).
|
|
3
4
|
//
|
|
4
5
|
// What is written is only the changed middle of the sentence: the shortest span from the first change to the last, widened to whole words
|
|
5
6
|
// until it occurs exactly once in the paragraph (or text box) it sits in, otherwise the whole sentence. The engine refuses a find that
|
|
6
7
|
// occurs more than once, so a sentence that repeats word for word in its paragraph is refused and nothing is written.
|
|
8
|
+
import { untrusted } from "./sanitize.js";
|
|
9
|
+
import { sentenceSpan } from "./sentence-span.js";
|
|
10
|
+
import { normText, searchCost, spend } from "./text.js";
|
|
7
11
|
|
|
8
12
|
/** The comment's result note and the run's summary (docs/contracts/routes.md). Fixed words: never the model's text. */
|
|
9
13
|
export const USE_NOTE = "Rewritten on this Mac. Nothing left this Mac.";
|
|
@@ -11,11 +15,16 @@ export const USE_SUMMARY = "Rewrote 1 sentence on this Mac. Nothing left this Ma
|
|
|
11
15
|
|
|
12
16
|
const isSpace = (ch) => /\s/u.test(ch);
|
|
13
17
|
|
|
14
|
-
/**
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
18
|
+
/**
|
|
19
|
+
* Whether `q` occurs exactly once in `text`, overlapping occurrences counted (stricter than the engine's own count): it stops at the
|
|
20
|
+
* second. Each try is two native searches, paid from the request's budget (src/text.js searchCost, at its worst case); null when the
|
|
21
|
+
* budget cannot pay. An outside review: counting every occurrence of each widened span in a paragraph of one sentence repeated 10,000
|
|
22
|
+
* times took 8.3 s before Use answered "not here".
|
|
23
|
+
*/
|
|
24
|
+
function once(text, q) {
|
|
25
|
+
if (!spend(2 * searchCost(text.length, q))) return null;
|
|
26
|
+
const i = text.indexOf(q);
|
|
27
|
+
return i >= 0 && text.indexOf(q, i + 1) < 0;
|
|
19
28
|
}
|
|
20
29
|
|
|
21
30
|
/**
|
|
@@ -36,7 +45,9 @@ export function changedMiddle(before, after, unitText = before) {
|
|
|
36
45
|
while (e < before.length && !isSpace(before[e])) e++;
|
|
37
46
|
for (;;) {
|
|
38
47
|
const find = before.slice(s, e);
|
|
39
|
-
|
|
48
|
+
const unique = find.trim() ? once(unitText, find) : false;
|
|
49
|
+
if (unique === null) return null; // past the budget: nothing is written (the comment goes to the agent)
|
|
50
|
+
if (unique) {
|
|
40
51
|
const replace = after.slice(s, after.length - (before.length - e));
|
|
41
52
|
if (before.slice(0, s) + replace + before.slice(e) !== after) return null; // cannot happen; refuse rather than write something else
|
|
42
53
|
return { find, replace };
|
|
@@ -82,3 +93,27 @@ export function buildUseReply(record, commentId, unitText) {
|
|
|
82
93
|
if (!m) return null;
|
|
83
94
|
return { summary: USE_SUMMARY, replies: [{ comment: commentId, find: m.find, replace: m.replace, note: USE_NOTE }] };
|
|
84
95
|
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* What a quick fix's selection means in `units` (src/suggest.js, before the model is asked), from the paragraph or text box it sits in,
|
|
99
|
+
* never from the page: `d` is the durable anchor built from `anchor`, `picked` the selected text, `limits` = {context, maxQuote}.
|
|
100
|
+
* {code: "protected" | "not_eligible" | "selection"} (refused), {ok: false} (not found at that place: the page's text is used), or
|
|
101
|
+
* {ok: true, quote, from, wholeParagraph}: the whole sentence the selection means (src/sentence-span.js), the anchor for it (`from`: the
|
|
102
|
+
* page's location with the sentence's offsets, and `context` characters either side) and whether it is the whole paragraph. Only these
|
|
103
|
+
* leave: the paragraph's own text stays where it was read.
|
|
104
|
+
*/
|
|
105
|
+
export function selectionOf(units, d, anchor, picked, { context, maxQuote }) {
|
|
106
|
+
const place = placeOf(units, d, picked);
|
|
107
|
+
if (place.code === "protected" || place.code === "not_eligible") return { code: place.code };
|
|
108
|
+
if (!place.ok) return { ok: false };
|
|
109
|
+
let quote = picked;
|
|
110
|
+
let from = anchor;
|
|
111
|
+
const span = Number.isInteger(place.start) && Number.isInteger(place.end) ? sentenceSpan(place.text, place.start, place.end) : null;
|
|
112
|
+
if (span) {
|
|
113
|
+
quote = place.text.slice(span.start, span.end);
|
|
114
|
+
if (untrusted(quote, maxQuote + 100) !== quote || quote.length > maxQuote + 10) return { code: "selection" };
|
|
115
|
+
const loc = Number.isInteger(anchor.location.start) && Number.isInteger(anchor.location.end) ? { ...anchor.location, start: span.start, end: span.end } : anchor.location;
|
|
116
|
+
from = { location: loc, quote, prefix: place.text.slice(Math.max(0, span.start - context), span.start), suffix: place.text.slice(span.end, span.end + context) };
|
|
117
|
+
}
|
|
118
|
+
return { ok: true, quote, from, wholeParagraph: normText(place.text) === normText(quote) };
|
|
119
|
+
}
|
package/src/suggest.js
CHANGED
|
@@ -12,17 +12,13 @@
|
|
|
12
12
|
// Ceilings: one suggestion at a time per file here, and one at a time for the whole server in the worker (src/rewriter.js answers "busy").
|
|
13
13
|
// The store is lost with the server (a reload after a restart shows no card; the stamp is tapped again).
|
|
14
14
|
import crypto from "node:crypto";
|
|
15
|
-
import { buildDurable, readUnits } from "./anchors.js";
|
|
16
15
|
import { diffText } from "./diff.js";
|
|
17
|
-
import { checkRegularOwned, readDoc } from "./fsutil.js";
|
|
18
16
|
import { REWRITE_TASKS, checkRewrite, inputQualifies } from "./rewrite-check.js";
|
|
19
17
|
import { APPLE_STATES, appleHelperState, appleStateLine, getRewriter, rewriterStatus } from "./rewriter.js";
|
|
20
18
|
import { setupRewriter } from "./rewriter-setup.js";
|
|
21
19
|
import { untrusted } from "./sanitize.js";
|
|
20
|
+
import { anchorsOffThread } from "./offload.js";
|
|
22
21
|
import { HttpError } from "./session.js";
|
|
23
|
-
import { sentenceSpan } from "./sentence-span.js";
|
|
24
|
-
import { placeOf } from "./suggest-reply.js";
|
|
25
|
-
import { normText } from "./text.js";
|
|
26
22
|
|
|
27
23
|
export { USE_NOTE, USE_SUMMARY, buildUseReply, changedMiddle, placeOf } from "./suggest-reply.js";
|
|
28
24
|
|
|
@@ -114,32 +110,21 @@ export async function suggest(sessions, key, body, opts = {}) {
|
|
|
114
110
|
const picked = anchor.quote; // exactly what the page selected; the file is searched for this
|
|
115
111
|
if (untrusted(picked.trim(), MAX_QUOTE + 100) !== picked.trim()) return no("selection"); // control, bidi or zero-width characters, a line break
|
|
116
112
|
|
|
117
|
-
// where it is in the file on disk now, read only
|
|
118
|
-
// on the bytes actually written, at Use
|
|
119
|
-
|
|
120
|
-
try {
|
|
121
|
-
checkRegularOwned(s.file);
|
|
122
|
-
const units = readUnits(s.format, readDoc(s.file));
|
|
123
|
-
place = placeOf(units, buildDurable(units, anchor), picked);
|
|
124
|
-
} catch {
|
|
125
|
-
place = null;
|
|
126
|
-
}
|
|
127
|
-
if (place?.code === "protected") return { state: "protected" };
|
|
128
|
-
if (place?.code === "not_eligible") return no("place");
|
|
113
|
+
// where it is in the file on disk now, read only, in the anchor worker (src/anchor-work.js; the paragraph's text stays there). A sentence
|
|
114
|
+
// the page changed and has not sent yet is not found: that is checked again, on the bytes actually written, at Use. Work that did not
|
|
115
|
+
// finish reads as not found.
|
|
129
116
|
// The sentence is read from the paragraph itself, never from the page: a selection that is a sentence minus its end mark, closing quote or
|
|
130
117
|
// bracket, or with spaces at its edges, means the whole sentence (src/sentence-span.js). The anchor Use will write on is the sentence's.
|
|
118
|
+
const r = await anchorsOffThread({ format: s.format, source: { path: s.file }, tasks: [{ op: "select", anchor, picked, limits: { context: CONTEXT, maxQuote: MAX_QUOTE } }] });
|
|
119
|
+
const place = r.ok && r.r.read.ok && r.r.results[0] && !r.r.results[0].error ? r.r.results[0] : null;
|
|
120
|
+
if (place?.code === "protected") return { state: "protected" };
|
|
121
|
+
if (place?.code === "not_eligible") return no("place");
|
|
122
|
+
if (place?.code === "selection") return no("selection");
|
|
131
123
|
let quote = picked;
|
|
132
124
|
let from = anchor;
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
quote = place.text.slice(span.start, span.end);
|
|
137
|
-
if (untrusted(quote, MAX_QUOTE + 100) !== quote || quote.length > MAX_QUOTE + 10) return no("selection");
|
|
138
|
-
const loc = Number.isInteger(anchor.location.start) && Number.isInteger(anchor.location.end) ? { ...anchor.location, start: span.start, end: span.end } : anchor.location;
|
|
139
|
-
from = { location: loc, quote, prefix: place.text.slice(Math.max(0, span.start - CONTEXT), span.start), suffix: place.text.slice(span.end, span.end + CONTEXT) };
|
|
140
|
-
}
|
|
141
|
-
} else if (untrusted(picked, MAX_QUOTE + 100) !== picked) return no("selection"); // edge spaces: nothing to read the sentence from
|
|
142
|
-
const wholeParagraph = place?.ok ? normText(place.text) === normText(quote) : body.wholeParagraph === true;
|
|
125
|
+
let wholeParagraph = body.wholeParagraph === true;
|
|
126
|
+
if (place?.ok) ({ quote, from, wholeParagraph } = place);
|
|
127
|
+
else if (untrusted(picked, MAX_QUOTE + 100) !== picked) return no("selection"); // edge spaces: nothing to read the sentence from
|
|
143
128
|
const q = inputQualifies(quote, { wholeParagraph });
|
|
144
129
|
if (!q.ok) return q.reasons.some((r) => r.code === "input_addresses_ai") ? { state: "addresses_ai" } : no("sentence");
|
|
145
130
|
|
|
@@ -192,12 +177,12 @@ export async function suggest(sessions, key, body, opts = {}) {
|
|
|
192
177
|
* body: {id, commentId, comments, note?, baseSha, bytesB64?, edits?}. The record is looked up here by id; Sessions.useSuggestion checks the
|
|
193
178
|
* rest and starts the local run. The record is used once: it is forgotten as soon as the run has started.
|
|
194
179
|
*/
|
|
195
|
-
export function useSuggestion(sessions, key, body) {
|
|
180
|
+
export async function useSuggestion(sessions, key, body) {
|
|
196
181
|
const id = typeof body?.id === "string" ? body.id : "";
|
|
197
182
|
const record = findSuggestion(sessions, key, id);
|
|
198
183
|
if (!record) throw new HttpError(409, "This suggestion is out of date. Tap the stamp again.", { code: "expired" });
|
|
199
184
|
const bytes = typeof body.bytesB64 === "string" ? Buffer.from(body.bytesB64, "base64") : null;
|
|
200
|
-
sessions.
|
|
185
|
+
await sessions.useSuggestionAsync(key, { suggestion: record, commentId: body.commentId, comments: body.comments, note: body.note, baseSha: String(body.baseSha || ""), bytes, edits: body.edits, rev: body.rev, seen: body.seen, noteSeen: body.noteSeen, tab: body.tab });
|
|
201
186
|
forgetSuggestion(sessions, key, id);
|
|
202
187
|
return sessions.get(key);
|
|
203
188
|
}
|
package/src/text.js
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
import crypto from "node:crypto";
|
|
2
|
+
import { nfc } from "./nfc.js";
|
|
2
3
|
|
|
3
|
-
/** NFC, whitespace collapsed, trimmed: the text every anchor, hash and comparison is built on. Null and undefined read as "". */
|
|
4
|
-
export const normText = (s) =>
|
|
4
|
+
/** NFC (src/nfc.js: linear, also on a run of thousands of combining marks), whitespace collapsed, trimmed: the text every anchor, hash and comparison is built on. Null and undefined read as "". */
|
|
5
|
+
export const normText = (s) => nfc(s).replace(/\s+/g, " ").trim();
|
|
5
6
|
|
|
6
7
|
const sha16 = (s) => crypto.createHash("sha256").update(s).digest("hex").slice(0, 16);
|
|
7
8
|
/** 16-hex hash of the normalized text (DOCX paragraphs, PPTX shapes) */
|
|
8
9
|
export const textHash = (s) => sha16(normText(s));
|
|
9
10
|
/** 16-hex hash of the NFC text with whitespace kept (XLSX cells and regions compare character for character) */
|
|
10
|
-
export const exactHash = (s) => sha16(
|
|
11
|
+
export const exactHash = (s) => sha16(nfc(s));
|
|
11
12
|
|
|
12
13
|
function bigrams(s) {
|
|
13
14
|
const m = new Map();
|
|
@@ -17,15 +18,164 @@ function bigrams(s) {
|
|
|
17
18
|
}
|
|
18
19
|
return m;
|
|
19
20
|
}
|
|
21
|
+
/**
|
|
22
|
+
* The character bigrams of a text, normalized and lowercased as similarity() reads it, as a sorted list of codes (two UTF-16 units in
|
|
23
|
+
* one number). Two such lists give the same Dice coefficient as similarity() by one merge, without building a map per comparison.
|
|
24
|
+
*/
|
|
25
|
+
export function bigramCodes(s) {
|
|
26
|
+
const t = normText(s).toLowerCase();
|
|
27
|
+
const out = new Float64Array(Math.max(0, t.length - 1));
|
|
28
|
+
for (let i = 0; i < out.length; i++) out[i] = t.charCodeAt(i) * 65536 + t.charCodeAt(i + 1);
|
|
29
|
+
out.sort();
|
|
30
|
+
return { codes: out, empty: !t, short: t.length < 2 ? t : null };
|
|
31
|
+
}
|
|
32
|
+
/** similarity(a, b) from bigramCodes(a) and bigramCodes(b). */
|
|
33
|
+
export function diceOf(A, B) {
|
|
34
|
+
if (A.empty && B.empty) return 1;
|
|
35
|
+
if (A.short !== null || B.short !== null) return A.short !== null && B.short !== null && A.short === B.short ? 1 : 0;
|
|
36
|
+
const a = A.codes;
|
|
37
|
+
const b = B.codes;
|
|
38
|
+
let i = 0;
|
|
39
|
+
let j = 0;
|
|
40
|
+
let hit = 0;
|
|
41
|
+
while (i < a.length && j < b.length) {
|
|
42
|
+
if (a[i] === b[j]) (hit++, i++, j++);
|
|
43
|
+
else if (a[i] < b[j]) i++;
|
|
44
|
+
else j++;
|
|
45
|
+
}
|
|
46
|
+
return (2 * hit) / (a.length + b.length);
|
|
47
|
+
}
|
|
48
|
+
|
|
20
49
|
/** Dice coefficient over character bigrams of the normalized, lowercased strings (0..1). */
|
|
21
50
|
export function similarity(a, b) {
|
|
22
|
-
|
|
51
|
+
return similarityTo(b)(a);
|
|
52
|
+
}
|
|
53
|
+
/** similarity(a, b) as a function of `a`, with b's bigrams counted once: comparing one text with every paragraph stays linear. */
|
|
54
|
+
export function similarityTo(b) {
|
|
23
55
|
b = normText(b).toLowerCase();
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
56
|
+
let B = null;
|
|
57
|
+
return (a) => {
|
|
58
|
+
a = normText(a).toLowerCase();
|
|
59
|
+
if (!a && !b) return 1;
|
|
60
|
+
if (a.length < 2 || b.length < 2) return a === b ? 1 : 0;
|
|
61
|
+
B ??= bigrams(b);
|
|
62
|
+
const A = bigrams(a);
|
|
63
|
+
let hit = 0;
|
|
64
|
+
for (const [g, c] of A) hit += Math.min(c, B.get(g) || 0);
|
|
65
|
+
return (2 * hit) / (a.length - 1 + b.length - 1);
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// Resolving a moved anchor scans the whole reading (every paragraph's text, every cell of a sheet): its cost is the document once per such
|
|
70
|
+
// anchor. scanBudget(n, fn) bounds the scans of every anchor resolved inside fn together (src/docx.js counts bigrams and paragraphs,
|
|
71
|
+
// src/xlsx.js cells); past it, an anchor its cheap steps do not find resolves as "moved or removed". The server resolves the anchors of
|
|
72
|
+
// one request under one budget (src/session.js).
|
|
73
|
+
// Making an anchor is paid the same way as resolving one: no work runs past the budget "because it must". An anchor whose text
|
|
74
|
+
// cannot be paid for is made without it (src/xlsx.js: the address and its context, no text hash), never by going over.
|
|
75
|
+
let scanLeft = Infinity;
|
|
76
|
+
let watcher = null;
|
|
77
|
+
export function scanBudget(n, fn) {
|
|
78
|
+
const before = scanLeft;
|
|
79
|
+
scanLeft = n;
|
|
80
|
+
try {
|
|
81
|
+
return fn();
|
|
82
|
+
} finally {
|
|
83
|
+
scanLeft = before;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
/** Spends `n` of the current budget; false (and nothing spent) when it does not have that much left. */
|
|
87
|
+
export function spend(n) {
|
|
88
|
+
const ok = scanLeft >= n;
|
|
89
|
+
watcher?.(n, ok, scanLeft === Infinity);
|
|
90
|
+
if (!ok) return false;
|
|
91
|
+
scanLeft -= n;
|
|
92
|
+
return true;
|
|
93
|
+
}
|
|
94
|
+
/** Can the budget pay `n`? Nothing is spent (a text longer than what is left is refused before its weight is counted). */
|
|
95
|
+
export const affords = (n) => scanLeft >= n;
|
|
96
|
+
/** Tests: `fn(n, ok, outside)` sees every spend (`outside`: no scanBudget around it); null stops watching. */
|
|
97
|
+
export const watchBudget = (fn) => (watcher = fn);
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* What reading `s` once costs the budget (normalizing, hashing or encoding it): a unit per character of ASCII text, else 2 units per
|
|
101
|
+
* UTF-8 byte. Measured per million characters here (nfc, then sha256): ASCII 1.2 ms; text with any non-ASCII character, which V8 keeps
|
|
102
|
+
* at two bytes a character, 4.7 ms (é: 4 units), CJK 8 ms (6 units), emoji 7 to 10 ms (4 units a UTF-16 unit). Buffer.byteLength reads
|
|
103
|
+
* 10 million characters in under 2 ms, about a hundredth of the work it prices.
|
|
104
|
+
* Ceiling: text that normalization rewrites (decomposed letters, "e" + U+0301) costs about 30 ms per million and is charged 3 units a
|
|
105
|
+
* character, so a budget's worth of it is slower than a budget's worth of ASCII (40 million units: about 0.4 s against 0.05 s).
|
|
106
|
+
* Upgrade path: count combining marks in the reader's own pass over the text and price them apart.
|
|
107
|
+
*/
|
|
108
|
+
export function weigh(s) {
|
|
109
|
+
const bytes = Buffer.byteLength(s, "utf8");
|
|
110
|
+
return bytes === s.length ? bytes : 2 * bytes;
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* What a native substring search for `q` in `n` characters costs the budget, by what V8 does on the text at hand (measured on 7.9
|
|
114
|
+
* million characters, and on 26,000 paragraphs of English prose, 6.6 million characters). V8 finds candidate places with memchr for
|
|
115
|
+
* one byte of the needle's first character (the character itself in a one-byte string; in a two-byte string, the larger of its two
|
|
116
|
+
* bytes, which also matches characters that only share that byte) and checks the rest at each: for a needle of 2 to 6 characters,
|
|
117
|
+
* 6.4 to 7 ns a candidate on dense text ("az" in 7.9 million "a", "aaz" in "abab...", "丁z" in 7.9 million "中"), 3 to 13 ns on
|
|
118
|
+
* prose (13 ns for sparse candidates). Longer needles switch to a skipping search only once candidates matched far enough; candidates
|
|
119
|
+
* that share only a byte ("丁丁丁丁丁丁丁丁丁z" in "中") never do and cost the same 7 ns each, and so does a one-character needle in a
|
|
120
|
+
* two-byte string ("丁" in "中": 55 ms). A needle whose first two characters match everywhere costs no more per candidate than one
|
|
121
|
+
* whose second does not ("abz" and "az" on "abab...": 6.1 ns). The rest of the text costs memchr's pass, 0.015 to 0.05 ns a character.
|
|
122
|
+
* Charged: a 16th of a unit a character, HIT units a candidate (hitsOf counts them, from a byte table kept per unit) and a unit for
|
|
123
|
+
* the call, whatever the needle's length; `hits` defaults to every character (the worst case) where no table is kept. Ceiling: a
|
|
124
|
+
* whole budget of searching is at most about 0.26 s (40 million / HIT candidates at 13 ns); a long needle on one-byte text that
|
|
125
|
+
* matches everywhere (V8 skips there, 0.5 to 1.1 ns a character) is charged as if it did not skip, about 4 times what it costs.
|
|
126
|
+
* Each occurrence a search lists costs OCCURRENCE.
|
|
127
|
+
*/
|
|
128
|
+
export const HIT = 2;
|
|
129
|
+
export const searchCost = (n, q, hits = n) => (n >> 4) + HIT * hits + 1;
|
|
130
|
+
// The byte table of a unit's text: for every byte value, how many characters hold it as their low byte or as a nonzero high byte (an
|
|
131
|
+
// upper bound on memchr's candidates for any needle in either string width). Kept as [byte * 2^32 + count] for the bytes present.
|
|
132
|
+
const SCRATCH = new Uint32Array(256);
|
|
133
|
+
const TWO32 = 2 ** 32;
|
|
134
|
+
function byteTable(s) {
|
|
135
|
+
const t = SCRATCH;
|
|
136
|
+
for (let i = 0; i < s.length; i++) {
|
|
137
|
+
const c = s.charCodeAt(i);
|
|
138
|
+
const lo = c & 255;
|
|
139
|
+
t[lo]++;
|
|
140
|
+
const hi = c >> 8;
|
|
141
|
+
if (hi && hi !== lo) t[hi]++;
|
|
142
|
+
}
|
|
143
|
+
const out = [];
|
|
144
|
+
for (let b = 0; b < 256; b++) if (t[b]) out.push(b * TWO32 + t[b]), (t[b] = 0);
|
|
145
|
+
return Float64Array.from(out);
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* How many places a native search for `q` in `s` (the text of `unit`, keyed in `cache`, a WeakMap per kind of text) checks, from the
|
|
149
|
+
* unit's byte table (`unit.text` is the text as written; `s` the text searched, the same or its normalized form). The table is built
|
|
150
|
+
* once per unit at 2 units a character (measured 1.75 ns); past the budget, every character.
|
|
151
|
+
*/
|
|
152
|
+
export function hitsOf(cache, unit, s, q) {
|
|
153
|
+
q = String(q ?? "");
|
|
154
|
+
const f = q.charCodeAt(0);
|
|
155
|
+
if (!(f >= 0)) return 0;
|
|
156
|
+
// one character below 256 in ASCII text: memchr looks for the character itself (its other byte is 0) and stops at the first match. Not
|
|
157
|
+
// for a character above: in ASCII text that V8 holds two bytes a character, "š" (0x161) stops at every "a" (0x61).
|
|
158
|
+
if (q.length === 1 && f < 256 && weightOf(unit) === unit.text.length) return 0;
|
|
159
|
+
let t = cache.get(unit);
|
|
160
|
+
if (t === undefined) {
|
|
161
|
+
if (!spend(2 * s.length)) return s.length;
|
|
162
|
+
cache.set(unit, (t = byteTable(s)));
|
|
163
|
+
}
|
|
164
|
+
const b = Math.max(f & 255, f >> 8);
|
|
165
|
+
for (const v of t) if (Math.floor(v / TWO32) === b) return v % TWO32;
|
|
166
|
+
return 0;
|
|
167
|
+
}
|
|
168
|
+
/** what listing one more occurrence costs (measured 27 ns each, with the search that finds it, on 395,000 of them) */
|
|
169
|
+
export const OCCURRENCE = 24;
|
|
170
|
+
/** normText(s), paid from the budget at weigh(s) first; null when it cannot pay (text the page sent, with no reading to keep it on). */
|
|
171
|
+
export function normPaid(s) {
|
|
172
|
+
s = String(s ?? "");
|
|
173
|
+
return affords(s.length) && spend(weigh(s)) ? normText(s) : null;
|
|
174
|
+
}
|
|
175
|
+
// weigh() of a unit's text (a paragraph, a shape), kept per unit of a reading: a text the budget refuses is weighed once, not once per anchor
|
|
176
|
+
const WEIGHT = new WeakMap();
|
|
177
|
+
export function weightOf(unit) {
|
|
178
|
+
let w = WEIGHT.get(unit);
|
|
179
|
+
if (w === undefined) WEIGHT.set(unit, (w = weigh(unit.text)));
|
|
180
|
+
return w;
|
|
31
181
|
}
|