onionskin-cli 0.0.2 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +111 -0
- package/INSTALL.md +5 -1
- package/NOTICE +34 -0
- package/README.md +101 -33
- package/docs/contracts/batch.schema.json +27 -4
- package/docs/contracts/ops.schema.json +117 -5
- package/package.json +4 -2
- package/skills/onionskin/SKILL.md +9 -6
- package/skills/onionskin/reference/batch.md +5 -3
- package/skills/onionskin/reference/formats.md +5 -1
- package/skills/onionskin/reference/ops.md +47 -3
- package/skills/onionskin/reference/troubleshooting.md +31 -0
- package/src/agent-detect.js +28 -2
- package/src/agent-prompt.js +12 -1
- package/src/agent-runner.js +77 -19
- package/src/anchor-work.js +154 -0
- package/src/anchor-worker.js +38 -0
- package/src/anchors.js +61 -11
- package/src/apply.js +119 -47
- package/src/batch.js +32 -8
- package/src/check-worker.js +8 -5
- package/src/cli.js +91 -25
- package/src/constants.js +34 -1
- package/src/diff.js +3 -1
- package/src/docx.js +368 -58
- package/src/edit-hash.js +19 -0
- package/src/fsutil.js +1 -1
- package/src/history-worker.js +8 -8
- package/src/history.js +135 -31
- package/src/html-active.js +384 -0
- package/src/html-envelope.js +51 -0
- package/src/html-parser.js +8 -0
- package/src/html-render-worker.js +25 -0
- package/src/html-render.js +880 -0
- package/src/html-utf8.js +65 -0
- package/src/html-write-worker.js +14 -0
- package/src/html-write.js +380 -0
- package/src/html.js +1257 -0
- package/src/local.js +68 -4
- package/src/merge-package.js +1110 -0
- package/src/merge-pptx.js +98 -38
- package/src/merge-proof.js +1019 -0
- package/src/merge-worker.js +60 -0
- package/src/merge-xlsx.js +84 -29
- package/src/merge.js +286 -83
- package/src/nfc.js +46 -0
- package/src/notes.js +122 -0
- package/src/offload.js +400 -1
- package/src/open-worker.js +27 -0
- package/src/ops/common.js +2 -1
- package/src/ops/docx.js +58 -29
- package/src/ops/html.js +394 -0
- package/src/ops/pptx.js +74 -21
- package/src/ops/run.js +23 -3
- package/src/ops/runs.js +215 -13
- package/src/ops/schema.js +8 -1
- package/src/ops/xlsx.js +12 -7
- package/src/ops-worker.js +17 -0
- package/src/package.js +26 -2
- package/src/person.js +8 -1
- package/src/pptx.js +85 -21
- package/src/quick-plan.js +13 -6
- package/src/redline-worker.js +10 -14
- package/src/reply.js +46 -19
- package/src/retain.js +223 -0
- package/src/review.js +32 -18
- package/src/rewrite-check.js +2 -2
- package/src/sanitize.js +16 -3
- package/src/server.js +88 -37
- package/src/session.js +1143 -208
- package/src/stop-gate.js +34 -0
- package/src/suggest-reply.js +42 -7
- package/src/suggest.js +14 -29
- package/src/text.js +161 -11
- package/src/update.js +288 -0
- package/src/voice-guide.js +4 -3
- package/src/voice-learn.js +4 -3
- package/src/voice-sample-worker.js +7 -6
- package/src/voice-worker.js +4 -6
- package/src/watch.js +188 -0
- package/src/worker-host.js +23 -0
- package/src/xlsx.js +355 -94
- package/src/xml-tree.js +6 -3
- package/src/xml.js +82 -34
- package/src/zip-write.js +2 -1
- package/src/zip.js +9 -5
- package/vendor/parse5/LICENSE.txt +32 -0
- package/vendor/parse5/parse5.mjs +7822 -0
- package/vendor/parse5.lock.json +23 -0
- package/web/dist/assets/index-DkNx0LSH.css +1 -0
- package/web/dist/assets/{index-BUtofdtr.js → index-DytKkMdi.js} +117 -107
- package/web/dist/index.html +2 -2
- package/web/dist/assets/index-Dfizlg0i.css +0 -1
package/src/xlsx.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { colLetters, parseA1 } from "./ops/common.js";
|
|
2
2
|
import { untrusted } from "./sanitize.js";
|
|
3
|
-
import
|
|
4
|
-
import {
|
|
3
|
+
import crypto from "node:crypto";
|
|
4
|
+
import { nfc } from "./nfc.js";
|
|
5
|
+
import { exactHash, normText, spend, weigh } from "./text.js";
|
|
6
|
+
import { CapError, readBudget, tooMany, walkXml } from "./xml.js";
|
|
5
7
|
import { openZip } from "./zip.js";
|
|
6
8
|
|
|
7
9
|
// XLSX durable anchors. Everything is read from the bytes with the small XML walker (no engine). Names are matched by local name,
|
|
@@ -20,8 +22,53 @@ const EMPTY = exactHash("");
|
|
|
20
22
|
/** hash of a parsed cell's text, computed once per cell (resolution and the diff would otherwise hash every cell per anchor) */
|
|
21
23
|
export const ch = (cell) => (cell.h ??= exactHash(cell.text));
|
|
22
24
|
const short = (s, n = 80) => normText(s).slice(0, n);
|
|
23
|
-
|
|
25
|
+
// The request's budget (src/text.js scanBudget) pays for every character hashed or cleaned, at the cell, the first time (ch and sh keep
|
|
26
|
+
// the result on the cell), and an anchor being made pays like one being resolved: a text the budget cannot pay for is not read (no hash,
|
|
27
|
+
// no header). An anchor made without its hash still goes out with its address and the context that was paid for (xlsxDurable); a hit
|
|
28
|
+
// whose text cannot be paid for while resolving is not one ("moved or removed" when nothing else places it). The cost of a cell's text
|
|
29
|
+
// is weigh() (src/text.js), kept on the cell: non-ASCII text costs the CPU about four times what ASCII does. The reviewer's case: 50
|
|
30
|
+
// anchors on A1:B1 through A1:AY1 of a cell of 31 million "é" each joined and hashed the cell, 6.6 s of CPU, while an anchor being made
|
|
31
|
+
// was charged whatever was left and went on.
|
|
32
|
+
const costOf = (cell) => (cell.w ??= weigh(cell.text));
|
|
33
|
+
const spent = (cell) => spend(costOf(cell));
|
|
34
|
+
/** short() of a parsed cell's text (or "" for no cell, or past the budget), computed once per cell like ch(): resolving one anchor compares
|
|
35
|
+
* the same header or neighbor with every hit, and normalizing a 20,000-character header per hit held the server 4 s on a column of 248,000 equal cells */
|
|
36
|
+
const sh = (cell) => (!cell ? "" : cell.sh ?? (spent(cell) ? (cell.sh = short(cell.text)) : ""));
|
|
37
|
+
/** ch(cell), paid from the budget when the cell's text is not hashed yet; null past it */
|
|
38
|
+
const chPaid = (cell) => (cell.h !== undefined || spent(cell) ? ch(cell) : null);
|
|
39
|
+
// What one reading of a workbook may hold, over every sheet it reads. The server reads a workbook on its own thread at Send, at each delivery
|
|
40
|
+
// to an agent, for a review and at the end of a run, so these bound how long that keeps it from answering (outside review: a sheet of
|
|
41
|
+
// 1.5 million cells held it 1.6 s; 100,000 merged ranges made a review take 18 s). Cells are counted as written (<c> elements), empty or
|
|
42
|
+
// not. A file past a cap is refused with the reason; it is never read in part. Set from measurement (docs/GUIDE.md "Size limits"), at a
|
|
43
|
+
// load average of 13 to 21 here: a plain workbook of 999,000 cells (10 to a row, half numbers, half short shared strings) kept /health
|
|
44
|
+
// waiting at most 0.74 to 0.83 s (the Send) and the server at 1.5 GB; with 30 more characters in each text, 0.86 s and 1.6 GB; 999,000
|
|
45
|
+
// cells on one shared string, with 49,000 merged ranges, or 100 number cells to a row, 0.57 to 0.65 s. 0.0.3 took 3.1 s at 1,000,000.
|
|
46
|
+
// A higher cap needs the Send's whole reading moved into a worker (the Send is the longest step, and the memory grows with it).
|
|
47
|
+
export const MAX_CELLS = 1_000_000;
|
|
48
|
+
export const MAX_MERGES = 50_000;
|
|
49
|
+
export const MAX_SHARED_STRINGS = 1_000_000;
|
|
50
|
+
export const MAX_NAMES = 20_000;
|
|
51
|
+
export const MAX_SHEETS = 1000;
|
|
52
|
+
// Characters of a sheet's name. Excel allows 31; every cell of the sheet carries the name in a review's and an agent's outline (a name of
|
|
53
|
+
// 100,000 characters made an outline of 2,000 cells take 1.8 s on the server thread).
|
|
54
|
+
export const MAX_SHEET_NAME = 255;
|
|
55
|
+
export const MAX_BOOK_ELEMENTS = 3_200_000;
|
|
56
|
+
export const MAX_BOOK_ATTRIBUTES = 6_400_000;
|
|
57
|
+
export const MAX_BOOK_TEXT = 32_000_000;
|
|
58
|
+
// Characters of XML one reading of a workbook may walk (other files: MAX_READ_CHARS, 32 million). A plain workbook of 999,000 cells with
|
|
59
|
+
// half of them short texts is about 48 to 62 million; two sheets of one 31-million-character attribute each (the worst shape) held /health 0.43 s.
|
|
60
|
+
export const MAX_BOOK_CHARS = 64_000_000;
|
|
61
|
+
// Characters of text the cells take from the shared strings, counted per cell as the cell is read. A shared string's text is counted once
|
|
62
|
+
// in MAX_BOOK_TEXT, but every cell that points at it is a cell of that text for the readers after this one (a review's outline hashes and
|
|
63
|
+
// cleans each cell's text): 249,000 cells pointing at one string of 20,000 characters (a 517 KB file within every other cap) held the
|
|
64
|
+
// server 4 to 6 s per review, delivery and done. At this cap a reading's cells hold at most 32 million characters from shared strings
|
|
65
|
+
// (plus at most MAX_BOOK_TEXT of their own). Measured per review or outline on the server thread, at a load average of 16: 249,000 cells
|
|
66
|
+
// of 128 characters about 170 ms; 4 cells of 7.9 million characters (the worst shape: each unit's text is cleaned whole) about 175 ms;
|
|
67
|
+
// 8 such cells (64 million) 350 ms.
|
|
68
|
+
export const MAX_SHARED_CELL_TEXT = 32_000_000;
|
|
69
|
+
const bookBudget = () => Object.assign(readBudget({ elements: MAX_BOOK_ELEMENTS, attrs: MAX_BOOK_ATTRIBUTES, text: MAX_BOOK_TEXT, chars: MAX_BOOK_CHARS, what: "workbook", split: "smaller workbooks" }), { cells: 0, merges: 0, sharedText: MAX_SHARED_CELL_TEXT });
|
|
24
70
|
const KEEP_ROW_CELLS = 64;
|
|
71
|
+
const SAME_TEXT_WORK = 4_000_000;
|
|
25
72
|
|
|
26
73
|
const relTarget = (base, target) => {
|
|
27
74
|
if (target.startsWith("/")) return target.slice(1);
|
|
@@ -33,7 +80,7 @@ const relTarget = (base, target) => {
|
|
|
33
80
|
return parts.join("/");
|
|
34
81
|
};
|
|
35
82
|
|
|
36
|
-
export function readRels(zip, part) {
|
|
83
|
+
export function readRels(zip, part, budget = null) {
|
|
37
84
|
const dir = part.split("/").slice(0, -1).join("/");
|
|
38
85
|
const rels = new Map();
|
|
39
86
|
const xml = zip.readText(`${dir ? `${dir}/` : ""}_rels/${part.split("/").pop()}.rels`);
|
|
@@ -42,7 +89,7 @@ export function readRels(zip, part) {
|
|
|
42
89
|
onOpen(name, a) {
|
|
43
90
|
if (local(name) === "Relationship" && a.Id && a.Target) rels.set(a.Id, relTarget(part, a.Target));
|
|
44
91
|
},
|
|
45
|
-
});
|
|
92
|
+
}, budget);
|
|
46
93
|
return rels;
|
|
47
94
|
}
|
|
48
95
|
export { attrOf, local };
|
|
@@ -52,14 +99,18 @@ export function parseWorkbook(bytes) {
|
|
|
52
99
|
const zip = openZip(bytes);
|
|
53
100
|
const xml = zip.readText("xl/workbook.xml");
|
|
54
101
|
if (xml == null) throw new Error("xl/workbook.xml missing");
|
|
55
|
-
const
|
|
102
|
+
const budget = bookBudget(); // the workbook part, its relationships, shared strings, styles and every sheet read share it
|
|
103
|
+
const rels = readRels(zip, "xl/workbook.xml", budget);
|
|
56
104
|
const sheets = [];
|
|
57
105
|
const names = [];
|
|
58
106
|
let curName = null;
|
|
59
107
|
walkXml(xml, {
|
|
60
108
|
onOpen(n, a) {
|
|
61
109
|
const l = local(n);
|
|
62
|
-
if (l === "sheet"
|
|
110
|
+
if (l === "sheet" && sheets.length >= MAX_SHEETS) throw tooMany(budget, MAX_SHEETS, "sheets");
|
|
111
|
+
if (l === "definedName" && names.length >= MAX_NAMES) throw tooMany(budget, MAX_NAMES, "defined names");
|
|
112
|
+
if (l === "sheet" && String(a.name ?? "").length > MAX_SHEET_NAME) throw new CapError(`a sheet in this workbook has a name longer than ${MAX_SHEET_NAME} characters, more than Onionskin can read. Rename it in Excel, which allows 31, and open it again`, "sheet name");
|
|
113
|
+
if (l === "sheet") sheets.push({ index: sheets.length, name: a.name ?? "", sheetId: a.sheetId ?? null, hidden: a.state === "hidden" || a.state === "veryHidden", state: a.state ?? "visible", part: rels.get(attrOf(a, "id")) || null });
|
|
63
114
|
else if (l === "definedName") curName = { name: a.name, ref: "" };
|
|
64
115
|
},
|
|
65
116
|
onText(t) {
|
|
@@ -71,11 +122,25 @@ export function parseWorkbook(bytes) {
|
|
|
71
122
|
curName = null;
|
|
72
123
|
}
|
|
73
124
|
},
|
|
74
|
-
});
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
125
|
+
}, budget);
|
|
126
|
+
// the shared strings and the number formats are read on first use, once; a reading that failed (past a cap) fails the same way
|
|
127
|
+
// every time (a half-read list kept after the first failure was accepted on the next call)
|
|
128
|
+
const once = (read) => {
|
|
129
|
+
let value = null;
|
|
130
|
+
let failed = null;
|
|
131
|
+
return (...args) => {
|
|
132
|
+
if (failed) throw failed;
|
|
133
|
+
if (value) return value;
|
|
134
|
+
try {
|
|
135
|
+
return (value = read(...args));
|
|
136
|
+
} catch (err) {
|
|
137
|
+
failed = err;
|
|
138
|
+
throw err;
|
|
139
|
+
}
|
|
140
|
+
};
|
|
141
|
+
};
|
|
142
|
+
const sharedStrings = once(() => {
|
|
143
|
+
const sst = [];
|
|
79
144
|
const sx = zip.readText("xl/sharedStrings.xml");
|
|
80
145
|
if (sx == null) return sst;
|
|
81
146
|
let cur = null;
|
|
@@ -96,38 +161,40 @@ export function parseWorkbook(bytes) {
|
|
|
96
161
|
if (l === "t") inT = false;
|
|
97
162
|
else if (l === "rPh") phon = Math.max(0, phon - 1);
|
|
98
163
|
else if (l === "si") {
|
|
164
|
+
if (sst.length >= MAX_SHARED_STRINGS) throw tooMany(budget, MAX_SHARED_STRINGS, "shared strings");
|
|
99
165
|
sst.push(cur ?? "");
|
|
100
166
|
cur = null;
|
|
101
167
|
}
|
|
102
168
|
},
|
|
103
|
-
});
|
|
169
|
+
}, budget);
|
|
104
170
|
return sst;
|
|
105
|
-
};
|
|
171
|
+
});
|
|
106
172
|
const cache = new Map();
|
|
107
173
|
const sheetCells = (s) => {
|
|
108
|
-
if (!cache.has(s.index)) cache.set(s.index, readSheet(zip, s, sharedStrings));
|
|
174
|
+
if (!cache.has(s.index)) cache.set(s.index, readSheet(zip, s, sharedStrings, budget));
|
|
109
175
|
return cache.get(s.index);
|
|
110
176
|
};
|
|
111
177
|
// number formats: the style index of a cell -> its format code (xl/styles.xml cellXfs + numFmts, with the built-in ids)
|
|
112
|
-
|
|
178
|
+
const formats = once(() => {
|
|
179
|
+
const fmts = { custom: new Map(), xf: [] };
|
|
180
|
+
const sx = zip.readText("xl/styles.xml");
|
|
181
|
+
let inXfs = false;
|
|
182
|
+
if (sx != null)
|
|
183
|
+
walkXml(sx, {
|
|
184
|
+
onOpen(n, a) {
|
|
185
|
+
const l = local(n);
|
|
186
|
+
if (l === "numFmt" && a.numFmtId != null) fmts.custom.set(Number(a.numFmtId), a.formatCode ?? "");
|
|
187
|
+
else if (l === "cellXfs") inXfs = true;
|
|
188
|
+
else if (l === "xf" && inXfs) fmts.xf.push(Number(a.numFmtId) || 0);
|
|
189
|
+
},
|
|
190
|
+
onClose(n) {
|
|
191
|
+
if (local(n) === "cellXfs") inXfs = false;
|
|
192
|
+
},
|
|
193
|
+
}, budget);
|
|
194
|
+
return fmts;
|
|
195
|
+
});
|
|
113
196
|
const numberFormat = (styleIdx) => {
|
|
114
|
-
|
|
115
|
-
fmts = { custom: new Map(), xf: [] };
|
|
116
|
-
const sx = zip.readText("xl/styles.xml");
|
|
117
|
-
let inXfs = false;
|
|
118
|
-
if (sx != null)
|
|
119
|
-
walkXml(sx, {
|
|
120
|
-
onOpen(n, a) {
|
|
121
|
-
const l = local(n);
|
|
122
|
-
if (l === "numFmt" && a.numFmtId != null) fmts.custom.set(Number(a.numFmtId), a.formatCode ?? "");
|
|
123
|
-
else if (l === "cellXfs") inXfs = true;
|
|
124
|
-
else if (l === "xf" && inXfs) fmts.xf.push(Number(a.numFmtId) || 0);
|
|
125
|
-
},
|
|
126
|
-
onClose(n) {
|
|
127
|
-
if (local(n) === "cellXfs") inXfs = false;
|
|
128
|
-
},
|
|
129
|
-
});
|
|
130
|
-
}
|
|
197
|
+
const fmts = formats();
|
|
131
198
|
const id = fmts.xf[styleIdx] ?? 0;
|
|
132
199
|
return fmts.custom.get(id) ?? BUILTIN_FORMATS[id] ?? (id ? `format ${id}` : "General");
|
|
133
200
|
};
|
|
@@ -139,7 +206,7 @@ const BUILTIN_FORMATS = { 0: "General", 1: "0", 2: "0.00", 3: "#,##0", 4: "#,##0
|
|
|
139
206
|
const key = (r, c) => r * 16384 + c;
|
|
140
207
|
|
|
141
208
|
/** {cells: Map(r*16384+c -> {r,c,text}), rows: Map(r -> cells sorted by column), maxR, maxC, truncated, merges: [{r0,c0,r1,c1}]}. Empty cells are not kept. */
|
|
142
|
-
function readSheet(zip, sheet, sharedStrings) {
|
|
209
|
+
function readSheet(zip, sheet, sharedStrings, budget = bookBudget()) {
|
|
143
210
|
const cells = new Map();
|
|
144
211
|
const rows = new Map();
|
|
145
212
|
const out = { cells, rows, maxR: -1, maxC: -1, truncated: false, merges: [] };
|
|
@@ -153,9 +220,11 @@ function readSheet(zip, sheet, sharedStrings) {
|
|
|
153
220
|
onOpen(n, a) {
|
|
154
221
|
const l = local(n);
|
|
155
222
|
if (l === "mergeCell") {
|
|
223
|
+
if (++budget.merges > MAX_MERGES) throw tooMany(budget, MAX_MERGES, "merged ranges");
|
|
156
224
|
const q = parseA1(a.ref ?? "");
|
|
157
225
|
if (q && (q.r1 > q.r0 || q.c1 > q.c0)) out.merges.push(q);
|
|
158
|
-
} else if (l === "c" &&
|
|
226
|
+
} else if (l === "c" && ++budget.cells > MAX_CELLS) throw tooMany(budget, MAX_CELLS, "cells");
|
|
227
|
+
else if (l === "c" && a.r) {
|
|
159
228
|
const m = /^([A-Z]{1,3})(\d+)$/.exec(a.r);
|
|
160
229
|
cur = m ? { c: [...m[1]].reduce((x, ch) => x * 26 + ch.charCodeAt(0) - 64, 0) - 1, r: Number(m[2]) - 1, t: a.t || "n", s: Number(a.s) || 0, v: "", f: null, is: "" } : null;
|
|
161
230
|
} else if (!cur) return;
|
|
@@ -176,28 +245,36 @@ function readSheet(zip, sheet, sharedStrings) {
|
|
|
176
245
|
else if (l === "is") inIs = false;
|
|
177
246
|
else if (l === "rPh") phon = Math.max(0, phon - 1);
|
|
178
247
|
else if (l === "c" && cur) {
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
248
|
+
let text;
|
|
249
|
+
if (cur.f) text = `=${cur.f}`;
|
|
250
|
+
else if (cur.t === "s") {
|
|
251
|
+
text = sharedStrings()[Number(cur.v)] ?? "";
|
|
252
|
+
if ((budget.sharedText -= text.length) < 0) throw tooMany(budget, MAX_SHARED_CELL_TEXT, "characters of text in its cells");
|
|
253
|
+
}
|
|
254
|
+
else if (cur.t === "inlineStr") text = cur.is;
|
|
255
|
+
else if (cur.t === "b") text = cur.v === "1" ? "TRUE" : "FALSE";
|
|
256
|
+
else if (cur.t === "n" && NUM.test(cur.v)) text = String(Number(cur.v));
|
|
257
|
+
else text = cur.v;
|
|
258
|
+
if (text !== "" && cur.c < 16384) {
|
|
259
|
+
const cell = { r: cur.r, c: cur.c, text, k: cur.f ? "formula" : cur.t === "s" || cur.t === "inlineStr" || cur.t === "str" ? "text" : cur.t === "b" ? "boolean" : cur.t === "e" ? "error" : "number", s: cur.s, ...(cur.f ? { cached: cur.v } : cur.f === "" ? { shared: true } : {}) }; // `cached`: the value a formula last computed (History shows it next to the formula)
|
|
260
|
+
const prev = cells.get(key(cur.r, cur.c));
|
|
261
|
+
if (prev) {
|
|
262
|
+
// a cell written twice: the later one counts, and its row lists it once
|
|
263
|
+
for (const k of Object.keys(prev)) delete prev[k];
|
|
264
|
+
Object.assign(prev, cell);
|
|
265
|
+
}
|
|
266
|
+
else {
|
|
190
267
|
cells.set(key(cur.r, cur.c), cell);
|
|
191
268
|
if (!rows.has(cur.r)) rows.set(cur.r, []);
|
|
192
269
|
rows.get(cur.r).push(cell);
|
|
193
|
-
if (cur.r > out.maxR) out.maxR = cur.r;
|
|
194
|
-
if (cur.c > out.maxC) out.maxC = cur.c;
|
|
195
270
|
}
|
|
271
|
+
if (cur.r > out.maxR) out.maxR = cur.r;
|
|
272
|
+
if (cur.c > out.maxC) out.maxC = cur.c;
|
|
196
273
|
}
|
|
197
274
|
cur = null;
|
|
198
275
|
}
|
|
199
276
|
},
|
|
200
|
-
});
|
|
277
|
+
}, budget);
|
|
201
278
|
for (const list of rows.values()) list.sort((a, b) => a.c - b.c);
|
|
202
279
|
return out;
|
|
203
280
|
}
|
|
@@ -207,6 +284,21 @@ function readSheet(zip, sheet, sharedStrings) {
|
|
|
207
284
|
export const a1 = (r, c) => `${colLetters(c)}${r + 1}`;
|
|
208
285
|
export const a1Range = (q) => (q.r0 === q.r1 && q.c0 === q.c1 ? a1(q.r0, q.c0) : `${a1(q.r0, q.c0)}:${a1(q.r1, q.c1)}`);
|
|
209
286
|
const at = (sd, r, c) => (r < 0 || c < 0 ? "" : sd.cells.get(key(r, c))?.text ?? "");
|
|
287
|
+
/** the hash of the text at (r, c) (EMPTY for no cell), paid as chPaid pays it; null past the budget */
|
|
288
|
+
const hashAt = (sd, r, c) => {
|
|
289
|
+
const cell = r < 0 || c < 0 ? undefined : sd.cells.get(key(r, c));
|
|
290
|
+
return cell ? chPaid(cell) : EMPTY;
|
|
291
|
+
};
|
|
292
|
+
/** The parsed cell at row r, column c (0-based) of a sheet's cells (sheetCells), or undefined. */
|
|
293
|
+
export const cellAt = (sd, r, c) => sd.cells.get(key(r, c));
|
|
294
|
+
/** Merged ranges of a sheet by their top-left cell, built once per sheet: a lookup per cell, never a scan of every range. */
|
|
295
|
+
export function mergeAt(sd, r, c) {
|
|
296
|
+
if (!sd.mergeIndex) {
|
|
297
|
+
sd.mergeIndex = new Map();
|
|
298
|
+
for (const q of sd.merges) if (!sd.mergeIndex.has(key(q.r0, q.c0))) sd.mergeIndex.set(key(q.r0, q.c0), q); // the first, as a scan finds it
|
|
299
|
+
}
|
|
300
|
+
return sd.mergeIndex.get(key(r, c));
|
|
301
|
+
}
|
|
210
302
|
|
|
211
303
|
/** the text of a rectangle as `apply` expects it: cells joined by TAB, rows by LF (bounded) */
|
|
212
304
|
export function rangeText(sd, q) {
|
|
@@ -219,6 +311,96 @@ export function rangeText(sd, q) {
|
|
|
219
311
|
return rows.join("\n");
|
|
220
312
|
}
|
|
221
313
|
|
|
314
|
+
// What an anchor keeps of a range's text: the batch quotes at most 500 characters of it and shows at most 20 x 10 of its cells, 200
|
|
315
|
+
// characters each (src/batch.js). An anchor kept the whole text: 50 anchors on ranges of a cell of 31 million characters held 50 copies
|
|
316
|
+
// of it (the server ran out of memory), and each was saved with the session and cleaned again for the batch.
|
|
317
|
+
const CELL_KEEP = 1000;
|
|
318
|
+
const TEXT_KEEP = 4000;
|
|
319
|
+
const VALUES = { rows: 20, cols: 10 };
|
|
320
|
+
const cut = (s) => (s.length > CELL_KEEP ? s.slice(0, CELL_KEEP) : s);
|
|
321
|
+
const lastRow = (q) => Math.min(q.r1, q.r0 + 2000);
|
|
322
|
+
const lastCol = (q) => Math.min(q.c1, q.c0 + 200);
|
|
323
|
+
|
|
324
|
+
/**
|
|
325
|
+
* A range as an anchor keeps it, per range on the sheet's reading: `text` (rangeText with each cell cut at CELL_KEEP characters, at most
|
|
326
|
+
* TEXT_KEEP in all), `values` (its top-left VALUES.rows x VALUES.cols cells, cut the same way) and `hash`, the hash of the whole rangeText
|
|
327
|
+
* (what `apply` checks an `expect.text_hash` against). Paid before the work: WALK_CELL units a cell to walk it, then HASH_CELL a cell and
|
|
328
|
+
* weigh() of every cell's text for the hash (measured on 2,000 x 201 cells of "a": the walk 50 to 65 ns a cell, the hash 125; charged
|
|
329
|
+
* at 3 and 1 units a cell, 500 comments on such ranges held a Send 3.1 s). Priced as a search is (src/text.js HIT, about 6 ns a unit):
|
|
330
|
+
* a budget of range work is about 0.25 s. Past it an anchor goes out without its range hash (the agent is offered full ops for it).
|
|
331
|
+
* Null when the walk cannot be paid; `hash` null when the hash cannot (asked again by the next request, on a fresh budget).
|
|
332
|
+
* The hash is fed in chunks of whole cells, not a call per cell: 1,000 x 20 cells of ordinary values (about 10 characters) took 125 to
|
|
333
|
+
* 140 ns a cell fed one by one and 25 to 35 ns in chunks (on "a": 100 and 25), so HASH_CELL is 5 and the text is paid by its weight.
|
|
334
|
+
* Ordinary ranges cost about 25 units a cell: 50 of 1,000 x 20 cells take 25 million units (40.25 million at 20 a cell, and the 50th
|
|
335
|
+
* went out without its hash). Ceiling: cells averaging past about 25 ASCII characters put 50 such ranges over the budget again.
|
|
336
|
+
* NFC of the whole is the NFC of each cell, since a tab or a line break never composes or reorders with what is next to it (src/nfc.js
|
|
337
|
+
* runs of marks end at either too), so a chunk ends after any cell. 50 distinct ranges on one large cell cost 50 hashes of
|
|
338
|
+
* it, which the budget pays for or refuses; the cell's own hash (one-cell ranges, the top-left identity) is kept on the cell.
|
|
339
|
+
*/
|
|
340
|
+
const WALK_CELL = 10;
|
|
341
|
+
const HASH_CELL = 5;
|
|
342
|
+
const CHUNK = 1 << 16;
|
|
343
|
+
function rangeOf(sd, q) {
|
|
344
|
+
const k = `${q.r0},${q.c0},${q.r1},${q.c1}`;
|
|
345
|
+
const memo = (sd.ranges ||= new Map()); // Ceiling: one entry per range an anchor names, so at most the anchors of the reading
|
|
346
|
+
let v = memo.get(k);
|
|
347
|
+
const one = q.r0 === q.r1 && q.c0 === q.c1;
|
|
348
|
+
if (!v) {
|
|
349
|
+
const r1 = lastRow(q);
|
|
350
|
+
const c1 = lastCol(q);
|
|
351
|
+
if (!spend(WALK_CELL * (r1 - q.r0 + 1) * (c1 - q.c0 + 1))) return null;
|
|
352
|
+
let text = "";
|
|
353
|
+
let cost = 0;
|
|
354
|
+
const values = [];
|
|
355
|
+
for (let r = q.r0; r <= r1; r++) {
|
|
356
|
+
const row = r - q.r0 < VALUES.rows ? [] : null;
|
|
357
|
+
for (let c = q.c0; c <= c1; c++) {
|
|
358
|
+
const cell = sd.cells.get(key(r, c));
|
|
359
|
+
const t = cell ? cell.text : "";
|
|
360
|
+
if (cell) cost += costOf(cell);
|
|
361
|
+
cost += HASH_CELL;
|
|
362
|
+
if (row && c - q.c0 < VALUES.cols) row.push(cut(t));
|
|
363
|
+
if (text.length < TEXT_KEEP) text += (c > q.c0 ? "\t" : r > q.r0 ? "\n" : "") + cut(t);
|
|
364
|
+
}
|
|
365
|
+
if (row) values.push(row);
|
|
366
|
+
}
|
|
367
|
+
memo.set(k, (v = { text: text.length > TEXT_KEEP ? text.slice(0, TEXT_KEEP) : text, values, hash: null, cost }));
|
|
368
|
+
}
|
|
369
|
+
if (v.hash === null) v.hash = one ? hashAt(sd, q.r0, q.c0) : spend(v.cost) ? streamHash(sd, q) : null;
|
|
370
|
+
return v;
|
|
371
|
+
}
|
|
372
|
+
/** exactHash(rangeText(sd, q)), fed in chunks that end after a whole cell */
|
|
373
|
+
function streamHash(sd, q) {
|
|
374
|
+
const h = crypto.createHash("sha256");
|
|
375
|
+
let buf = "";
|
|
376
|
+
for (let r = q.r0; r <= lastRow(q); r++) {
|
|
377
|
+
if (r > q.r0) buf += "\n";
|
|
378
|
+
for (let c = q.c0; c <= lastCol(q); c++) {
|
|
379
|
+
if (c > q.c0) buf += "\t";
|
|
380
|
+
buf += at(sd, r, c);
|
|
381
|
+
if (buf.length >= CHUNK) (h.update(nfc(buf)), (buf = ""));
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
h.update(nfc(buf));
|
|
385
|
+
return h.digest("hex").slice(0, 16);
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
// A number format as an anchor keeps it: its first FORMAT_KEEP characters (the batch shows 100), kept per style of a reading. An outside
|
|
389
|
+
// review: a custom format of 1,000 decomposed letters on every cell of A1:J20 was kept whole per cell of each of 500 anchors (323 MB).
|
|
390
|
+
// numberFormat() itself stays whole: the merge compares formats.
|
|
391
|
+
const FORMAT_KEEP = 100;
|
|
392
|
+
const FORMATS = new WeakMap();
|
|
393
|
+
function formatOf(wb, s) {
|
|
394
|
+
let m = FORMATS.get(wb);
|
|
395
|
+
if (!m) FORMATS.set(wb, (m = new Map()));
|
|
396
|
+
let v = m.get(s);
|
|
397
|
+
if (v === undefined) {
|
|
398
|
+
v = wb.numberFormat(s);
|
|
399
|
+
if (v.length > FORMAT_KEEP) v = v.slice(0, FORMAT_KEEP);
|
|
400
|
+
m.set(s, v);
|
|
401
|
+
}
|
|
402
|
+
return v;
|
|
403
|
+
}
|
|
222
404
|
/** Per cell of a range (at most 20 x 10, like the evidence `values`): its kind (number, text, formula, boolean, error, empty) and number format. */
|
|
223
405
|
function cellMeta(wb, sd, q) {
|
|
224
406
|
const types = [];
|
|
@@ -229,7 +411,7 @@ function cellMeta(wb, sd, q) {
|
|
|
229
411
|
for (let c = q.c0; c <= Math.min(q.c1, q.c0 + 9); c++) {
|
|
230
412
|
const cell = sd.cells.get(key(r, c));
|
|
231
413
|
t.push(cell ? cell.k : "empty");
|
|
232
|
-
f.push(cell ? wb
|
|
414
|
+
f.push(cell ? formatOf(wb, cell.s) : "General");
|
|
233
415
|
}
|
|
234
416
|
types.push(t);
|
|
235
417
|
formats.push(f);
|
|
@@ -239,29 +421,37 @@ function cellMeta(wb, sd, q) {
|
|
|
239
421
|
|
|
240
422
|
/** The nearest non-empty cell above `r` in column `c`: the column header (the topmost one, which is the header row of a table). */
|
|
241
423
|
function colHeaderAt(sd, r, c) {
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
424
|
+
// the first cell of each column (only cells with text are kept), found once per sheet: a scan up the column per call was quadratic
|
|
425
|
+
if (!sd.colFirst) {
|
|
426
|
+
sd.colFirst = new Map();
|
|
427
|
+
for (const cell of sd.cells.values()) {
|
|
428
|
+
const f = sd.colFirst.get(cell.c);
|
|
429
|
+
if (!f || cell.r < f.r) sd.colFirst.set(cell.c, cell);
|
|
430
|
+
}
|
|
245
431
|
}
|
|
246
|
-
|
|
432
|
+
const f = sd.colFirst.get(c);
|
|
433
|
+
return f && f.r < r ? sh(f) : "";
|
|
247
434
|
}
|
|
248
435
|
|
|
249
436
|
/** What surrounds a range: neighbors, the row's other cells (hashes), the row's leftmost cell, the column's header. */
|
|
250
437
|
function contextOf(sd, q) {
|
|
251
438
|
const rowCells = [];
|
|
252
|
-
let rowHeader =
|
|
253
|
-
|
|
439
|
+
let rowHeader = null;
|
|
440
|
+
const row = sd.rows.get(q.r0) || [];
|
|
441
|
+
for (const cell of spend(row.length) ? row : []) { // a row read from the request's budget (src/text.js scanBudget); past it the row's context is not kept
|
|
254
442
|
if (cell.c >= q.c0 && cell.c <= q.c1) continue;
|
|
255
|
-
if (!rowHeader && cell.c < q.c0) rowHeader = cell
|
|
256
|
-
if (rowCells.length
|
|
443
|
+
if (!rowHeader && cell.c < q.c0 && cell.text) rowHeader = cell;
|
|
444
|
+
if (rowCells.length >= KEEP_ROW_CELLS) continue;
|
|
445
|
+
const h = chPaid(cell); // past the budget the cell is left out of what the row is remembered by
|
|
446
|
+
if (h !== null) rowCells.push(h);
|
|
257
447
|
}
|
|
258
448
|
return {
|
|
259
|
-
rowHeader:
|
|
449
|
+
rowHeader: sh(rowHeader),
|
|
260
450
|
colHeader: colHeaderAt(sd, q.r0, q.c0),
|
|
261
|
-
left:
|
|
262
|
-
right:
|
|
263
|
-
up:
|
|
264
|
-
down:
|
|
451
|
+
left: sh(cellAt(sd, q.r0, q.c0 - 1)),
|
|
452
|
+
right: sh(cellAt(sd, q.r0, q.c1 + 1)),
|
|
453
|
+
up: sh(cellAt(sd, q.r0 - 1, q.c0)),
|
|
454
|
+
down: sh(cellAt(sd, q.r1 + 1, q.c0)),
|
|
265
455
|
rowCells: rowCells.sort(),
|
|
266
456
|
};
|
|
267
457
|
}
|
|
@@ -272,7 +462,8 @@ function rowOverlap(sd, ctx, r, c0, c1) {
|
|
|
272
462
|
const have = new Map();
|
|
273
463
|
for (const cell of sd.rows.get(r) || []) {
|
|
274
464
|
if (cell.c >= c0 && cell.c <= c1) continue;
|
|
275
|
-
const h =
|
|
465
|
+
const h = chPaid(cell);
|
|
466
|
+
if (h === null) continue;
|
|
276
467
|
have.set(h, (have.get(h) || 0) + 1);
|
|
277
468
|
}
|
|
278
469
|
let hit = 0;
|
|
@@ -286,12 +477,20 @@ function rowOverlap(sd, ctx, r, c0, c1) {
|
|
|
286
477
|
}
|
|
287
478
|
const hasRowInfo = (ctx) => ctx.rowCells.length > 0 || !!ctx.rowHeader;
|
|
288
479
|
const firstOutside = (sd, r, c0, c1) => (sd.rows.get(r) || []).find((x) => x.c < c0 || x.c > c1);
|
|
289
|
-
/**
|
|
290
|
-
|
|
480
|
+
/**
|
|
481
|
+
* row `r` looks like the remembered row: most of its other cells are there, or its leftmost cell is the same. Reading the row is spent
|
|
482
|
+
* from the request's budget (src/text.js scanBudget), at a cell a unit and again for the leftmost one; past it the row does not look
|
|
483
|
+
* like it (so the anchor resolves as "moved or removed"). `charge` false when the caller spent for its rows already.
|
|
484
|
+
*/
|
|
485
|
+
const sameRow = (sd, ctx, r, c0, c1, charge = true) => {
|
|
486
|
+
if (charge && !spend(2 * (sd.rows.get(r)?.length || 0) + 1)) return false;
|
|
487
|
+
return rowOverlap(sd, ctx, r, c0, c1) >= 0.6 || (!!ctx.rowHeader && sh(firstOutside(sd, r, c0, c1)) === ctx.rowHeader);
|
|
488
|
+
};
|
|
291
489
|
|
|
292
490
|
function neighborScore(sd, ctx, r, c, d) {
|
|
293
491
|
const eq = (a, b) => (a !== "" && a === b ? 0.5 : 0);
|
|
294
|
-
|
|
492
|
+
const near = (rr, cc) => (rr < 0 || cc < 0 ? "" : sh(sd.cells.get(key(rr, cc))));
|
|
493
|
+
return eq(ctx.left, near(r, c - 1)) + eq(ctx.right, near(r, c + (d.c1 - d.c0) + 1)) + eq(ctx.up, near(r - 1, c)) + eq(ctx.down, near(r + (d.r1 - d.r0) + 1, c));
|
|
295
494
|
}
|
|
296
495
|
|
|
297
496
|
// ---- anchors --------------------------------------------------------------------------------------------------
|
|
@@ -301,6 +500,33 @@ const bookRef = (ref) => {
|
|
|
301
500
|
if (!m) return null;
|
|
302
501
|
return { sheet: (m[1] ?? m[2]).replace(/''/g, "'"), a1: m[5] ? `${m[3]}${m[4]}:${m[5]}${m[6]}` : `${m[3]}${m[4]}` };
|
|
303
502
|
};
|
|
503
|
+
// the defined names by the cell they point at and by name, read once per workbook (an anchor looked up every name, with a regex each)
|
|
504
|
+
function nameIndex(wb) {
|
|
505
|
+
if (wb.nameIndex) return wb.nameIndex;
|
|
506
|
+
const at = new Map();
|
|
507
|
+
const byName = new Map();
|
|
508
|
+
for (const n of wb.names) {
|
|
509
|
+
if (!byName.has(n.name)) byName.set(n.name, n);
|
|
510
|
+
const b = bookRef(n.ref);
|
|
511
|
+
if (!b) continue;
|
|
512
|
+
const k = `${b.sheet}\u0000${b.a1}`;
|
|
513
|
+
(at.get(k) || at.set(k, []).get(k)).push(n.name);
|
|
514
|
+
}
|
|
515
|
+
return (wb.nameIndex = { at, byName });
|
|
516
|
+
}
|
|
517
|
+
// What an anchor keeps of the names that point at its range: resolving reads them in order and takes the first that still checks out, so
|
|
518
|
+
// a few are enough. At most NAMES_KEEP of at most 255 characters (Excel's own limit), each paid from the budget by its length (20,000
|
|
519
|
+
// names of 250 characters on A1: every comment on A1 kept all of them, 5 MB each, 400 MB of session file for 80 comments).
|
|
520
|
+
const NAMES_KEEP = 8;
|
|
521
|
+
const namesAt = (wb, sheetName, full) => {
|
|
522
|
+
const out = [];
|
|
523
|
+
for (const nm of nameIndex(wb).at.get(`${sheetName}\u0000${full}`) || []) {
|
|
524
|
+
if (out.length >= NAMES_KEEP) break;
|
|
525
|
+
if (nm.length <= 255 && spend(nm.length)) out.push(nm);
|
|
526
|
+
}
|
|
527
|
+
return out;
|
|
528
|
+
};
|
|
529
|
+
const nameDef = (wb, nm) => nameIndex(wb).byName.get(nm);
|
|
304
530
|
const pickSheet = (wb, { sheetName, sheetIndex }) => wb.sheets.find((s) => s.name === sheetName) || wb.sheets[sheetIndex] || null;
|
|
305
531
|
|
|
306
532
|
/**
|
|
@@ -314,18 +540,23 @@ export function xlsxDurable(wb, loc) {
|
|
|
314
540
|
const q = parseA1(loc.range || "");
|
|
315
541
|
if (!q) return { ...base, cell: false, text: "", textHash: null, ctx: null, names: [] };
|
|
316
542
|
const sd = wb.sheetCells(sheet);
|
|
317
|
-
|
|
543
|
+
// Within the request's budget, like resolving (src/text.js scanBudget): the identity first, so a range too large to hash whole still
|
|
544
|
+
// knows its top-left cell. Past the budget the anchor still goes out, weaker: the address and what context was paid for, with a null
|
|
545
|
+
// hash (the batch offers the agent full `ops` for it, never a `set` checked against a hash it does not have).
|
|
546
|
+
const textHash = hashAt(sd, q.r0, q.c0); // of the top-left cell: the identity
|
|
547
|
+
const got = rangeOf(sd, q);
|
|
318
548
|
const full = a1Range(q);
|
|
319
549
|
return {
|
|
320
550
|
...base,
|
|
321
551
|
cell: true,
|
|
322
552
|
r0: q.r0, c0: q.c0, r1: q.r1, c1: q.c1,
|
|
323
|
-
text, // typed text of the
|
|
324
|
-
|
|
325
|
-
|
|
553
|
+
text: got ? got.text : "", // typed text of the range, as much as the batch can show (TEXT_KEEP)
|
|
554
|
+
values: got ? got.values : [],
|
|
555
|
+
textHash,
|
|
556
|
+
rangeHash: got ? got.hash : null,
|
|
326
557
|
ctx: contextOf(sd, q),
|
|
327
558
|
...cellMeta(wb, sd, q),
|
|
328
|
-
names: wb
|
|
559
|
+
names: namesAt(wb, sheet.name, full),
|
|
329
560
|
};
|
|
330
561
|
}
|
|
331
562
|
|
|
@@ -368,43 +599,73 @@ function resolveIn(wb, s, d) {
|
|
|
368
599
|
const hh = d.r1 - d.r0;
|
|
369
600
|
const shape = (r, c) => ({ r0: r, c0: c, r1: r + hh, c1: c + w });
|
|
370
601
|
const res = (method, q) => {
|
|
371
|
-
|
|
372
|
-
|
|
602
|
+
// rangeOf reads up to 2001 x 201 cells: paid before it is read, nothing of it once the budget is out. An anchor made without its
|
|
603
|
+
// range hash (past the budget at Send), or a range whose hash this request did not pay for, compares the text both kept (its first
|
|
604
|
+
// TEXT_KEEP characters): a difference there is a change. Ceiling: an edit past that text reads as unchanged; ordinary ranges keep
|
|
605
|
+
// their hash (rangeOf). An anchor stored by an earlier version (no `values`) kept its whole text, and one whose walk was not paid
|
|
606
|
+
// kept none (empty `values`): neither is compared.
|
|
607
|
+
const got = rangeOf(sd, q);
|
|
608
|
+
const text = got ? got.text : "";
|
|
609
|
+
const changed = d.rangeHash != null && got?.hash != null ? got.hash !== d.rangeHash : !!got && d.values?.length > 0 && got.text !== d.text;
|
|
610
|
+
return { method, sheetId: s.sheetId, sheetName: s.name, sheetIndex: s.index, range: a1Range(q), changed, moved: q.r0 !== d.r0 || q.c0 !== d.c0, text: text.slice(0, 200), ...q };
|
|
373
611
|
};
|
|
374
612
|
const noCtx = !hasRowInfo(ctx) && !ctx.left && !ctx.right && !ctx.up && !ctx.down;
|
|
375
|
-
const rowOk = (r, c) => sameRow(sd, ctx, r, c, c + w);
|
|
613
|
+
const rowOk = (r, c, charged = false) => sameRow(sd, ctx, r, c, c + w, !charged); // charged: the caller spent for these rows
|
|
376
614
|
const colOk = (r, c) => !ctx.colHeader || colHeaderAt(sd, r, c) === ctx.colHeader;
|
|
377
615
|
const gate = (r, c) => (hasRowInfo(ctx) ? rowOk(r, c) : noCtx || neighborScore(sd, ctx, r, c, d) >= 1);
|
|
616
|
+
// an anchor made without the top-left hash (past the budget at Send) is found by its context only (steps 3 to 5); with no context
|
|
617
|
+
// either, by its address, as a whole-row or whole-sheet anchor is. hashAt is null past the budget: never equal to a missing hash.
|
|
618
|
+
const known = d.textHash != null;
|
|
378
619
|
// 1. same address, same text, same row
|
|
379
|
-
if (
|
|
620
|
+
if (known && hashAt(sd, d.r0, d.c0) === d.textHash && gate(d.r0, d.c0)) return res("position", shape(d.r0, d.c0));
|
|
380
621
|
// 2. the same text somewhere else; the context picks between equal values
|
|
381
|
-
if (d.textHash !== EMPTY) {
|
|
622
|
+
if (known && d.textHash !== EMPTY && spend(sd.cells.size * 8)) {
|
|
623
|
+
// the scan of every cell runs within the budget of the anchors resolved together (src/text.js scanBudget)
|
|
382
624
|
const hits = [];
|
|
383
|
-
for (const cell of sd.cells.values()) if (
|
|
625
|
+
for (const cell of sd.cells.values()) if (chPaid(cell) === d.textHash) hits.push(cell);
|
|
626
|
+
// Ceiling: scoring a hit scans its row, so when the hits' rows hold more than SAME_TEXT_WORK cells in all (a sheet of one value
|
|
627
|
+
// repeated in long rows), they are not scored one by one; the steps below place the anchor by its row and column instead.
|
|
628
|
+
// Upgrade path: score each row once per anchor.
|
|
629
|
+
// The rows the hits sit in are spent from the request's budget too (the ceiling above is per anchor; 500 anchors of it were not bounded).
|
|
630
|
+
let work = 0;
|
|
631
|
+
for (const x of hits) work += sd.rows.get(x.r)?.length || 0;
|
|
632
|
+
if (work > SAME_TEXT_WORK || !spend(2 * work + hits.length)) hits.length = 0;
|
|
384
633
|
// one other cell with the same text only counts when the row or column header agree with it (or the anchor never had any): an edited
|
|
385
634
|
// cell must not be stolen by an unrelated cell that happens to hold the old text; the in-place step below gets its chance first
|
|
386
|
-
const fits = (x) => (hasRowInfo(ctx) ? rowOk(x.r, x.c) || (!!ctx.colHeader && colHeaderAt(sd, x.r, x.c) === ctx.colHeader) : !ctx.colHeader || colHeaderAt(sd, x.r, x.c) === ctx.colHeader);
|
|
635
|
+
const fits = (x) => (hasRowInfo(ctx) ? rowOk(x.r, x.c, true) || (!!ctx.colHeader && colHeaderAt(sd, x.r, x.c) === ctx.colHeader) : !ctx.colHeader || colHeaderAt(sd, x.r, x.c) === ctx.colHeader);
|
|
387
636
|
if (hits.length === 1 && fits(hits[0])) return res("value", shape(hits[0].r, hits[0].c));
|
|
388
637
|
if (hits.length > 1) {
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
638
|
+
// the best and the runner-up in one pass (a sort of every hit is not needed for two)
|
|
639
|
+
let best = null;
|
|
640
|
+
let second = -Infinity;
|
|
641
|
+
for (const x of hits) {
|
|
642
|
+
const score = (rowOk(x.r, x.c, true) ? 2 : 0) + (ctx.colHeader && colHeaderAt(sd, x.r, x.c) === ctx.colHeader ? 1.5 : 0) + neighborScore(sd, ctx, x.r, x.c, d);
|
|
643
|
+
if (!best || score > best.score) (second = best ? best.score : -Infinity), (best = { x, score });
|
|
644
|
+
else if (score > second) second = score;
|
|
645
|
+
}
|
|
646
|
+
if (best.score > 0 && best.score > second) return res("value", shape(best.x.r, best.x.c));
|
|
393
647
|
}
|
|
394
648
|
}
|
|
395
649
|
// 3. edited in place: the text differs but the row and the column header still say it is the same cell
|
|
396
|
-
|
|
650
|
+
// (an anchor without its hash: one remembered neighbor still there is enough, as it has nothing else to go by)
|
|
651
|
+
if (hasRowInfo(ctx) ? rowOk(d.r0, d.c0) && colOk(d.r0, d.c0) : (!known && noCtx) || (!noCtx && neighborScore(sd, ctx, d.r0, d.c0, d) >= (known ? 1 : 0.5))) return res("position", shape(d.r0, d.c0));
|
|
397
652
|
// 4. the text changed and the cell moved: the row by its other cells, the column by its header
|
|
398
|
-
if (hasRowInfo(ctx)) {
|
|
399
|
-
|
|
653
|
+
if (hasRowInfo(ctx) && spend(sd.cells.size * 8)) {
|
|
654
|
+
let best = null;
|
|
655
|
+
let second = null;
|
|
656
|
+
let count = 0;
|
|
400
657
|
for (const r of sd.rows.keys()) {
|
|
401
658
|
const ov = rowOverlap(sd, ctx, r, -1, -1);
|
|
402
|
-
const hdr = !!ctx.rowHeader &&
|
|
403
|
-
if (ov >= 0.6 || hdr)
|
|
659
|
+
const hdr = !!ctx.rowHeader && sh(sd.rows.get(r)[0]) === ctx.rowHeader;
|
|
660
|
+
if (ov >= 0.6 || hdr) {
|
|
661
|
+
const hit = { r, score: ov + (hdr ? 0.5 : 0) };
|
|
662
|
+
count++;
|
|
663
|
+
if (!best || hit.score > best.score) (second = best), (best = hit);
|
|
664
|
+
else if (!second || hit.score > second.score) second = hit;
|
|
665
|
+
}
|
|
404
666
|
}
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
const row = rowsHit[0].r;
|
|
667
|
+
if (best && (count === 1 || best.score > second.score) && spend(sd.maxC + 1)) {
|
|
668
|
+
const row = best.r;
|
|
408
669
|
const cols = [];
|
|
409
670
|
if (ctx.colHeader) {
|
|
410
671
|
for (let c = 0; c <= sd.maxC; c++) if (colHeaderAt(sd, row, c) === ctx.colHeader) cols.push(c);
|
|
@@ -414,11 +675,11 @@ function resolveIn(wb, s, d) {
|
|
|
414
675
|
}
|
|
415
676
|
// 5. a defined name that pointed at this range; last, because a library that does not shift names leaves them stale
|
|
416
677
|
for (const nm of d.names || []) {
|
|
417
|
-
const def = wb
|
|
678
|
+
const def = nameDef(wb, nm);
|
|
418
679
|
const b = def ? bookRef(def.ref) : null;
|
|
419
680
|
const q = b && b.sheet === s.name ? parseA1(b.a1) : null;
|
|
420
681
|
// a name that moved was updated by the writer (trusted); one that still points at the old address is stale unless the cell checks out
|
|
421
|
-
const verified = q && (
|
|
682
|
+
const verified = q && ((known && hashAt(sd, q.r0, q.c0) === d.textHash) || (hasRowInfo(ctx) ? sameRow(sd, ctx, q.r0, q.c0, q.c0 + w) : neighborScore(sd, ctx, q.r0, q.c0, d) >= 1));
|
|
422
683
|
if (q && (q.r0 !== d.r0 || q.c0 !== d.c0 || verified)) return res("name", q);
|
|
423
684
|
}
|
|
424
685
|
return { method: "none" };
|
|
@@ -426,7 +687,7 @@ function resolveIn(wb, s, d) {
|
|
|
426
687
|
|
|
427
688
|
/** The sheet is gone (renamed and renumbered): the cell by its text and context in every sheet; only one clear winner counts. */
|
|
428
689
|
function contentSearch(wb, d) {
|
|
429
|
-
if (!d.cell || d.textHash === EMPTY) return { method: "none" };
|
|
690
|
+
if (!d.cell || d.textHash === EMPTY || d.textHash == null) return { method: "none" };
|
|
430
691
|
const found = [];
|
|
431
692
|
for (const s of wb.sheets) {
|
|
432
693
|
const r = resolveIn(wb, s, d);
|