@qvac/core 1.0.0-rc.8 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +243 -3
- package/README.md +1 -0
- package/dist-lib/lib/api/attachments.d.ts +1 -1
- package/dist-lib/lib/api/attachments.js +11 -0
- package/dist-lib/lib/api/chunks.d.ts +4 -2
- package/dist-lib/lib/api/chunks.js +6 -3
- package/dist-lib/lib/api/devices.js +6 -18
- package/dist-lib/lib/api/execution.d.ts +6 -2
- package/dist-lib/lib/api/execution.js +29 -3
- package/dist-lib/lib/api/host-run-log.d.ts +7 -1
- package/dist-lib/lib/api/host-run-log.js +3 -1
- package/dist-lib/lib/api/index.d.ts +3 -1
- package/dist-lib/lib/api/index.js +17 -6
- package/dist-lib/lib/api/members.js +13 -29
- package/dist-lib/lib/api/mesh.d.ts +2 -1
- package/dist-lib/lib/api/projects.d.ts +1 -1
- package/dist-lib/lib/api/projects.js +9 -5
- package/dist-lib/lib/api/rag-label.d.ts +4 -0
- package/dist-lib/lib/api/rag-label.js +21 -0
- package/dist-lib/lib/api/rag-lexical.d.ts +22 -0
- package/dist-lib/lib/api/rag-lexical.js +1058 -0
- package/dist-lib/lib/api/rag-manifest.d.ts +13 -0
- package/dist-lib/lib/api/rag-manifest.js +46 -0
- package/dist-lib/lib/api/rag-outline.d.ts +36 -0
- package/dist-lib/lib/api/rag-outline.js +426 -0
- package/dist-lib/lib/api/rag-plugin.d.ts +20 -0
- package/dist-lib/lib/api/rag-plugin.js +70 -0
- package/dist-lib/lib/api/rag-synthesis.d.ts +7 -0
- package/dist-lib/lib/api/rag-synthesis.js +102 -0
- package/dist-lib/lib/api/rag-targets.d.ts +23 -0
- package/dist-lib/lib/api/rag-targets.js +215 -0
- package/dist-lib/lib/api/rag-window.d.ts +5 -0
- package/dist-lib/lib/api/rag-window.js +46 -0
- package/dist-lib/lib/api/rag.d.ts +3 -2
- package/dist-lib/lib/api/rag.js +421 -181
- package/dist-lib/lib/api/served.js +8 -14
- package/dist-lib/lib/api/workspace.d.ts +8 -1
- package/dist-lib/lib/api/workspace.js +27 -13
- package/dist-lib/lib/assistant/harness.d.ts +9 -1
- package/dist-lib/lib/assistant/history.d.ts +11 -2
- package/dist-lib/lib/assistant/history.js +74 -4
- package/dist-lib/lib/assistant/media-round.d.ts +61 -0
- package/dist-lib/lib/assistant/media-round.js +111 -0
- package/dist-lib/lib/assistant/message.d.ts +7 -0
- package/dist-lib/lib/assistant/model-round.d.ts +12 -0
- package/dist-lib/lib/assistant/model-round.js +130 -17
- package/dist-lib/lib/assistant/pdf-pages.d.ts +16 -1
- package/dist-lib/lib/assistant/pdf-pages.js +67 -3
- package/dist-lib/lib/assistant/round-host.d.ts +22 -0
- package/dist-lib/lib/assistant/round-host.js +142 -8
- package/dist-lib/lib/assistant/upload.d.ts +12 -0
- package/dist-lib/lib/assistant/upload.js +36 -0
- package/dist-lib/lib/assistant-client.d.ts +1 -0
- package/dist-lib/lib/assistant-client.js +40 -19
- package/dist-lib/lib/assistant-tree.d.ts +2 -0
- package/dist-lib/lib/assistant-tree.js +41 -1
- package/dist-lib/lib/assistant.d.ts +15 -0
- package/dist-lib/lib/assistant.js +474 -89
- package/dist-lib/lib/client.d.ts +1 -0
- package/dist-lib/lib/client.js +4 -1
- package/dist-lib/lib/compaction.js +2 -2
- package/dist-lib/lib/core/agent.js +12 -45
- package/dist-lib/lib/core/approval-rules.js +1 -8
- package/dist-lib/lib/core/archived-mesh.d.ts +2 -1
- package/dist-lib/lib/core/archived-mesh.js +12 -8
- package/dist-lib/lib/core/chat-copy.d.ts +18 -3
- package/dist-lib/lib/core/chat-copy.js +74 -25
- package/dist-lib/lib/core/chat-fork.js +15 -6
- package/dist-lib/lib/core/chat-group.js +16 -24
- package/dist-lib/lib/core/chat-move.d.ts +9 -0
- package/dist-lib/lib/core/chat-move.js +94 -3
- package/dist-lib/lib/core/chat-order.js +12 -18
- package/dist-lib/lib/core/chat.js +38 -31
- package/dist-lib/lib/core/chunk.d.ts +2 -2
- package/dist-lib/lib/core/chunk.js +18 -25
- package/dist-lib/lib/core/collections.d.ts +2 -0
- package/dist-lib/lib/core/collections.js +2 -0
- package/dist-lib/lib/core/contract/activity.d.ts +4 -0
- package/dist-lib/lib/core/contract/activity.js +34 -4
- package/dist-lib/lib/core/contract/chat-group.d.ts +1 -0
- package/dist-lib/lib/core/contract/chat-group.js +21 -1
- package/dist-lib/lib/core/contract/chat.d.ts +9 -0
- package/dist-lib/lib/core/contract/chat.js +127 -5
- package/dist-lib/lib/core/contract/chunks.d.ts +1 -0
- package/dist-lib/lib/core/contract/chunks.js +37 -6
- package/dist-lib/lib/core/contract/features.d.ts +21 -0
- package/dist-lib/lib/core/contract/features.js +22 -0
- package/dist-lib/lib/core/contract/index.js +2 -0
- package/dist-lib/lib/core/contract/indexing.js +4 -2
- package/dist-lib/lib/core/contract/knowledge.js +5 -4
- package/dist-lib/lib/core/contract/members.js +16 -10
- package/dist-lib/lib/core/contract/mesh-kind.d.ts +1 -0
- package/dist-lib/lib/core/contract/mesh-kind.js +3 -1
- package/dist-lib/lib/core/contract/projects.d.ts +5 -0
- package/dist-lib/lib/core/contract/projects.js +13 -5
- package/dist-lib/lib/core/contract/roles.js +32 -4
- package/dist-lib/lib/core/contract/run-requests.d.ts +1 -0
- package/dist-lib/lib/core/contract/run-requests.js +35 -2
- package/dist-lib/lib/core/contract/seed-defaults.js +2 -2
- package/dist-lib/lib/core/contract/tool-configs.js +3 -2
- package/dist-lib/lib/core/contract/workspace.d.ts +2 -0
- package/dist-lib/lib/core/contract/workspace.js +40 -7
- package/dist-lib/lib/core/contract/writers.js +2 -2
- package/dist-lib/lib/core/departures.d.ts +3 -1
- package/dist-lib/lib/core/departures.js +3 -0
- package/dist-lib/lib/core/devices.js +1 -8
- package/dist-lib/lib/core/files/attachments.d.ts +2 -0
- package/dist-lib/lib/core/files/attachments.js +19 -1
- package/dist-lib/lib/core/files/index.d.ts +20 -4
- package/dist-lib/lib/core/files/index.js +132 -21
- package/dist-lib/lib/core/files/processing/document.d.ts +4 -0
- package/dist-lib/lib/core/files/processing/document.js +7 -0
- package/dist-lib/lib/core/files/processing/index.d.ts +13 -0
- package/dist-lib/lib/core/files/processing/index.js +144 -12
- package/dist-lib/lib/core/files/processing/protocol.js +13 -34
- package/dist-lib/lib/core/files/processing/state.d.ts +2 -0
- package/dist-lib/lib/core/files/rag/build.d.ts +19 -2
- package/dist-lib/lib/core/files/rag/build.js +123 -19
- package/dist-lib/lib/core/files/rag/failure.js +2 -1
- package/dist-lib/lib/core/files/rag/index.d.ts +11 -3
- package/dist-lib/lib/core/files/rag/index.js +13 -1
- package/dist-lib/lib/core/files/rag/load.d.ts +6 -3
- package/dist-lib/lib/core/files/rag/load.js +27 -6
- package/dist-lib/lib/core/files/rag/plugin-index.d.ts +32 -0
- package/dist-lib/lib/core/files/rag/plugin-index.js +127 -0
- package/dist-lib/lib/core/files/rag/produce.d.ts +5 -3
- package/dist-lib/lib/core/files/rag/produce.js +104 -16
- package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +3 -1
- package/dist-lib/lib/core/files/rag/retrieval/full-text.js +62 -13
- package/dist-lib/lib/core/files/rag/text-cache.d.ts +28 -0
- package/dist-lib/lib/core/files/rag/text-cache.js +141 -0
- package/dist-lib/lib/core/files/records/knowledge.d.ts +1 -3
- package/dist-lib/lib/core/files/records/knowledge.js +7 -40
- package/dist-lib/lib/core/files/records/reclaim.js +5 -10
- package/dist-lib/lib/core/files/upload.js +68 -41
- package/dist-lib/lib/core/fs-drives.js +14 -26
- package/dist-lib/lib/core/invite.d.ts +1 -0
- package/dist-lib/lib/core/invite.js +4 -2
- package/dist-lib/lib/core/local/files.d.ts +1 -1
- package/dist-lib/lib/core/local/files.js +5 -5
- package/dist-lib/lib/core/local/index.d.ts +13 -6
- package/dist-lib/lib/core/local/index.js +103 -69
- package/dist-lib/lib/core/local/run-logs.d.ts +1 -1
- package/dist-lib/lib/core/local/run-logs.js +5 -5
- package/dist-lib/lib/core/mesh-processor.d.ts +19 -0
- package/dist-lib/lib/core/mesh-processor.js +87 -0
- package/dist-lib/lib/core/mesh.d.ts +48 -5
- package/dist-lib/lib/core/mesh.js +318 -129
- package/dist-lib/lib/core/migrations/agent-catalog.js +2 -8
- package/dist-lib/lib/core/migrations/agent-tools-scope.js +5 -11
- package/dist-lib/lib/core/migrations/workspace-from-0-21/index.d.ts +88 -0
- package/dist-lib/lib/core/migrations/workspace-from-0-21/index.js +731 -0
- package/dist-lib/lib/core/migrations/writer-links.d.ts +2 -0
- package/dist-lib/lib/core/migrations/writer-links.js +32 -0
- package/dist-lib/lib/core/project-keys.d.ts +6 -0
- package/dist-lib/lib/core/project-keys.js +11 -0
- package/dist-lib/lib/core/public-mesh.d.ts +3 -0
- package/dist-lib/lib/core/public-mesh.js +10 -8
- package/dist-lib/lib/core/run-requests.d.ts +3 -0
- package/dist-lib/lib/core/run-requests.js +21 -1
- package/dist-lib/lib/core/skill-availability.d.ts +33 -0
- package/dist-lib/lib/core/skill-availability.js +67 -0
- package/dist-lib/lib/core/sync/background.d.ts +0 -1
- package/dist-lib/lib/core/sync/background.js +2 -1
- package/dist-lib/lib/core/sync/changed.d.ts +4 -2
- package/dist-lib/lib/core/sync/changed.js +1 -1
- package/dist-lib/lib/core/sync/index.js +52 -10
- package/dist-lib/lib/core/sync/view-updates.d.ts +18 -12
- package/dist-lib/lib/core/sync/view-updates.js +82 -91
- package/dist-lib/lib/core/sync-status.js +97 -41
- package/dist-lib/lib/core/tool-activity.js +6 -12
- package/dist-lib/lib/core/tool-configs.js +17 -29
- package/dist-lib/lib/core/upgrade-summary.d.ts +19 -0
- package/dist-lib/lib/core/upgrade-summary.js +67 -0
- package/dist-lib/lib/core/view-read.d.ts +4 -0
- package/dist-lib/lib/core/view-read.js +23 -0
- package/dist-lib/lib/core/workspace-upgrade.d.ts +7 -0
- package/dist-lib/lib/core/workspace-upgrade.js +27 -16
- package/dist-lib/lib/core.d.ts +39 -42
- package/dist-lib/lib/core.js +661 -639
- package/dist-lib/lib/errors.d.ts +20 -0
- package/dist-lib/lib/errors.js +20 -0
- package/dist-lib/lib/llm-model-names.d.ts +1 -1
- package/dist-lib/lib/llm-model-names.js +1 -1
- package/dist-lib/lib/mesh-scoped-engine.js +4 -0
- package/dist-lib/lib/model-context.js +1 -2
- package/dist-lib/lib/probe-rounds.d.ts +7 -2
- package/dist-lib/lib/probe-rounds.js +6 -1
- package/dist-lib/lib/run-transcript.d.ts +4 -1
- package/dist-lib/lib/run-transcript.js +15 -6
- package/dist-lib/schema/engine.js +83 -6
- package/dist-lib/schema/indexes.d.ts +3 -0
- package/dist-lib/schema/indexes.js +5 -0
- package/dist-lib/schema/local.js +11 -2
- package/dist-lib/schema/qvac.js +49 -5
- package/dist-lib/spec/bind.js +34 -0
- package/dist-lib/spec/calls.js +5 -0
- package/dist-lib/spec/capabilities.d.ts +17 -0
- package/dist-lib/spec/hrpc/hrpc.json +65 -0
- package/dist-lib/spec/hrpc/index.js +68 -5
- package/dist-lib/spec/hrpc/messages.js +260 -23
- package/dist-lib/spec/hyperschema/index.js +260 -23
- package/dist-lib/spec/hyperschema/schema.json +181 -0
- package/dist-lib/spec/hyperschema/types.d.ts +48 -0
- package/dist-lib/spec/local/hyperdb/messages.js +270 -224
- package/dist-lib/spec/local/hyperschema/index.js +73 -38
- package/dist-lib/spec/local/hyperschema/schema.json +31 -0
- package/dist-lib/spec/local/hyperschema/types.d.ts +8 -0
- package/dist-lib/spec/mesh/hyperdb/db.json +39 -2
- package/dist-lib/spec/mesh/hyperdb/index.js +120 -3
- package/dist-lib/spec/mesh/hyperdb/messages.js +1328 -838
- package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +30 -0
- package/dist-lib/spec/mesh/hyperdispatch/index.js +86 -1
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +682 -287
- package/dist-lib/spec/mesh/hyperschema/index.js +682 -287
- package/dist-lib/spec/mesh/hyperschema/schema.json +201 -0
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +52 -0
- package/package.json +75 -70
- package/tui/README.md +2 -1
- package/tui/app.mjs +28 -7
- package/tui/boot.mjs +5 -1
- package/tui/lib/attachment-files.mjs +82 -0
- package/tui/lib/reopen-meshes.mjs +15 -0
- package/tui/main.mjs +4 -43
- package/worker/assistant-sidecar.mjs +10 -3
- package/worker/stow-entry.mjs +3 -52
|
@@ -0,0 +1,1058 @@
|
|
|
1
|
+
import { isSubstantive, outlineLines, outlineOf } from './rag-outline.js';
|
|
2
|
+
import { DEFINITION_SHARE, planOf, referenced, setsValue } from './rag-targets.js';
|
|
3
|
+
import { newChars, render, unionOf, windowAround } from './rag-window.js';
|
|
4
|
+
const CHARS_PER_TOKEN = 4;
|
|
5
|
+
const MIN_WORD = 3;
|
|
6
|
+
const STOP = new Set(('the and for from with this that these those into onto over under about above below between ' +
|
|
7
|
+
'what which when where who whom whose why how are was were been being have has had does did ' +
|
|
8
|
+
'doing will would shall should can could may might must not any all each every some such than ' +
|
|
9
|
+
'then there their them they you your our its also use using used make made give given list ' +
|
|
10
|
+
'show tell find write produce generate concise exact mark anything attached document please').split(' '));
|
|
11
|
+
const DEFINITION_BOOST = 10;
|
|
12
|
+
const KIND_BOOST = 4;
|
|
13
|
+
const DEFINES = /^\s*\[?["“][^"”]{1,80}["”] means/;
|
|
14
|
+
const HEADING_BOOST = 6;
|
|
15
|
+
const HEADING_PREFIX_BOOST = 3;
|
|
16
|
+
const HEADING_WORDS = 8;
|
|
17
|
+
const CONTENTS_LINES = 5;
|
|
18
|
+
const SNIPPET_CHARS = 450;
|
|
19
|
+
const MIN_WINDOW_CHARS = 60;
|
|
20
|
+
const MIN_CLIP = 20;
|
|
21
|
+
export function queryTerms(query) {
|
|
22
|
+
return queryFields(query).flat();
|
|
23
|
+
}
|
|
24
|
+
export function queryFields(query) {
|
|
25
|
+
return askedItems(query).map(({ terms }) => terms);
|
|
26
|
+
}
|
|
27
|
+
export function askedItems(query) {
|
|
28
|
+
const items = listedItems(query);
|
|
29
|
+
if (items.length >= MIN_LISTED) {
|
|
30
|
+
return items.map(({ item, parent }) => ({
|
|
31
|
+
label: parent ? `${item} (${parent})` : item,
|
|
32
|
+
terms: fieldOf(item, parent)
|
|
33
|
+
}));
|
|
34
|
+
}
|
|
35
|
+
return wordFields(query).map((terms) => ({ label: terms[0] ?? '', terms }));
|
|
36
|
+
}
|
|
37
|
+
function wordFields(query) {
|
|
38
|
+
const words = keptWords(query);
|
|
39
|
+
const phrases = queryPhrases(query, words);
|
|
40
|
+
const fields = phrases.map((phrase) => [phrase]);
|
|
41
|
+
for (const word of new Set(words)) {
|
|
42
|
+
const owner = fields.find(([phrase = '']) => phrase.split(' ').includes(word));
|
|
43
|
+
if (owner) {
|
|
44
|
+
owner.push(word);
|
|
45
|
+
}
|
|
46
|
+
else {
|
|
47
|
+
fields.push([word]);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
const at = (field) => query.toLowerCase().indexOf(field[0] ?? '');
|
|
51
|
+
return fields.sort((a, b) => at(a) - at(b));
|
|
52
|
+
}
|
|
53
|
+
const MIN_LISTED = 3;
|
|
54
|
+
export function asksForItems(query) {
|
|
55
|
+
return listedItems(query).length >= MIN_LISTED;
|
|
56
|
+
}
|
|
57
|
+
const HINT = '~';
|
|
58
|
+
const HINT_WEIGHT = 0.1;
|
|
59
|
+
export function keptWords(text) {
|
|
60
|
+
return [
|
|
61
|
+
...normalize(text).matchAll(/[\p{L}\p{N}][\p{L}\p{N}\p{M}]*(?:[-.,][\p{L}\p{N}][\p{L}\p{N}\p{M}]*)*/gu)
|
|
62
|
+
]
|
|
63
|
+
.map(([word]) => word)
|
|
64
|
+
.filter((word) => (word.length >= MIN_WORD || /[\p{N}\u0080-\uffff]/u.test(word)) && !STOP.has(word));
|
|
65
|
+
}
|
|
66
|
+
function normalize(text) {
|
|
67
|
+
return text.normalize('NFKC').toLowerCase();
|
|
68
|
+
}
|
|
69
|
+
function fieldOf(item, parent) {
|
|
70
|
+
const modal = /\b(may|can|must|should|shall)\s+(.+)$/.exec(item);
|
|
71
|
+
const topic = modal ? item.slice(0, modal.index).trim() : item;
|
|
72
|
+
const words = keptWords(topic);
|
|
73
|
+
const phrase = words.length > 1 && !/ and | or /.test(topic) ? [topic] : [];
|
|
74
|
+
const hints = parent ? keptWords(parent).map((word) => `${HINT}${word}`) : [];
|
|
75
|
+
const action = modal
|
|
76
|
+
? [modal[1] ?? '', ...keptWords(modal[2] ?? '')].map((word) => `${HINT}${word}`)
|
|
77
|
+
: [];
|
|
78
|
+
return [...new Set([...phrase, ...words, ...hints, ...action])];
|
|
79
|
+
}
|
|
80
|
+
function listedItems(query) {
|
|
81
|
+
const lists = [...query.matchAll(/:([^:.?!]+)/g)].map(([, list = '']) => list);
|
|
82
|
+
const list = lists.sort((a, b) => commas(b) - commas(a))[0] ?? '';
|
|
83
|
+
const items = [];
|
|
84
|
+
for (const [, head = '', inner] of list.matchAll(/([^,(]+)(?:\(([^)]*)\))?/g)) {
|
|
85
|
+
const parent = clean(head);
|
|
86
|
+
if (inner === undefined) {
|
|
87
|
+
items.push(...(parent ? [{ item: parent, parent: null }] : []));
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
for (const child of inner.split(',').map(clean).filter(Boolean)) {
|
|
91
|
+
items.push({ item: child, parent: parent || null });
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return items;
|
|
95
|
+
}
|
|
96
|
+
function queryHints(query) {
|
|
97
|
+
const lists = [...query.matchAll(/:([^:.?!]+)/g)];
|
|
98
|
+
const list = lists.sort((a, b) => commas(b[1] ?? '') - commas(a[1] ?? ''))[0];
|
|
99
|
+
const after = list ? query.slice((list.index ?? 0) + list[0].length) : '';
|
|
100
|
+
return [...new Set(keptWords(after))];
|
|
101
|
+
}
|
|
102
|
+
function commas(text) {
|
|
103
|
+
return text.split(',').length - 1;
|
|
104
|
+
}
|
|
105
|
+
function clean(text) {
|
|
106
|
+
return text
|
|
107
|
+
.trim()
|
|
108
|
+
.toLowerCase()
|
|
109
|
+
.replace(/^(?:and|or)\s+/, '')
|
|
110
|
+
.trim();
|
|
111
|
+
}
|
|
112
|
+
function queryPhrases(query, words) {
|
|
113
|
+
const phrases = new Set();
|
|
114
|
+
for (const [, quoted = ''] of query.matchAll(/["“”']([^"“”']{2,80})["“”']/g)) {
|
|
115
|
+
phrases.add(quoted.trim().toLowerCase());
|
|
116
|
+
}
|
|
117
|
+
for (const [run] of query.matchAll(/\b[A-Z][a-z]+(?: [A-Z][a-z]+)+\b/g)) {
|
|
118
|
+
phrases.add(run.toLowerCase());
|
|
119
|
+
}
|
|
120
|
+
const lower = query.toLowerCase();
|
|
121
|
+
for (const [at, word] of words.entries()) {
|
|
122
|
+
const next = words[at + 1];
|
|
123
|
+
if (next && lower.includes(`${word} ${next}`)) {
|
|
124
|
+
phrases.add(`${word} ${next}`);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
return [...phrases];
|
|
128
|
+
}
|
|
129
|
+
const OUTLINE_TOKENS = 600;
|
|
130
|
+
const SYNTHESIS_SPAN = 4;
|
|
131
|
+
const UNCOVERED_LISTED = 12;
|
|
132
|
+
const MIN_OWN_WORDS = 2;
|
|
133
|
+
const NEAR_TIE = 0.9;
|
|
134
|
+
export function rankLexical(hits, query, options) {
|
|
135
|
+
if (hits.length === 0) {
|
|
136
|
+
return null;
|
|
137
|
+
}
|
|
138
|
+
const text = new DocumentText(hits);
|
|
139
|
+
if (/\b(?:each|every)\s+(?:(?:attached|uploaded)\s+)?(?:file|document)s?\b/i.test(query)) {
|
|
140
|
+
return eachDocument(text, query, options);
|
|
141
|
+
}
|
|
142
|
+
return options.synthesis ? synthesize(text, options) : targeted(text, query, options);
|
|
143
|
+
}
|
|
144
|
+
const FILE_INSTRUCTIONS = new Set(('file files documents filename filenames page pages section sections cite citation citations ' +
|
|
145
|
+
'briefly describe identify main applicable rules different separate flag read then keep').split(' '));
|
|
146
|
+
const CONCRETE_VALUE = /[$€£¥]\s*\d[\d.,]*|\bminimum\s+(?:wages?|salar(?:y|ies))\b|\d+(?:[.,]\d+)?\s*(?:%|(?:(?:calendar|business|working)\s+)?days?|hours?|months?|years?|d[ií]as?|horas?|mes(?:es)?|a[nñ]os?)(?=$|[^\p{L}])/giu;
|
|
147
|
+
function eachDocument(text, query, { budgetTokens }) {
|
|
148
|
+
const terms = keptWords(query).filter((word) => !FILE_INSTRUCTIONS.has(word));
|
|
149
|
+
const ranked = text.rank(terms, { contextual: false });
|
|
150
|
+
const uncovered = [];
|
|
151
|
+
const parts = text.documents().flatMap(([start, end]) => {
|
|
152
|
+
const lines = text.lines.slice(start, end);
|
|
153
|
+
const noise = new Set(outlineOf(lines.map((line) => line.text)).noise.map((at) => start + at));
|
|
154
|
+
const repeats = new Map();
|
|
155
|
+
const key = (line) => normalize(line).replace(/\d+/g, '#').trim();
|
|
156
|
+
for (const line of lines) {
|
|
157
|
+
if (line.text.length < 200) {
|
|
158
|
+
const label = key(line.text);
|
|
159
|
+
repeats.set(label, (repeats.get(label) ?? 0) + 1);
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
const pool = lines.flatMap((line, offset) => {
|
|
163
|
+
const at = start + offset;
|
|
164
|
+
const caps = line.text.length < 200 && !/\p{Ll}/u.test(line.text);
|
|
165
|
+
if (noise.has(at) ||
|
|
166
|
+
caps ||
|
|
167
|
+
(line.text.length < 200 && (repeats.get(key(line.text)) ?? 0) >= 3)) {
|
|
168
|
+
noise.add(at);
|
|
169
|
+
return [];
|
|
170
|
+
}
|
|
171
|
+
return isSubstantive(line.text) ? [at] : [];
|
|
172
|
+
});
|
|
173
|
+
text.mute(noise);
|
|
174
|
+
if (pool.length === 0) {
|
|
175
|
+
uncovered.push(`${text.title(start)}: no substantive text`);
|
|
176
|
+
return [];
|
|
177
|
+
}
|
|
178
|
+
return [{ start, end, pool }];
|
|
179
|
+
});
|
|
180
|
+
const picked = new Picker(budgetTokens * CHARS_PER_TOKEN);
|
|
181
|
+
const share = Math.min(SNIPPET_CHARS * 2, Math.floor((budgetTokens * CHARS_PER_TOKEN) / Math.max(1, parts.length)));
|
|
182
|
+
for (const { start, end, pool } of parts) {
|
|
183
|
+
const candidates = new Set(pool);
|
|
184
|
+
const first = ranked.find(({ at }) => candidates.has(at));
|
|
185
|
+
const fallback = (fraction) => pool[Math.min(pool.length - 1, Math.floor(pool.length * fraction))] ?? start;
|
|
186
|
+
const primary = first
|
|
187
|
+
? { at: first.at, focus: text.offset(first.at, first.focus) }
|
|
188
|
+
: {
|
|
189
|
+
at: fallback(1 / 3),
|
|
190
|
+
focus: Math.floor((text.lines[fallback(1 / 3)]?.text.length ?? 0) / 3)
|
|
191
|
+
};
|
|
192
|
+
const take = ({ at, focus }) => picked.take(picked.fit((room) => text.snippet(at, { focus, end, chars: SNIPPET_CHARS, room: Math.min(room, share / 2) }), share / 2));
|
|
193
|
+
take(primary);
|
|
194
|
+
const values = pool.flatMap((at) => {
|
|
195
|
+
const raw = text.lines[at]?.text ?? '';
|
|
196
|
+
const found = [...raw.matchAll(CONCRETE_VALUE)];
|
|
197
|
+
return found.flatMap((match) => {
|
|
198
|
+
const focus = match.index ?? 0;
|
|
199
|
+
if (picked.lines.get(at)?.some(([from, to]) => focus >= from && focus < to)) {
|
|
200
|
+
return [];
|
|
201
|
+
}
|
|
202
|
+
const score = found.reduce((sum, other) => sum +
|
|
203
|
+
Number(Math.abs((other.index ?? 0) - focus) < SNIPPET_CHARS) *
|
|
204
|
+
(/[$€£¥%]|minimum/iu.test(other[0]) ? 2 : 1), 0);
|
|
205
|
+
return [{ at, focus, score }];
|
|
206
|
+
});
|
|
207
|
+
});
|
|
208
|
+
values.sort((a, b) => b.score - a.score || a.at - b.at || a.focus - b.focus);
|
|
209
|
+
const second = values[0] ?? {
|
|
210
|
+
at: fallback(2 / 3),
|
|
211
|
+
focus: Math.floor((text.lines[fallback(2 / 3)]?.text.length ?? 0) / 3)
|
|
212
|
+
};
|
|
213
|
+
take(second);
|
|
214
|
+
}
|
|
215
|
+
return { hits: text.excerpts(picked.lines, { marks: picked.marks }), uncovered };
|
|
216
|
+
}
|
|
217
|
+
function targeted(text, query, { budgetTokens, maxBudgetTokens, previousQuery }) {
|
|
218
|
+
const items = itemsOf(query, previousQuery);
|
|
219
|
+
const hints = queryHints(query);
|
|
220
|
+
const contextual = asksForItems(query) && !text.hasDefinitions;
|
|
221
|
+
const lists = items.map(({ terms }) => text.rank(terms, { hints, contextual }));
|
|
222
|
+
const ranked = lists.map((list) => new Field(list));
|
|
223
|
+
if (ranked.every((field) => field.empty)) {
|
|
224
|
+
return null;
|
|
225
|
+
}
|
|
226
|
+
const budgetChars = budgetTokens * CHARS_PER_TOKEN;
|
|
227
|
+
const chars = Math.min(TARGET_CHARS, Math.max(SNIPPET_CHARS, Math.floor(budgetChars / items.length / (hints.length ? 2 : 1))));
|
|
228
|
+
const picked = new Picker(budgetChars * (contextual ? 1 : 2 / 3));
|
|
229
|
+
if (contextual) {
|
|
230
|
+
const share = Math.floor(budgetChars / items.length);
|
|
231
|
+
for (const { terms } of items) {
|
|
232
|
+
const first = text.rank(terms, { contextual: false })[0];
|
|
233
|
+
if (first) {
|
|
234
|
+
picked.take(picked.fit((room) => text.snippet(first.at, {
|
|
235
|
+
focus: text.offset(first.at, first.focus),
|
|
236
|
+
room: Math.min(room, share)
|
|
237
|
+
})));
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
while (ranked.some((field) => !field.empty) && !picked.full) {
|
|
242
|
+
for (const [fieldAt, field] of ranked.entries()) {
|
|
243
|
+
const line = field.next(picked.documents);
|
|
244
|
+
if (line) {
|
|
245
|
+
const allowance = contextual
|
|
246
|
+
? Math.max(0, picked.remaining -
|
|
247
|
+
ranked.slice(fieldAt + 1).filter((remaining) => !remaining.empty).length *
|
|
248
|
+
Math.min(SNIPPET_CHARS, Math.floor(picked.remaining / items.length)))
|
|
249
|
+
: Infinity;
|
|
250
|
+
picked.take(picked.fit((room) => contextual
|
|
251
|
+
? text.target(line.at, {
|
|
252
|
+
focus: text.offset(line.at, line.focus),
|
|
253
|
+
chars,
|
|
254
|
+
room
|
|
255
|
+
})
|
|
256
|
+
: text.snippet(line.at, { focus: text.offset(line.at, line.focus), room }), allowance));
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
if (maxBudgetTokens !== undefined &&
|
|
261
|
+
maxBudgetTokens > budgetTokens &&
|
|
262
|
+
text.incompleteList(picked.lines)) {
|
|
263
|
+
return targeted(text, query, { budgetTokens: maxBudgetTokens, previousQuery });
|
|
264
|
+
}
|
|
265
|
+
const uncovered = asksForItems(query)
|
|
266
|
+
? items
|
|
267
|
+
.filter(({ terms }, field) => !lists[field]?.some(({ at }) => picked.lines.has(at) && text.sets(at, terms)))
|
|
268
|
+
.map(({ label }) => label)
|
|
269
|
+
: [];
|
|
270
|
+
return { hits: text.excerpts(picked.lines, { marks: picked.marks }), uncovered };
|
|
271
|
+
}
|
|
272
|
+
function itemsOf(query, previousQuery) {
|
|
273
|
+
if (previousQuery === undefined || keptWords(query).length >= MIN_OWN_WORDS) {
|
|
274
|
+
return askedItems(query);
|
|
275
|
+
}
|
|
276
|
+
return wordFields(`${query} ${previousQuery}`).map((terms) => ({ label: terms[0] ?? '', terms }));
|
|
277
|
+
}
|
|
278
|
+
function synthesize(text, { budgetTokens }) {
|
|
279
|
+
const budgetChars = budgetTokens * CHARS_PER_TOKEN;
|
|
280
|
+
const documents = text.documents();
|
|
281
|
+
const parts = documents.map(([start, end]) => {
|
|
282
|
+
const lines = text.lines.slice(start, end).map((line) => line.text);
|
|
283
|
+
const outline = outlineOf(lines);
|
|
284
|
+
const plan = planOf(outline, lines, {
|
|
285
|
+
budgetChars,
|
|
286
|
+
snippetChars: SNIPPET_CHARS,
|
|
287
|
+
documents: documents.length
|
|
288
|
+
});
|
|
289
|
+
return { start, end, outline, plan };
|
|
290
|
+
});
|
|
291
|
+
parts.forEach(({ start, outline }) => text.mute(outline.noise.map((line) => line + start)));
|
|
292
|
+
if (parts.every(({ plan }) => plan.front.length + plan.terms.length + plan.rest.length === 0)) {
|
|
293
|
+
return null;
|
|
294
|
+
}
|
|
295
|
+
const picked = new Picker(budgetChars);
|
|
296
|
+
const uncovered = [];
|
|
297
|
+
const take = (target) => {
|
|
298
|
+
const pieces = picked.fit((room) => text.snippet(target.start, {
|
|
299
|
+
end: target.end,
|
|
300
|
+
chars: target.chars,
|
|
301
|
+
focus: target.focus,
|
|
302
|
+
room
|
|
303
|
+
}));
|
|
304
|
+
const cost = picked.take(pieces);
|
|
305
|
+
const said = cost !== null && pieces.some(({ at }) => states(text.lines[at]?.text ?? ''));
|
|
306
|
+
if (!said && target.label) {
|
|
307
|
+
uncovered.push(target.label);
|
|
308
|
+
}
|
|
309
|
+
return said ? cost : null;
|
|
310
|
+
};
|
|
311
|
+
const placed = (list) => interleaved(parts.map(({ plan, start }) => list(plan).map((target) => shifted(target, start))));
|
|
312
|
+
placed((plan) => plan.front).forEach(take);
|
|
313
|
+
let termsLeft = budgetChars * DEFINITION_SHARE - parts.reduce((total, { plan }) => total + plan.spent, 0);
|
|
314
|
+
const used = parts.map(({ plan, start, end }) => {
|
|
315
|
+
const own = new Map([...picked.lines].filter(([line]) => line >= start && line < end));
|
|
316
|
+
const quote = text
|
|
317
|
+
.excerpts(own)
|
|
318
|
+
.map((hit) => hit.content)
|
|
319
|
+
.join('\n');
|
|
320
|
+
return interleaved([plan.terms, referenced(plan.terms, quote)]).map((target) => shifted(target, start));
|
|
321
|
+
});
|
|
322
|
+
for (const target of interleaved(used)) {
|
|
323
|
+
const cost = target.chars <= termsLeft ? take(target) : null;
|
|
324
|
+
termsLeft -= cost ?? 0;
|
|
325
|
+
}
|
|
326
|
+
placed((plan) => plan.rest).forEach(take);
|
|
327
|
+
const only = parts.length === 1 ? parts[0] : undefined;
|
|
328
|
+
return {
|
|
329
|
+
hits: text.excerpts(picked.lines, { span: SYNTHESIS_SPAN, marks: picked.marks }),
|
|
330
|
+
uncovered: uncovered.slice(0, UNCOVERED_LISTED),
|
|
331
|
+
outline: only && outlineLines(only.outline, OUTLINE_TOKENS * CHARS_PER_TOKEN)
|
|
332
|
+
};
|
|
333
|
+
}
|
|
334
|
+
function states(line) {
|
|
335
|
+
return DEFINES.test(line) || isSubstantive(line) || setsValue([line]);
|
|
336
|
+
}
|
|
337
|
+
function shifted(target, by) {
|
|
338
|
+
return { ...target, start: target.start + by, end: target.end + by };
|
|
339
|
+
}
|
|
340
|
+
function interleaved(lists) {
|
|
341
|
+
const longest = Math.max(0, ...lists.map((list) => list.length));
|
|
342
|
+
return Array.from({ length: longest }, (_, at) => lists.map((list) => list[at]))
|
|
343
|
+
.flat()
|
|
344
|
+
.filter((target) => target !== undefined);
|
|
345
|
+
}
|
|
346
|
+
class DocumentText {
|
|
347
|
+
lines = [];
|
|
348
|
+
_hits;
|
|
349
|
+
_places;
|
|
350
|
+
_weights = new Map();
|
|
351
|
+
_terms = new Map();
|
|
352
|
+
_muted = new Set();
|
|
353
|
+
_aliases = new Map();
|
|
354
|
+
_expanded = new Map();
|
|
355
|
+
_listContexts = new Map();
|
|
356
|
+
_sections = null;
|
|
357
|
+
constructor(hits) {
|
|
358
|
+
this._hits = hits;
|
|
359
|
+
this._places = hits.map((hit, at) => placeOf(hit.metadataJson, at));
|
|
360
|
+
const documents = new Map();
|
|
361
|
+
for (const [at, place] of this._places.entries()) {
|
|
362
|
+
const id = place.chunkIndex !== null ? place.documentId : `\u0000${at}`;
|
|
363
|
+
const members = documents.get(id);
|
|
364
|
+
if (members) {
|
|
365
|
+
members.push(at);
|
|
366
|
+
}
|
|
367
|
+
else {
|
|
368
|
+
documents.set(id, [at]);
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
for (const [document, members] of [...documents.values()].entries()) {
|
|
372
|
+
this._stitch(document, members);
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
get hasDefinitions() {
|
|
376
|
+
return this.lines.filter((line) => DEFINES.test(line.text)).length >= MIN_LISTED;
|
|
377
|
+
}
|
|
378
|
+
mute(lines) {
|
|
379
|
+
for (const at of lines) {
|
|
380
|
+
this._muted.add(at);
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
documents() {
|
|
384
|
+
const spans = [];
|
|
385
|
+
for (const [at, line] of this.lines.entries()) {
|
|
386
|
+
const last = spans[line.document];
|
|
387
|
+
spans[line.document] = [last ? last[0] : at, at + 1];
|
|
388
|
+
}
|
|
389
|
+
return spans;
|
|
390
|
+
}
|
|
391
|
+
title(at) {
|
|
392
|
+
const line = this.lines[at];
|
|
393
|
+
return ((line && this._places[chunkAt(line, 0)]?.title) || `Document ${(line?.document ?? 0) + 1}`);
|
|
394
|
+
}
|
|
395
|
+
incompleteList(picked) {
|
|
396
|
+
return [...this._listContexts.values()].some((pieces) => pieces.some(({ at, cut }) => newChars(picked.get(at) ?? [], cut) > 0));
|
|
397
|
+
}
|
|
398
|
+
rank(terms, { hints = [], contextual = true } = {}) {
|
|
399
|
+
if (!contextual) {
|
|
400
|
+
return this.lines
|
|
401
|
+
.map((line, at) => ({
|
|
402
|
+
at,
|
|
403
|
+
document: line.document,
|
|
404
|
+
...this._score(line, terms, { anchors: [], contextual: false })
|
|
405
|
+
}))
|
|
406
|
+
.filter((entry) => entry.score > 0)
|
|
407
|
+
.sort((a, b) => b.score - a.score || a.at - b.at);
|
|
408
|
+
}
|
|
409
|
+
const modals = terms.filter((term) => /^~(?:may|can|must|should|shall)$/.test(term));
|
|
410
|
+
const ranked = this.lines.map((line, at) => {
|
|
411
|
+
const expanded = this._withAliases(terms, line.document);
|
|
412
|
+
const anchors = expanded.filter((term) => !term.startsWith(HINT) && (term.includes(' ') || !terms.includes(term)));
|
|
413
|
+
return { at, document: line.document, ...this._score(line, expanded, { anchors }) };
|
|
414
|
+
});
|
|
415
|
+
const sections = this._sectionBounds();
|
|
416
|
+
const evidence = new Map();
|
|
417
|
+
const bonuses = new Map();
|
|
418
|
+
const qualified = new Map();
|
|
419
|
+
for (const { at, score } of ranked) {
|
|
420
|
+
const section = sections[at];
|
|
421
|
+
const line = this.lines[at];
|
|
422
|
+
if (section && score > 0 && line && isSubstantive(line.text) && !line.contents) {
|
|
423
|
+
evidence.set(section, (evidence.get(section) ?? 0) + 1);
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
return ranked
|
|
427
|
+
.filter((entry) => entry.score > 0)
|
|
428
|
+
.map((entry) => {
|
|
429
|
+
const section = sections[entry.at];
|
|
430
|
+
const density = Math.min(3, section ? (evidence.get(section) ?? 0) : 0);
|
|
431
|
+
let hint = section ? bonuses.get(section) : 0;
|
|
432
|
+
if (section && hint === undefined) {
|
|
433
|
+
const context = this.lines
|
|
434
|
+
.slice(...section)
|
|
435
|
+
.map((line) => line.lower)
|
|
436
|
+
.join('\n');
|
|
437
|
+
hint = hints.reduce((sum, word) => sum + termScore(context, this._term(word)) * this._weight(word), 0);
|
|
438
|
+
bonuses.set(section, hint);
|
|
439
|
+
qualified.set(section, hints.some((word) => this._term(word).match.test(this.lines[section[0]]?.lower ?? '')));
|
|
440
|
+
}
|
|
441
|
+
const opening = section
|
|
442
|
+
? this.lines
|
|
443
|
+
.slice(section[0], Math.min(section[1], section[0] + 2))
|
|
444
|
+
.map((line) => line.lower)
|
|
445
|
+
.join(' ')
|
|
446
|
+
: '';
|
|
447
|
+
const modal = modals.some((word) => this._term(word.slice(HINT.length)).match.test(opening));
|
|
448
|
+
return {
|
|
449
|
+
...entry,
|
|
450
|
+
qualified: modal || (entry.anchored && section && qualified.get(section)) ? 1 : 0,
|
|
451
|
+
score: entry.score * (1 + density / 12) + (hint ?? 0) * HINT_WEIGHT
|
|
452
|
+
};
|
|
453
|
+
})
|
|
454
|
+
.sort((a, b) => Number(b.anchored) - Number(a.anchored) ||
|
|
455
|
+
b.qualified - a.qualified ||
|
|
456
|
+
b.score - a.score ||
|
|
457
|
+
a.at - b.at);
|
|
458
|
+
}
|
|
459
|
+
target(start, options) {
|
|
460
|
+
const { chars = SNIPPET_CHARS, room = Infinity } = options;
|
|
461
|
+
let allowance = chars;
|
|
462
|
+
const section = this._sectionBounds()[start];
|
|
463
|
+
let from = start;
|
|
464
|
+
const end = section?.[1] ?? this.lines.length;
|
|
465
|
+
if (section) {
|
|
466
|
+
const size = this.lines.slice(...section).reduce((sum, line) => sum + line.text.length + 1, 0);
|
|
467
|
+
if (size <= chars * 1.5) {
|
|
468
|
+
from = section[0];
|
|
469
|
+
allowance = size;
|
|
470
|
+
}
|
|
471
|
+
else {
|
|
472
|
+
const list = this._listContext(start, { section, chars, room });
|
|
473
|
+
if (list) {
|
|
474
|
+
return list;
|
|
475
|
+
}
|
|
476
|
+
from = this._leadIn(from, section[0], chars);
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
const pieces = this.snippet(from, {
|
|
480
|
+
...options,
|
|
481
|
+
focus: from === start ? options.focus : 0,
|
|
482
|
+
end,
|
|
483
|
+
chars: allowance,
|
|
484
|
+
room
|
|
485
|
+
});
|
|
486
|
+
const complete = completePieces(pieces, this.lines);
|
|
487
|
+
return complete.some(({ at, cut }) => at === start && cut[1] > (options.focus ?? 0))
|
|
488
|
+
? complete
|
|
489
|
+
: pieces;
|
|
490
|
+
}
|
|
491
|
+
_leadIn(from, first, chars) {
|
|
492
|
+
let before = 0;
|
|
493
|
+
while (from > first) {
|
|
494
|
+
const previous = this.lines[from - 1];
|
|
495
|
+
if (!previous?.text.trim() ||
|
|
496
|
+
/[.;!?]\s*$/.test(previous.text) ||
|
|
497
|
+
before + previous.text.length > chars / 3) {
|
|
498
|
+
break;
|
|
499
|
+
}
|
|
500
|
+
before += previous.text.length + 1;
|
|
501
|
+
from--;
|
|
502
|
+
}
|
|
503
|
+
return from;
|
|
504
|
+
}
|
|
505
|
+
_listContext(start, { section, chars, room }) {
|
|
506
|
+
const spans = listLeads(this.lines, section, start);
|
|
507
|
+
if (!spans) {
|
|
508
|
+
return null;
|
|
509
|
+
}
|
|
510
|
+
const whole = completePieces(spans.flatMap(([from, end]) => this.snippet(from, { end, chars: Infinity })), this.lines);
|
|
511
|
+
const cost = whole.reduce((sum, { cut }) => sum + cut[1] - cut[0] + PIECE_OVERHEAD, 0);
|
|
512
|
+
if (cost > chars * 1.5) {
|
|
513
|
+
return null;
|
|
514
|
+
}
|
|
515
|
+
this._listContexts.set(section, whole);
|
|
516
|
+
if (cost <= room) {
|
|
517
|
+
return whole;
|
|
518
|
+
}
|
|
519
|
+
const pieces = [];
|
|
520
|
+
let left = room;
|
|
521
|
+
for (const [from, end] of spans) {
|
|
522
|
+
const part = this.snippet(from, { end, chars: Infinity, room: left });
|
|
523
|
+
pieces.push(...part);
|
|
524
|
+
left -= part.reduce((sum, { cut }) => sum + cut[1] - cut[0] + PIECE_OVERHEAD, 0);
|
|
525
|
+
if (left < MIN_EXCERPT_CHARS) {
|
|
526
|
+
break;
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
return completePieces(pieces, this.lines);
|
|
530
|
+
}
|
|
531
|
+
_withAliases(terms, document) {
|
|
532
|
+
const key = JSON.stringify([document, terms]);
|
|
533
|
+
const known = this._expanded.get(key);
|
|
534
|
+
if (known) {
|
|
535
|
+
return known;
|
|
536
|
+
}
|
|
537
|
+
const aliases = this._aliases.get(document);
|
|
538
|
+
const found = terms.flatMap((term) => term.startsWith(HINT)
|
|
539
|
+
? []
|
|
540
|
+
: variants(normalize(term)).flatMap((form) => aliases?.get(form) ?? []));
|
|
541
|
+
const expanded = [...new Set([...terms, ...found])];
|
|
542
|
+
this._expanded.set(key, expanded);
|
|
543
|
+
return expanded;
|
|
544
|
+
}
|
|
545
|
+
_sectionBounds() {
|
|
546
|
+
if (this._sections) {
|
|
547
|
+
return this._sections;
|
|
548
|
+
}
|
|
549
|
+
const sections = [];
|
|
550
|
+
for (const [start, end] of this.documents()) {
|
|
551
|
+
const lines = this.lines.slice(start, end).map((line) => line.text);
|
|
552
|
+
const body = outlineOf(lines).headings.filter((heading) => !heading.toc);
|
|
553
|
+
const keyed = lines.flatMap((line, at) => /^\s*(?:art(?:icle)?|section|clause)\.?\s+\d+(?:\.\d+)*[.)-]+\s+\S/i.test(line) ? [at] : []);
|
|
554
|
+
const headings = keyed.length >= MIN_LISTED ? keyed : body.map(({ line }) => line);
|
|
555
|
+
const segments = this.lines.flatMap((line, at) => at > start && at < end && line.segment !== this.lines[at - 1]?.segment ? [at] : []);
|
|
556
|
+
const starts = [
|
|
557
|
+
...new Set([start, ...headings.map((line) => start + line), ...segments, end])
|
|
558
|
+
].sort((a, b) => a - b);
|
|
559
|
+
for (let i = 0; i < starts.length - 1; i++) {
|
|
560
|
+
const from = starts[i] ?? start;
|
|
561
|
+
const to = starts[i + 1] ?? end;
|
|
562
|
+
const cut = [from, to];
|
|
563
|
+
for (let at = from; at < to; at++) {
|
|
564
|
+
sections[at] = cut;
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
this._sections = sections;
|
|
569
|
+
return sections;
|
|
570
|
+
}
|
|
571
|
+
sets(at, terms) {
|
|
572
|
+
const line = at === undefined ? undefined : this.lines[at];
|
|
573
|
+
return (!!line &&
|
|
574
|
+
terms.some((term) => !term.startsWith(HINT) &&
|
|
575
|
+
termScore(normalize(line.text.slice(0, SNIPPET_CHARS)), this._term(term), {
|
|
576
|
+
contents: line.contents
|
|
577
|
+
}) >= HEADING_PREFIX_BOOST));
|
|
578
|
+
}
|
|
579
|
+
snippet(start, { focus = 0, end = this.lines.length, chars = SNIPPET_CHARS, room = Infinity } = {}) {
|
|
580
|
+
const first = this.lines[start];
|
|
581
|
+
const definition = DEFINES.test(first?.text ?? '');
|
|
582
|
+
const pieces = [];
|
|
583
|
+
let used = 0;
|
|
584
|
+
let left = room;
|
|
585
|
+
for (let at = start; at < end && (used < chars || at === start); at++) {
|
|
586
|
+
const line = this.lines[at];
|
|
587
|
+
if (!line || line.document !== first?.document || line.segment !== first?.segment) {
|
|
588
|
+
break;
|
|
589
|
+
}
|
|
590
|
+
if (definition && at > start && DEFINES.test(line.text)) {
|
|
591
|
+
break;
|
|
592
|
+
}
|
|
593
|
+
if (!line.text.trim() || this._muted.has(at)) {
|
|
594
|
+
continue;
|
|
595
|
+
}
|
|
596
|
+
const size = at === start ? chars : Math.max(chars - used, MIN_WINDOW_CHARS);
|
|
597
|
+
const fit = Math.min(size, left - PIECE_OVERHEAD);
|
|
598
|
+
if (fit < size && fit < MIN_CLIP) {
|
|
599
|
+
break;
|
|
600
|
+
}
|
|
601
|
+
const cut = windowAround(line.text, at === start ? focus : 0, fit);
|
|
602
|
+
pieces.push({ at, cut, document: line.document, focus: at === start ? focus : cut[0] });
|
|
603
|
+
used += cut[1] - cut[0] + 1;
|
|
604
|
+
left -= cut[1] - cut[0] + PIECE_OVERHEAD;
|
|
605
|
+
}
|
|
606
|
+
return pieces;
|
|
607
|
+
}
|
|
608
|
+
excerpts(picked, { span = Infinity, marks = new Map() } = {}) {
|
|
609
|
+
const merged = [];
|
|
610
|
+
let previous = -1;
|
|
611
|
+
for (const [at, cuts] of [...picked].sort(([a], [b]) => a - b)) {
|
|
612
|
+
const line = this.lines[at];
|
|
613
|
+
for (const [n, cut] of cuts.entries()) {
|
|
614
|
+
const match = marks.get(at)?.find((mark) => mark >= cut[0] && mark < cut[1]) ?? cut[0];
|
|
615
|
+
const owner = line ? chunkAt(line, match) : -1;
|
|
616
|
+
const hit = this._hits[owner];
|
|
617
|
+
const place = this._places[owner];
|
|
618
|
+
if (!line || !hit || !place) {
|
|
619
|
+
continue;
|
|
620
|
+
}
|
|
621
|
+
const index = place.chunkIndex;
|
|
622
|
+
const last = merged[merged.length - 1];
|
|
623
|
+
const text = render(line.text, cut);
|
|
624
|
+
const apart = n > 0 || (previous >= 0 && !this._blank(previous + 1, at));
|
|
625
|
+
const gap = apart ? `\n${GAP}\n` : '\n';
|
|
626
|
+
if (last &&
|
|
627
|
+
last.document === line.document &&
|
|
628
|
+
last.page === place.page &&
|
|
629
|
+
last.segment === line.segment &&
|
|
630
|
+
(index === null || (index - last.last <= MERGE_GAP && index - last.first < span))) {
|
|
631
|
+
last.hit = { ...last.hit, content: last.hit.content + gap + text };
|
|
632
|
+
last.last = index ?? last.last;
|
|
633
|
+
}
|
|
634
|
+
else {
|
|
635
|
+
merged.push({
|
|
636
|
+
hit: { ...hit, content: text },
|
|
637
|
+
first: index ?? 0,
|
|
638
|
+
last: index ?? 0,
|
|
639
|
+
document: line.document,
|
|
640
|
+
page: place.page,
|
|
641
|
+
segment: line.segment
|
|
642
|
+
});
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
previous = at;
|
|
646
|
+
}
|
|
647
|
+
return merged.map(({ hit }) => hit);
|
|
648
|
+
}
|
|
649
|
+
offset(at, index) {
|
|
650
|
+
const raw = this.lines[at]?.text ?? '';
|
|
651
|
+
if (/^[\u0000-\u007f]*$/.test(raw)) {
|
|
652
|
+
return index;
|
|
653
|
+
}
|
|
654
|
+
let seen = 0;
|
|
655
|
+
for (const { 0: cluster, index: start = 0 } of raw.matchAll(/\P{M}\p{M}*/gu)) {
|
|
656
|
+
seen += normalize(cluster).length;
|
|
657
|
+
if (seen > index) {
|
|
658
|
+
return start;
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
return raw.length;
|
|
662
|
+
}
|
|
663
|
+
_blank(from, to) {
|
|
664
|
+
return this.lines
|
|
665
|
+
.slice(from, to)
|
|
666
|
+
.every((line, i) => !line.text.trim() || this._muted.has(from + i));
|
|
667
|
+
}
|
|
668
|
+
_stitch(document, members) {
|
|
669
|
+
const ordered = [...members].sort((a, b) => (this._places[a]?.chunkIndex ?? a) - (this._places[b]?.chunkIndex ?? b));
|
|
670
|
+
let text = '';
|
|
671
|
+
const starts = [];
|
|
672
|
+
let before;
|
|
673
|
+
let segment = 0;
|
|
674
|
+
for (const at of ordered) {
|
|
675
|
+
const hit = this._hits[at];
|
|
676
|
+
const place = this._places[at];
|
|
677
|
+
if (!hit || !place) {
|
|
678
|
+
continue;
|
|
679
|
+
}
|
|
680
|
+
const previous = before === undefined ? undefined : this._hits[before];
|
|
681
|
+
const previousPlace = before === undefined ? undefined : this._places[before];
|
|
682
|
+
const follows = previousPlace?.chunkIndex !== null &&
|
|
683
|
+
previousPlace !== undefined &&
|
|
684
|
+
place.chunkIndex === previousPlace.chunkIndex + 1;
|
|
685
|
+
const samePage = previousPlace?.page === place.page;
|
|
686
|
+
const shared = follows && samePage && previous ? overlap(previous.content, hit.content) : 0;
|
|
687
|
+
if (!follows) {
|
|
688
|
+
segment++;
|
|
689
|
+
}
|
|
690
|
+
const joint = text && (!follows || !samePage || shared === 0) ? '\n' : '';
|
|
691
|
+
starts.push({
|
|
692
|
+
from: text.length + joint.length,
|
|
693
|
+
owner: at,
|
|
694
|
+
segment,
|
|
695
|
+
contents: isContentsPage(hit.content)
|
|
696
|
+
});
|
|
697
|
+
text += joint + hit.content.slice(shared);
|
|
698
|
+
before = at;
|
|
699
|
+
}
|
|
700
|
+
this._aliases.set(document, acronymBindings(text));
|
|
701
|
+
let offset = 0;
|
|
702
|
+
let cursor = 0;
|
|
703
|
+
for (const raw of text.split('\n')) {
|
|
704
|
+
while ((starts[cursor + 1]?.from ?? Infinity) <= offset) {
|
|
705
|
+
cursor++;
|
|
706
|
+
}
|
|
707
|
+
const chunk = starts[cursor];
|
|
708
|
+
this.lines.push({
|
|
709
|
+
text: raw,
|
|
710
|
+
lower: normalize(raw),
|
|
711
|
+
document,
|
|
712
|
+
segment: chunk?.segment ?? 0,
|
|
713
|
+
contents: chunk?.contents ?? false,
|
|
714
|
+
origin: offset,
|
|
715
|
+
chunks: starts
|
|
716
|
+
});
|
|
717
|
+
offset += raw.length + 1;
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
_score(line, terms, { anchors, contextual = true }) {
|
|
721
|
+
let score = 0;
|
|
722
|
+
let best = 0;
|
|
723
|
+
let focus = 0;
|
|
724
|
+
let own = false;
|
|
725
|
+
const phrases = terms.filter((term) => !term.startsWith(HINT) && term.includes(' '));
|
|
726
|
+
const partialDefinition = contextual &&
|
|
727
|
+
DEFINES.test(line.text) &&
|
|
728
|
+
phrases.length > 0 &&
|
|
729
|
+
!phrases.some((phrase) => termScore(line.lower, this._term(phrase), { contents: line.contents }) >= KIND_BOOST);
|
|
730
|
+
for (const term of terms) {
|
|
731
|
+
const hint = term.startsWith(HINT);
|
|
732
|
+
const word = hint ? term.slice(HINT.length) : term;
|
|
733
|
+
const termInfo = this._term(word);
|
|
734
|
+
const enumerates = contextual &&
|
|
735
|
+
!line.contents &&
|
|
736
|
+
termInfo.forms.some((form) => line.lower.includes(`following ${form}`));
|
|
737
|
+
const match = Math.max(termScore(line.lower, termInfo, { contents: line.contents }), enumerates ? HEADING_BOOST : 0);
|
|
738
|
+
const matched = (partialDefinition ? Math.min(1, match) : match) * this._weight(word);
|
|
739
|
+
const weighed = hint ? matched * HINT_WEIGHT : matched;
|
|
740
|
+
score += weighed;
|
|
741
|
+
own ||= !hint && matched > 0;
|
|
742
|
+
if (weighed > best) {
|
|
743
|
+
best = weighed;
|
|
744
|
+
focus = Math.max(0, this._term(word).match.exec(line.lower)?.index ?? 0);
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
const anchored = anchors.length === 0 || anchors.some((term) => this._term(term).match.test(line.lower));
|
|
748
|
+
return { score: own || !contextual ? score : 0, focus, anchored };
|
|
749
|
+
}
|
|
750
|
+
_weight(term) {
|
|
751
|
+
const known = this._weights.get(term);
|
|
752
|
+
if (known !== undefined) {
|
|
753
|
+
return known;
|
|
754
|
+
}
|
|
755
|
+
const matcher = this._term(term).match;
|
|
756
|
+
const matched = this.lines.filter((line) => matcher.test(line.lower));
|
|
757
|
+
const weight = matched.length === 0 ? 0 : Math.log(1 + this.lines.length / matched.length);
|
|
758
|
+
this._weights.set(term, weight);
|
|
759
|
+
return weight;
|
|
760
|
+
}
|
|
761
|
+
_term(word) {
|
|
762
|
+
let term = this._terms.get(word);
|
|
763
|
+
if (!term) {
|
|
764
|
+
const forms = variants(normalize(word));
|
|
765
|
+
const boundary = /^[\d.,]+$/.test(word) ? '\\p{L}\\p{N}\\p{M}_,.' : '\\p{L}\\p{N}\\p{M}_';
|
|
766
|
+
const pattern = forms.map((form) => form.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('|');
|
|
767
|
+
term = {
|
|
768
|
+
forms,
|
|
769
|
+
match: new RegExp(`(?:^|[^${boundary}])(?:${pattern})(?=$|[^${boundary}])`, 'u')
|
|
770
|
+
};
|
|
771
|
+
this._terms.set(word, term);
|
|
772
|
+
}
|
|
773
|
+
return term;
|
|
774
|
+
}
|
|
775
|
+
}
|
|
776
|
+
const GAP = '…';
|
|
777
|
+
const PIECE_OVERHEAD = 5;
|
|
778
|
+
const MIN_EXCERPT_CHARS = MIN_CLIP + PIECE_OVERHEAD;
|
|
779
|
+
const TARGET_CHARS = 1600;
|
|
780
|
+
const MERGE_GAP = 40;
|
|
781
|
+
const MAX_OVERLAP_CHARS = 1200;
|
|
782
|
+
const ALIAS_SMALL = new Set('a an and as at by for from in of on or the to with'.split(' '));
|
|
783
|
+
function acronymBindings(text) {
|
|
784
|
+
const aliases = new Map();
|
|
785
|
+
const meanings = new Map();
|
|
786
|
+
const ambiguous = new Set();
|
|
787
|
+
for (const { 1: short = '', index = 0 } of text.matchAll(/\(([A-Z][A-Z0-9]{1,7})s?\)/g)) {
|
|
788
|
+
const before = text.slice(Math.max(0, index - 160), index).trimEnd();
|
|
789
|
+
const words = before.match(/[\p{L}][\p{L}\p{M}'’-]*/gu) ?? [];
|
|
790
|
+
for (let at = words.length - 2; at >= Math.max(0, words.length - 10); at--) {
|
|
791
|
+
const phrase = words.slice(at);
|
|
792
|
+
const initials = phrase
|
|
793
|
+
.filter((word) => !ALIAS_SMALL.has(word.toLowerCase()))
|
|
794
|
+
.map((word) => word[0])
|
|
795
|
+
.join('')
|
|
796
|
+
.toUpperCase();
|
|
797
|
+
if (initials !== short) {
|
|
798
|
+
continue;
|
|
799
|
+
}
|
|
800
|
+
const abbreviation = short.toLowerCase();
|
|
801
|
+
const forms = variants(normalize(phrase.join(' ')));
|
|
802
|
+
const previous = meanings.get(abbreviation);
|
|
803
|
+
if (previous && !previous.some((form) => forms.includes(form))) {
|
|
804
|
+
ambiguous.add(abbreviation);
|
|
805
|
+
}
|
|
806
|
+
meanings.set(abbreviation, forms);
|
|
807
|
+
for (const form of forms) {
|
|
808
|
+
aliases.set(form, abbreviation);
|
|
809
|
+
}
|
|
810
|
+
break;
|
|
811
|
+
}
|
|
812
|
+
}
|
|
813
|
+
for (const [phrase, abbreviation] of aliases) {
|
|
814
|
+
if (ambiguous.has(abbreviation)) {
|
|
815
|
+
aliases.delete(phrase);
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
return aliases;
|
|
819
|
+
}
|
|
820
|
+
function completePieces(pieces, lines) {
|
|
821
|
+
const size = pieces.reduce((sum, { cut }) => sum + cut[1] - cut[0], 0);
|
|
822
|
+
let seen = 0;
|
|
823
|
+
let ending = null;
|
|
824
|
+
for (const [piece, { at, cut }] of pieces.entries()) {
|
|
825
|
+
const text = lines[at]?.text.slice(...cut) ?? '';
|
|
826
|
+
for (const { 0: mark, index = 0 } of text.matchAll(/[.;!?]["”')\]]*(?=\s|$)/g)) {
|
|
827
|
+
ending = { piece, at: cut[0] + index + mark.length, size: seen + index + mark.length };
|
|
828
|
+
}
|
|
829
|
+
seen += text.length;
|
|
830
|
+
}
|
|
831
|
+
const last = ending;
|
|
832
|
+
if (!last || last.size < size / 2) {
|
|
833
|
+
return pieces;
|
|
834
|
+
}
|
|
835
|
+
return pieces
|
|
836
|
+
.slice(0, last.piece + 1)
|
|
837
|
+
.map((piece, at) => at === last.piece ? { ...piece, cut: [piece.cut[0], last.at] } : piece);
|
|
838
|
+
}
|
|
839
|
+
function listLeads(lines, section, start) {
|
|
840
|
+
if (!/\bfollowing\b/.test(lines[start]?.lower ?? '')) {
|
|
841
|
+
return null;
|
|
842
|
+
}
|
|
843
|
+
const markers = lines.slice(start + 1, section[1]).flatMap((line, index) => {
|
|
844
|
+
const match = /^\s*(?:([a-z]|\d{1,3})([.)])|([•*-]))\s+/i.exec(line.text);
|
|
845
|
+
return match
|
|
846
|
+
? [
|
|
847
|
+
{
|
|
848
|
+
at: start + 1 + index,
|
|
849
|
+
label: (match[1] ?? '').toLowerCase(),
|
|
850
|
+
mark: match[2] ?? match[3]
|
|
851
|
+
}
|
|
852
|
+
]
|
|
853
|
+
: [];
|
|
854
|
+
});
|
|
855
|
+
const first = markers[0];
|
|
856
|
+
if (!first || (first.label !== 'a' && first.label !== '1')) {
|
|
857
|
+
return null;
|
|
858
|
+
}
|
|
859
|
+
const letters = first.label === 'a';
|
|
860
|
+
const own = markers.filter(({ label, mark }) => /^[a-z]$/.test(label) === letters && label !== '' && mark === first.mark);
|
|
861
|
+
if (own.length < MIN_LISTED ||
|
|
862
|
+
own.some(({ label }, at) => (letters ? label.charCodeAt(0) - 97 : Number(label) - 1) !== at)) {
|
|
863
|
+
return null;
|
|
864
|
+
}
|
|
865
|
+
return [
|
|
866
|
+
[section[0], first.at],
|
|
867
|
+
...own.map(({ at }, index) => {
|
|
868
|
+
const end = own[index + 1]?.at ?? section[1];
|
|
869
|
+
const nested = markers.find((marker) => marker.at > at && marker.at < end);
|
|
870
|
+
return [at, nested?.at ?? end];
|
|
871
|
+
})
|
|
872
|
+
];
|
|
873
|
+
}
|
|
874
|
+
function overlap(first, second) {
|
|
875
|
+
for (let size = Math.min(first.length, second.length, MAX_OVERLAP_CHARS); size > 0; size--) {
|
|
876
|
+
if (first.endsWith(second.slice(0, size))) {
|
|
877
|
+
return size;
|
|
878
|
+
}
|
|
879
|
+
}
|
|
880
|
+
return 0;
|
|
881
|
+
}
|
|
882
|
+
function chunkAt({ origin, chunks }, offset) {
|
|
883
|
+
const at = origin + offset;
|
|
884
|
+
let low = 0;
|
|
885
|
+
let high = chunks.length - 1;
|
|
886
|
+
while (low < high) {
|
|
887
|
+
const middle = Math.ceil((low + high) / 2);
|
|
888
|
+
if ((chunks[middle]?.from ?? Infinity) <= at) {
|
|
889
|
+
low = middle;
|
|
890
|
+
}
|
|
891
|
+
else {
|
|
892
|
+
high = middle - 1;
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
return chunks[low]?.owner ?? -1;
|
|
896
|
+
}
|
|
897
|
+
class Field {
|
|
898
|
+
_ranked;
|
|
899
|
+
_taken = new Set();
|
|
900
|
+
_from = 0;
|
|
901
|
+
constructor(ranked) {
|
|
902
|
+
this._ranked = ranked;
|
|
903
|
+
}
|
|
904
|
+
get empty() {
|
|
905
|
+
this._skipTaken();
|
|
906
|
+
return this._from >= this._ranked.length;
|
|
907
|
+
}
|
|
908
|
+
next(quoted) {
|
|
909
|
+
this._skipTaken();
|
|
910
|
+
const best = this._ranked[this._from];
|
|
911
|
+
if (!best) {
|
|
912
|
+
return undefined;
|
|
913
|
+
}
|
|
914
|
+
const floor = best.score * NEAR_TIE;
|
|
915
|
+
let chosen = this._from;
|
|
916
|
+
for (const [i, line] of this._ranked.slice(this._from).entries()) {
|
|
917
|
+
const at = this._from + i;
|
|
918
|
+
if (line.score < floor) {
|
|
919
|
+
break;
|
|
920
|
+
}
|
|
921
|
+
if (!this._taken.has(at) && !quoted.has(line.document)) {
|
|
922
|
+
chosen = at;
|
|
923
|
+
break;
|
|
924
|
+
}
|
|
925
|
+
}
|
|
926
|
+
this._taken.add(chosen);
|
|
927
|
+
return this._ranked[chosen];
|
|
928
|
+
}
|
|
929
|
+
_skipTaken() {
|
|
930
|
+
while (this._taken.has(this._from)) {
|
|
931
|
+
this._from++;
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
}
|
|
935
|
+
class Picker {
|
|
936
|
+
lines = new Map();
|
|
937
|
+
marks = new Map();
|
|
938
|
+
documents = new Set();
|
|
939
|
+
_left;
|
|
940
|
+
constructor(budgetChars) {
|
|
941
|
+
this._left = budgetChars;
|
|
942
|
+
}
|
|
943
|
+
get full() {
|
|
944
|
+
return this.lines.size > 0 && this._left < MIN_EXCERPT_CHARS;
|
|
945
|
+
}
|
|
946
|
+
get remaining() {
|
|
947
|
+
return this._left;
|
|
948
|
+
}
|
|
949
|
+
fit(snippet, limit = Infinity) {
|
|
950
|
+
const whole = snippet(Infinity);
|
|
951
|
+
const cost = this._cost(whole);
|
|
952
|
+
const room = Math.min(this._left, limit);
|
|
953
|
+
if (cost <= room) {
|
|
954
|
+
return whole;
|
|
955
|
+
}
|
|
956
|
+
const clipped = snippet(room);
|
|
957
|
+
const target = limit === Infinity ? cost : Math.min(cost, SNIPPET_CHARS);
|
|
958
|
+
return this.lines.size === 0 || this._cost(clipped) * 2 >= target ? clipped : [];
|
|
959
|
+
}
|
|
960
|
+
take(pieces) {
|
|
961
|
+
const cost = this._cost(pieces);
|
|
962
|
+
if (cost > this._left) {
|
|
963
|
+
return null;
|
|
964
|
+
}
|
|
965
|
+
for (const { at, cut, document, focus } of pieces) {
|
|
966
|
+
this.lines.set(at, unionOf(this.lines.get(at) ?? [], cut));
|
|
967
|
+
this.marks.set(at, [...(this.marks.get(at) ?? []), focus]);
|
|
968
|
+
this.documents.add(document);
|
|
969
|
+
}
|
|
970
|
+
this._left -= cost;
|
|
971
|
+
return cost;
|
|
972
|
+
}
|
|
973
|
+
_cost(pieces) {
|
|
974
|
+
return pieces.reduce((sum, piece) => sum + this._added(piece), 0);
|
|
975
|
+
}
|
|
976
|
+
_added({ at, cut }) {
|
|
977
|
+
const known = this.lines.get(at) ?? [];
|
|
978
|
+
const grown = newChars(known, cut);
|
|
979
|
+
return grown > 0 ? grown + PIECE_OVERHEAD : 0;
|
|
980
|
+
}
|
|
981
|
+
}
|
|
982
|
+
function placeOf(json, at) {
|
|
983
|
+
let fields = new Map();
|
|
984
|
+
try {
|
|
985
|
+
const parsed = JSON.parse(json);
|
|
986
|
+
if (parsed && typeof parsed === 'object') {
|
|
987
|
+
fields = new Map(Object.entries(parsed));
|
|
988
|
+
}
|
|
989
|
+
}
|
|
990
|
+
catch { }
|
|
991
|
+
const documentId = fields.get('documentId');
|
|
992
|
+
const chunkIndex = fields.get('chunkIndex');
|
|
993
|
+
const page = fields.get('page');
|
|
994
|
+
const title = fields.get('title');
|
|
995
|
+
return {
|
|
996
|
+
documentId: typeof documentId === 'string' && documentId ? documentId : `\u0000${at}`,
|
|
997
|
+
chunkIndex: typeof chunkIndex === 'number' && Number.isInteger(chunkIndex) ? chunkIndex : null,
|
|
998
|
+
page: typeof page === 'number' && Number.isInteger(page) ? page : null,
|
|
999
|
+
title: typeof title === 'string' ? title : null
|
|
1000
|
+
};
|
|
1001
|
+
}
|
|
1002
|
+
function termScore(text, { forms, match }, { contents = false } = {}) {
|
|
1003
|
+
if (!match.test(text)) {
|
|
1004
|
+
return 0;
|
|
1005
|
+
}
|
|
1006
|
+
if (forms.some((form) => text.includes(`"${form}" means`) || text.includes(`“${form}” means`))) {
|
|
1007
|
+
return DEFINITION_BOOST;
|
|
1008
|
+
}
|
|
1009
|
+
if (forms.some((form) => text.includes(`${form}" means`) || text.includes(`${form}” means`))) {
|
|
1010
|
+
return KIND_BOOST;
|
|
1011
|
+
}
|
|
1012
|
+
if (contents) {
|
|
1013
|
+
return 1;
|
|
1014
|
+
}
|
|
1015
|
+
const heading = headed(text, { forms, match });
|
|
1016
|
+
return heading || 1;
|
|
1017
|
+
}
|
|
1018
|
+
function variants(term) {
|
|
1019
|
+
if (!/^[a-z]+(?: [a-z]+)*$/.test(term)) {
|
|
1020
|
+
return [term];
|
|
1021
|
+
}
|
|
1022
|
+
if (term.endsWith('ies')) {
|
|
1023
|
+
return [term, `${term.slice(0, -3)}y`];
|
|
1024
|
+
}
|
|
1025
|
+
if (/(?:ches|shes|xes|zes|sses)$/.test(term)) {
|
|
1026
|
+
return [term, term.slice(0, -2)];
|
|
1027
|
+
}
|
|
1028
|
+
if (term.endsWith('s') && !/(?:ss|us|is)$/.test(term)) {
|
|
1029
|
+
return [term, term.slice(0, -1)];
|
|
1030
|
+
}
|
|
1031
|
+
if (/[^aeiou]y$/.test(term)) {
|
|
1032
|
+
return [term, `${term.slice(0, -1)}ies`];
|
|
1033
|
+
}
|
|
1034
|
+
return [term, /(?:ch|sh|s|x|z)$/.test(term) ? `${term}es` : `${term}s`];
|
|
1035
|
+
}
|
|
1036
|
+
const NUMBERED = /^\s*(?:(?:art(?:icle)?|section|clause|rule)\.?\s*)?\d+(?:\.\d+)*\s*[.)-]*\s*-?\s*/;
|
|
1037
|
+
const OPENING_CHARS = 60;
|
|
1038
|
+
function headed(text, { forms, match }) {
|
|
1039
|
+
let boost = 0;
|
|
1040
|
+
for (const raw of text.split('\n')) {
|
|
1041
|
+
const numbered = NUMBERED.exec(raw);
|
|
1042
|
+
const line = raw.slice(numbered?.[0].length ?? 0).trim();
|
|
1043
|
+
if (forms.includes(line)) {
|
|
1044
|
+
return HEADING_BOOST;
|
|
1045
|
+
}
|
|
1046
|
+
const short = !line.endsWith('.') && line.split(/\s+/).length <= HEADING_WORDS;
|
|
1047
|
+
if ((short &&
|
|
1048
|
+
forms.some((form) => line.startsWith(form) &&
|
|
1049
|
+
!/[\p{L}\p{N}\p{M}_]/u.test(line.slice(form.length, form.length + 1)))) ||
|
|
1050
|
+
(numbered && match.test(line.slice(0, OPENING_CHARS)))) {
|
|
1051
|
+
boost = HEADING_PREFIX_BOOST;
|
|
1052
|
+
}
|
|
1053
|
+
}
|
|
1054
|
+
return boost;
|
|
1055
|
+
}
|
|
1056
|
+
function isContentsPage(text) {
|
|
1057
|
+
return text.split('\n').filter((line) => /\s\d+\s*$/.test(line)).length > CONTENTS_LINES;
|
|
1058
|
+
}
|