@adonis-agora/agent 0.35.0 → 0.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/NOTICE +31 -0
- package/README.md +5 -5
- package/dist/configure.d.ts.map +1 -1
- package/dist/configure.js +1 -0
- package/dist/configure.js.map +1 -1
- package/dist/src/index.d.ts +1 -1
- package/dist/src/index.js +1 -1
- package/dist/src/rag/index.d.ts +3 -1
- package/dist/src/rag/index.d.ts.map +1 -1
- package/dist/src/rag/index.js +3 -1
- package/dist/src/rag/index.js.map +1 -1
- package/dist/src/rag/lexical-query.d.ts +28 -0
- package/dist/src/rag/lexical-query.d.ts.map +1 -0
- package/dist/src/rag/lexical-query.js +90 -0
- package/dist/src/rag/lexical-query.js.map +1 -0
- package/dist/src/rag/openai-embeddings.d.ts +20 -2
- package/dist/src/rag/openai-embeddings.d.ts.map +1 -1
- package/dist/src/rag/openai-embeddings.js +107 -42
- package/dist/src/rag/openai-embeddings.js.map +1 -1
- package/dist/src/rag/pg-vector-store.d.ts +56 -7
- package/dist/src/rag/pg-vector-store.d.ts.map +1 -1
- package/dist/src/rag/pg-vector-store.js +65 -17
- package/dist/src/rag/pg-vector-store.js.map +1 -1
- package/dist/src/rag/tree/build.d.ts +96 -0
- package/dist/src/rag/tree/build.d.ts.map +1 -0
- package/dist/src/rag/tree/build.js +659 -0
- package/dist/src/rag/tree/build.js.map +1 -0
- package/dist/src/rag/tree/headings.d.ts +29 -0
- package/dist/src/rag/tree/headings.d.ts.map +1 -0
- package/dist/src/rag/tree/headings.js +227 -0
- package/dist/src/rag/tree/headings.js.map +1 -0
- package/dist/src/rag/tree/index-document.d.ts +33 -0
- package/dist/src/rag/tree/index-document.d.ts.map +1 -0
- package/dist/src/rag/tree/index-document.js +36 -0
- package/dist/src/rag/tree/index-document.js.map +1 -0
- package/dist/src/rag/tree/index.d.ts +13 -0
- package/dist/src/rag/tree/index.d.ts.map +1 -0
- package/dist/src/rag/tree/index.js +12 -0
- package/dist/src/rag/tree/index.js.map +1 -0
- package/dist/src/rag/tree/keyword-tree-llm.d.ts +15 -0
- package/dist/src/rag/tree/keyword-tree-llm.d.ts.map +1 -0
- package/dist/src/rag/tree/keyword-tree-llm.js +105 -0
- package/dist/src/rag/tree/keyword-tree-llm.js.map +1 -0
- package/dist/src/rag/tree/llm.d.ts +179 -0
- package/dist/src/rag/tree/llm.d.ts.map +1 -0
- package/dist/src/rag/tree/llm.js +264 -0
- package/dist/src/rag/tree/llm.js.map +1 -0
- package/dist/src/rag/tree/navigate-tool.d.ts +58 -0
- package/dist/src/rag/tree/navigate-tool.d.ts.map +1 -0
- package/dist/src/rag/tree/navigate-tool.js +95 -0
- package/dist/src/rag/tree/navigate-tool.js.map +1 -0
- package/dist/src/rag/tree/navigate.d.ts +137 -0
- package/dist/src/rag/tree/navigate.d.ts.map +1 -0
- package/dist/src/rag/tree/navigate.js +404 -0
- package/dist/src/rag/tree/navigate.js.map +1 -0
- package/dist/src/rag/tree/pg-tree-store.d.ts +45 -0
- package/dist/src/rag/tree/pg-tree-store.d.ts.map +1 -0
- package/dist/src/rag/tree/pg-tree-store.js +205 -0
- package/dist/src/rag/tree/pg-tree-store.js.map +1 -0
- package/dist/src/rag/tree/store.d.ts +57 -0
- package/dist/src/rag/tree/store.d.ts.map +1 -0
- package/dist/src/rag/tree/store.js +66 -0
- package/dist/src/rag/tree/store.js.map +1 -0
- package/dist/src/rag/tree/two-stage.d.ts +54 -0
- package/dist/src/rag/tree/two-stage.d.ts.map +1 -0
- package/dist/src/rag/tree/two-stage.js +92 -0
- package/dist/src/rag/tree/two-stage.js.map +1 -0
- package/dist/src/rag/tree/types.d.ts +135 -0
- package/dist/src/rag/tree/types.d.ts.map +1 -0
- package/dist/src/rag/tree/types.js +21 -0
- package/dist/src/rag/tree/types.js.map +1 -0
- package/dist/stubs/database/migrations/create_agent_rag_trees.stub +40 -0
- package/package.json +3 -2
|
@@ -0,0 +1,659 @@
|
|
|
1
|
+
import { detectHeadings, runningLines } from './headings.js';
|
|
2
|
+
import { BudgetedLlm, parseJsonReply, stableHash, TreeBudgetExceededError, } from './llm.js';
|
|
3
|
+
import { walkTree } from './types.js';
|
|
4
|
+
const DEFAULT_BUDGET = {
|
|
5
|
+
maxCalls: 60,
|
|
6
|
+
maxInputTokens: 400_000,
|
|
7
|
+
maxOutputTokens: 40_000,
|
|
8
|
+
timeoutMs: 300_000,
|
|
9
|
+
callTimeoutMs: 60_000,
|
|
10
|
+
};
|
|
11
|
+
const STRUCTURE_SYSTEM = 'You extract the table of contents of a document. You answer with JSON only, no prose.';
|
|
12
|
+
const SUMMARY_SYSTEM = 'You summarize sections of a document so a reader can decide whether a section answers their question. Be specific: name the topics, clauses, figures and defined terms it covers.';
|
|
13
|
+
/** Turn the input into units (pages or sections) plus any headings it declares. */
|
|
14
|
+
export function toUnits(input) {
|
|
15
|
+
if (input.pages !== undefined && input.sections !== undefined) {
|
|
16
|
+
throw new Error('buildDocumentTree: give `pages` or `sections`, not both');
|
|
17
|
+
}
|
|
18
|
+
if (input.sections !== undefined) {
|
|
19
|
+
const units = input.sections.map((section, index) => ({
|
|
20
|
+
index,
|
|
21
|
+
text: section.text,
|
|
22
|
+
...(section.page !== undefined ? { page: section.page } : {}),
|
|
23
|
+
}));
|
|
24
|
+
const titled = input.sections.some((section) => section.title?.trim());
|
|
25
|
+
const declared = titled
|
|
26
|
+
? input.sections.flatMap((section, unit) => section.title?.trim()
|
|
27
|
+
? [
|
|
28
|
+
{
|
|
29
|
+
title: section.title.trim(),
|
|
30
|
+
level: Math.max(1, Math.min(6, section.level ?? 1)),
|
|
31
|
+
unit,
|
|
32
|
+
atUnitStart: true,
|
|
33
|
+
},
|
|
34
|
+
]
|
|
35
|
+
: [])
|
|
36
|
+
: undefined;
|
|
37
|
+
return { units, declared, kind: 'sections' };
|
|
38
|
+
}
|
|
39
|
+
const pages = input.pages ?? [];
|
|
40
|
+
return {
|
|
41
|
+
units: pages.map((text, index) => ({ index, text, page: index + 1 })),
|
|
42
|
+
declared: undefined,
|
|
43
|
+
kind: 'pages',
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
function outlineHeadings(input, units) {
|
|
47
|
+
const headings = [];
|
|
48
|
+
for (const entry of input.outline ?? []) {
|
|
49
|
+
let unit = entry.unit;
|
|
50
|
+
if (unit === undefined && entry.page !== undefined) {
|
|
51
|
+
unit = units.findIndex((candidate) => candidate.page === entry.page);
|
|
52
|
+
if (unit < 0) {
|
|
53
|
+
unit = entry.page - 1;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
if (unit === undefined || unit < 0 || unit >= units.length || !entry.title.trim()) {
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
headings.push({
|
|
60
|
+
title: entry.title.trim(),
|
|
61
|
+
level: Math.max(1, Math.min(6, Math.floor(entry.level))),
|
|
62
|
+
unit,
|
|
63
|
+
atUnitStart: false,
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
// Outlines are usually ordered, but some PDFs list bookmarks out of page order.
|
|
67
|
+
return headings.sort((a, b) => a.unit - b.unit);
|
|
68
|
+
}
|
|
69
|
+
/** Nest headings by level (a stack, like PageIndex's markdown tree) and compute each span. */
|
|
70
|
+
function nestHeadings(headings, unitCount) {
|
|
71
|
+
const roots = [];
|
|
72
|
+
const stack = [];
|
|
73
|
+
headings.forEach((heading, index) => {
|
|
74
|
+
// The node ends where the next heading of the same or a higher level starts.
|
|
75
|
+
let end = unitCount - 1;
|
|
76
|
+
for (let next = index + 1; next < headings.length; next++) {
|
|
77
|
+
const candidate = headings[next];
|
|
78
|
+
if (candidate !== undefined && candidate.level <= heading.level) {
|
|
79
|
+
end = candidate.atUnitStart ? candidate.unit - 1 : candidate.unit;
|
|
80
|
+
break;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
const node = {
|
|
84
|
+
title: heading.title,
|
|
85
|
+
unitStart: heading.unit,
|
|
86
|
+
unitEnd: Math.max(heading.unit, end),
|
|
87
|
+
children: [],
|
|
88
|
+
};
|
|
89
|
+
while (stack.length > 0 && (stack[stack.length - 1]?.level ?? 0) >= heading.level) {
|
|
90
|
+
stack.pop();
|
|
91
|
+
}
|
|
92
|
+
const parent = stack[stack.length - 1];
|
|
93
|
+
if (parent === undefined) {
|
|
94
|
+
roots.push(node);
|
|
95
|
+
}
|
|
96
|
+
else {
|
|
97
|
+
parent.node.children.push(node);
|
|
98
|
+
parent.node.unitEnd = Math.max(parent.node.unitEnd, node.unitEnd);
|
|
99
|
+
}
|
|
100
|
+
stack.push({ node, level: heading.level });
|
|
101
|
+
});
|
|
102
|
+
// Front matter before the first heading stays reachable.
|
|
103
|
+
const first = roots[0];
|
|
104
|
+
if (first !== undefined && first.unitStart > 0) {
|
|
105
|
+
roots.unshift({
|
|
106
|
+
title: 'Front matter',
|
|
107
|
+
unitStart: 0,
|
|
108
|
+
unitEnd: first.unitStart - 1,
|
|
109
|
+
children: [],
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
return roots;
|
|
113
|
+
}
|
|
114
|
+
/** The first meaningful line of a unit: not page furniture (a running header, a page number). */
|
|
115
|
+
function firstLine(text, running) {
|
|
116
|
+
const line = text
|
|
117
|
+
.split('\n')
|
|
118
|
+
.map((candidate) => candidate.trim())
|
|
119
|
+
.find((candidate) => /[A-Za-z]{3,}/.test(candidate) &&
|
|
120
|
+
!running.has(candidate.replace(/\d+/g, '#').toLowerCase()));
|
|
121
|
+
return line === undefined ? '' : line.length > 80 ? `${line.slice(0, 77)}…` : line;
|
|
122
|
+
}
|
|
123
|
+
function rangeLabel(units, start, end) {
|
|
124
|
+
const first = units[start];
|
|
125
|
+
const last = units[end];
|
|
126
|
+
if (first?.page !== undefined && last?.page !== undefined) {
|
|
127
|
+
return first.page === last.page ? `Page ${first.page}` : `Pages ${first.page}–${last.page}`;
|
|
128
|
+
}
|
|
129
|
+
return start === end ? `Section ${start + 1}` : `Sections ${start + 1}–${end + 1}`;
|
|
130
|
+
}
|
|
131
|
+
/** Fixed-size groups over `[start, end]`, grouped again while a level is wider than `maxChildren`. */
|
|
132
|
+
function groupNodes(units, start, end, groupSize, maxChildren, levelsLeft, running) {
|
|
133
|
+
let level = [];
|
|
134
|
+
for (let from = start; from <= end; from += groupSize) {
|
|
135
|
+
const to = Math.min(end, from + groupSize - 1);
|
|
136
|
+
const lead = firstLine(units[from]?.text ?? '', running);
|
|
137
|
+
level.push({
|
|
138
|
+
title: `${rangeLabel(units, from, to)}${lead ? `: ${lead}` : ''}`,
|
|
139
|
+
unitStart: from,
|
|
140
|
+
unitEnd: to,
|
|
141
|
+
children: [],
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
let depth = 1;
|
|
145
|
+
while (level.length > maxChildren && depth < levelsLeft) {
|
|
146
|
+
const parents = [];
|
|
147
|
+
const per = Math.max(maxChildren, Math.ceil(level.length / maxChildren));
|
|
148
|
+
for (let index = 0; index < level.length; index += per) {
|
|
149
|
+
const children = level.slice(index, index + per);
|
|
150
|
+
const from = children[0]?.unitStart ?? start;
|
|
151
|
+
const to = children[children.length - 1]?.unitEnd ?? end;
|
|
152
|
+
parents.push({ title: rangeLabel(units, from, to), unitStart: from, unitEnd: to, children });
|
|
153
|
+
}
|
|
154
|
+
level = parents;
|
|
155
|
+
depth += 1;
|
|
156
|
+
}
|
|
157
|
+
return level;
|
|
158
|
+
}
|
|
159
|
+
/** Titles of every descendant, at most 30 — routing information kept when the nodes go. */
|
|
160
|
+
function descendantTitles(nodes, into = []) {
|
|
161
|
+
for (const { node } of walkDraft(nodes)) {
|
|
162
|
+
if (into.length < 30) {
|
|
163
|
+
into.push(node.title);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
return into;
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* Enforce `maxDepth` (deeper titles become `keyItems`), split long leaves into page groups, and
|
|
170
|
+
* collapse what retrieval could not tell apart anyway — retrieval reads whole units, so:
|
|
171
|
+
*
|
|
172
|
+
* - a lone child spanning exactly its parent is folded into it;
|
|
173
|
+
* - a node spanning a single unit keeps its descendants only as `keyItems`;
|
|
174
|
+
* - consecutive leaf siblings spanning the same units merge into the first (the others' titles
|
|
175
|
+
* become its `keyItems`), instead of each getting its own — identical — summary call.
|
|
176
|
+
*/
|
|
177
|
+
function shape(nodes, units, depth, options) {
|
|
178
|
+
const shaped = nodes.map((node) => {
|
|
179
|
+
if (depth >= options.maxDepth ||
|
|
180
|
+
(node.unitStart === node.unitEnd && node.children.length > 0)) {
|
|
181
|
+
const keyItems = descendantTitles(node.children, [...(node.keyItems ?? [])]);
|
|
182
|
+
return {
|
|
183
|
+
...node,
|
|
184
|
+
...(keyItems.length > 0 ? { keyItems } : {}),
|
|
185
|
+
children: [],
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
let children = node.children;
|
|
189
|
+
const only = children[0];
|
|
190
|
+
if (children.length === 1 &&
|
|
191
|
+
only !== undefined &&
|
|
192
|
+
only.unitStart === node.unitStart &&
|
|
193
|
+
only.unitEnd === node.unitEnd) {
|
|
194
|
+
children = only.children;
|
|
195
|
+
}
|
|
196
|
+
if (children.length === 0 && node.unitEnd - node.unitStart + 1 > options.maxLeafUnits) {
|
|
197
|
+
children = groupNodes(units, node.unitStart, node.unitEnd, options.groupSize, options.maxChildren, options.maxDepth - depth, options.running);
|
|
198
|
+
}
|
|
199
|
+
return { ...node, children: shape(children, units, depth + 1, options) };
|
|
200
|
+
});
|
|
201
|
+
const merged = [];
|
|
202
|
+
for (const node of shaped) {
|
|
203
|
+
const previous = merged[merged.length - 1];
|
|
204
|
+
if (previous !== undefined &&
|
|
205
|
+
previous.children.length === 0 &&
|
|
206
|
+
node.children.length === 0 &&
|
|
207
|
+
previous.unitStart === node.unitStart &&
|
|
208
|
+
previous.unitEnd === node.unitEnd) {
|
|
209
|
+
previous.keyItems = [
|
|
210
|
+
...(previous.keyItems ?? []),
|
|
211
|
+
node.title,
|
|
212
|
+
...(node.keyItems ?? []),
|
|
213
|
+
].slice(0, 30);
|
|
214
|
+
continue;
|
|
215
|
+
}
|
|
216
|
+
merged.push(node);
|
|
217
|
+
}
|
|
218
|
+
return merged;
|
|
219
|
+
}
|
|
220
|
+
function* walkDraft(nodes) {
|
|
221
|
+
for (const node of nodes) {
|
|
222
|
+
yield { node };
|
|
223
|
+
yield* walkDraft(node.children);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
/** The text a node spans (children included), joined in reading order. */
|
|
227
|
+
export function spanText(units, start, end) {
|
|
228
|
+
const parts = [];
|
|
229
|
+
for (let index = Math.max(0, start); index <= end && index < units.length; index++) {
|
|
230
|
+
parts.push(units[index]?.text ?? '');
|
|
231
|
+
}
|
|
232
|
+
return parts.join('\n\n');
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* The text of a node before its first child (its own introduction), or its whole span for a leaf —
|
|
236
|
+
* one string per unit, the first starting at the node's heading when it can be found in the unit
|
|
237
|
+
* (a heading in the middle of a page should not be summarized from the top of that page).
|
|
238
|
+
*/
|
|
239
|
+
function ownTexts(units, node) {
|
|
240
|
+
const first = node.children[0];
|
|
241
|
+
const end = first === undefined ? node.unitEnd : Math.max(node.unitStart, first.unitStart - 1);
|
|
242
|
+
const texts = [];
|
|
243
|
+
for (let index = node.unitStart; index <= end && index < units.length; index++) {
|
|
244
|
+
texts.push(units[index]?.text ?? '');
|
|
245
|
+
}
|
|
246
|
+
const head = texts[0];
|
|
247
|
+
if (head !== undefined) {
|
|
248
|
+
const probe = node.title.slice(0, 30);
|
|
249
|
+
const at = probe.length >= 4 ? head.indexOf(probe) : -1;
|
|
250
|
+
if (at > 0) {
|
|
251
|
+
texts[0] = head.slice(at);
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
return texts;
|
|
255
|
+
}
|
|
256
|
+
/**
|
|
257
|
+
* At most `max` characters of `texts`, spread evenly across them — a node spanning ten pages is
|
|
258
|
+
* summarized from all ten, not from the first two.
|
|
259
|
+
*/
|
|
260
|
+
function excerptOf(texts, max) {
|
|
261
|
+
const total = texts.reduce((sum, text) => sum + text.length, 0);
|
|
262
|
+
if (total <= max) {
|
|
263
|
+
return texts.join('\n\n');
|
|
264
|
+
}
|
|
265
|
+
const per = Math.max(400, Math.floor(max / texts.length));
|
|
266
|
+
const parts = [];
|
|
267
|
+
let used = 0;
|
|
268
|
+
for (const text of texts) {
|
|
269
|
+
if (used >= max) {
|
|
270
|
+
break;
|
|
271
|
+
}
|
|
272
|
+
const part = text.length > per ? `${text.slice(0, per)}…` : text;
|
|
273
|
+
parts.push(part);
|
|
274
|
+
used += part.length;
|
|
275
|
+
}
|
|
276
|
+
return parts.join('\n\n');
|
|
277
|
+
}
|
|
278
|
+
function collapse(text, max) {
|
|
279
|
+
const flat = text.replace(/\s+/g, ' ').trim();
|
|
280
|
+
return flat.length > max ? `${flat.slice(0, max - 1)}…` : flat;
|
|
281
|
+
}
|
|
282
|
+
/** Ask the LLM for section starts, window by window. Throws on any failure (caller falls back). */
|
|
283
|
+
async function llmHeadings(llm, units, options) {
|
|
284
|
+
const perUnit = Math.max(500, Math.floor(options.windowChars / 4));
|
|
285
|
+
const windows = [];
|
|
286
|
+
let current = [];
|
|
287
|
+
let size = 0;
|
|
288
|
+
for (const unit of units) {
|
|
289
|
+
const length = Math.min(unit.text.length, perUnit);
|
|
290
|
+
if (current.length > 0 && size + length > options.windowChars) {
|
|
291
|
+
windows.push(current);
|
|
292
|
+
current = [];
|
|
293
|
+
size = 0;
|
|
294
|
+
}
|
|
295
|
+
current.push(unit);
|
|
296
|
+
size += length;
|
|
297
|
+
}
|
|
298
|
+
if (current.length > 0) {
|
|
299
|
+
windows.push(current);
|
|
300
|
+
}
|
|
301
|
+
if (windows.length > options.maxCalls) {
|
|
302
|
+
throw new Error(`document needs ${windows.length} structuring calls, more than maxStructureCalls (${options.maxCalls})`);
|
|
303
|
+
}
|
|
304
|
+
const headings = [];
|
|
305
|
+
for (const window of windows) {
|
|
306
|
+
const shown = window.map((unit) => ({
|
|
307
|
+
unit: unit.index,
|
|
308
|
+
...(unit.page !== undefined ? { page: unit.page } : {}),
|
|
309
|
+
text: unit.text.length > perUnit ? `${unit.text.slice(0, perUnit)}…` : unit.text,
|
|
310
|
+
}));
|
|
311
|
+
const body = shown
|
|
312
|
+
.map((unit) => `<unit ${unit.unit}>\n${unit.text}\n</unit ${unit.unit}>`)
|
|
313
|
+
.join('\n');
|
|
314
|
+
const prompt = `Below is part of a document, split into numbered units (pages or sections).
|
|
315
|
+
List the headings of the sections that START in these units, in reading order, with their nesting level (1 = top-level chapter/part, 2 = its subsections, ...) and the unit number they start in.
|
|
316
|
+
Only list real headings (titles of parts, chapters, sections, clauses, items), not running page headers, table rows or sentences. If no section starts here, return an empty list.
|
|
317
|
+
|
|
318
|
+
${body}
|
|
319
|
+
|
|
320
|
+
Reply with JSON only: {"sections": [{"title": "...", "level": 1, "unit": 0}]}`;
|
|
321
|
+
const reply = parseJsonReply(await llm.call({ kind: 'structure', units: shown }, STRUCTURE_SYSTEM, prompt, 2_000));
|
|
322
|
+
const sections = reply?.sections;
|
|
323
|
+
if (!Array.isArray(sections)) {
|
|
324
|
+
throw new Error('structuring reply had no `sections` list');
|
|
325
|
+
}
|
|
326
|
+
const first = window[0]?.index ?? 0;
|
|
327
|
+
const last = window[window.length - 1]?.index ?? first;
|
|
328
|
+
for (const entry of sections) {
|
|
329
|
+
if (entry === null || typeof entry !== 'object') {
|
|
330
|
+
continue;
|
|
331
|
+
}
|
|
332
|
+
const { title, level, unit } = entry;
|
|
333
|
+
const at = typeof unit === 'number' ? unit : Number(unit);
|
|
334
|
+
if (typeof title !== 'string' ||
|
|
335
|
+
!title.trim() ||
|
|
336
|
+
!Number.isInteger(at) ||
|
|
337
|
+
at < first ||
|
|
338
|
+
at > last) {
|
|
339
|
+
continue;
|
|
340
|
+
}
|
|
341
|
+
const previous = headings[headings.length - 1];
|
|
342
|
+
if (previous !== undefined && at < previous.unit) {
|
|
343
|
+
continue;
|
|
344
|
+
}
|
|
345
|
+
headings.push({
|
|
346
|
+
title: collapse(title, 160),
|
|
347
|
+
level: Math.max(1, Math.min(6, Math.floor(typeof level === 'number' ? level : 1))),
|
|
348
|
+
unit: at,
|
|
349
|
+
atUnitStart: false,
|
|
350
|
+
});
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
return headings;
|
|
354
|
+
}
|
|
355
|
+
/** Run `work` over `items` with at most `limit` in flight. */
|
|
356
|
+
export async function mapLimit(items, limit, work) {
|
|
357
|
+
const results = new Array(items.length);
|
|
358
|
+
let next = 0;
|
|
359
|
+
const workers = Array.from({ length: Math.max(1, Math.min(limit, items.length)) }, async () => {
|
|
360
|
+
while (next < items.length) {
|
|
361
|
+
const index = next++;
|
|
362
|
+
results[index] = await work(items[index], index);
|
|
363
|
+
}
|
|
364
|
+
});
|
|
365
|
+
await Promise.all(workers);
|
|
366
|
+
return results;
|
|
367
|
+
}
|
|
368
|
+
/**
|
|
369
|
+
* Build a document's navigation tree: a hierarchical table of contents whose nodes carry a title,
|
|
370
|
+
* the page (or section) range they span, and a summary.
|
|
371
|
+
*
|
|
372
|
+
* Structure, cheapest first, never looping:
|
|
373
|
+
*
|
|
374
|
+
* 1. the `outline` the input declares (PDF bookmarks);
|
|
375
|
+
* 2. section titles/levels (`sections` input), or headings detected in the text;
|
|
376
|
+
* 3. the LLM, window by window (only when a document has no usable outline, and only if every
|
|
377
|
+
* window fits `maxStructureCalls` and the budget up front);
|
|
378
|
+
* 4. page groups — always works; the fallback for any failure above.
|
|
379
|
+
*
|
|
380
|
+
* Then summaries: top levels first (they matter most for navigation), each node's text sent once,
|
|
381
|
+
* within the budget; what the budget leaves unsummarized gets the start of its own text. With a
|
|
382
|
+
* `previous` tree, an unchanged input is returned as is (zero calls), and nodes whose title and text
|
|
383
|
+
* are unchanged keep their summary.
|
|
384
|
+
*
|
|
385
|
+
* The result is deterministic for a deterministic model: node ids are depth-first positions, and
|
|
386
|
+
* nothing depends on timing except which summaries a timed-out build got to.
|
|
387
|
+
*/
|
|
388
|
+
export async function buildDocumentTree(input, options) {
|
|
389
|
+
const started = Date.now();
|
|
390
|
+
const { units, declared, kind } = toUnits(input);
|
|
391
|
+
const maxDepth = Math.max(1, options.maxDepth ?? 4);
|
|
392
|
+
const groupSize = Math.max(1, options.groupSize ?? 5);
|
|
393
|
+
const maxLeafUnits = Math.max(groupSize, options.maxLeafUnits ?? 10);
|
|
394
|
+
const maxChildren = Math.max(2, options.maxChildren ?? 12);
|
|
395
|
+
const minHeadings = Math.max(1, options.minHeadings ?? 3);
|
|
396
|
+
const llmEnabled = options.llm !== undefined;
|
|
397
|
+
const summaries = options.summaries ?? llmEnabled;
|
|
398
|
+
const describe = options.describe ?? llmEnabled;
|
|
399
|
+
const llmStructure = options.llmStructure ?? llmEnabled;
|
|
400
|
+
const detect = options.detectHeadings ?? true;
|
|
401
|
+
const summaryMinChars = options.summaryMinChars ?? 600;
|
|
402
|
+
const summaryInputChars = options.summaryInputChars ?? 8_000;
|
|
403
|
+
const unitHashes = units.map((unit) => stableHash(unit.text));
|
|
404
|
+
const fingerprint = stableHash(JSON.stringify({
|
|
405
|
+
v: 1,
|
|
406
|
+
kind,
|
|
407
|
+
title: input.title ?? null,
|
|
408
|
+
outline: input.outline ?? null,
|
|
409
|
+
sections: input.sections?.map((section) => [
|
|
410
|
+
section.title ?? null,
|
|
411
|
+
section.level ?? null,
|
|
412
|
+
section.page ?? null,
|
|
413
|
+
]) ?? null,
|
|
414
|
+
units: unitHashes,
|
|
415
|
+
maxDepth,
|
|
416
|
+
groupSize,
|
|
417
|
+
maxLeafUnits,
|
|
418
|
+
maxChildren,
|
|
419
|
+
minHeadings,
|
|
420
|
+
detect,
|
|
421
|
+
summaries,
|
|
422
|
+
describe,
|
|
423
|
+
llmStructure: llmStructure && llmEnabled,
|
|
424
|
+
summaryMinChars,
|
|
425
|
+
summaryInputChars,
|
|
426
|
+
}));
|
|
427
|
+
if (options.previous?.fingerprint === fingerprint) {
|
|
428
|
+
// Same text, same shape: only the access metadata / citation may have moved.
|
|
429
|
+
const { metadata: _metadata, source: _source, ...previous } = options.previous;
|
|
430
|
+
const metadata = options.metadata ?? options.previous.metadata;
|
|
431
|
+
const source = options.source ?? options.previous.source;
|
|
432
|
+
return {
|
|
433
|
+
tree: {
|
|
434
|
+
...previous,
|
|
435
|
+
...(metadata !== undefined ? { metadata } : {}),
|
|
436
|
+
...(source !== undefined ? { source } : {}),
|
|
437
|
+
},
|
|
438
|
+
units,
|
|
439
|
+
unchanged: true,
|
|
440
|
+
};
|
|
441
|
+
}
|
|
442
|
+
const budget = new BudgetedLlm(options.llm ?? (() => Promise.reject(new Error('no llm'))), { ...DEFAULT_BUDGET, ...options.budget }, options.signal);
|
|
443
|
+
let fallbackReason;
|
|
444
|
+
// 1–4: structure.
|
|
445
|
+
let structure = 'flat';
|
|
446
|
+
let headings = [];
|
|
447
|
+
const running = runningLines(units.map((unit) => unit.text));
|
|
448
|
+
const outline = outlineHeadings(input, units);
|
|
449
|
+
if (outline.length >= Math.min(minHeadings, 2)) {
|
|
450
|
+
structure = 'outline';
|
|
451
|
+
headings = outline;
|
|
452
|
+
}
|
|
453
|
+
else if (declared !== undefined && declared.length >= 1) {
|
|
454
|
+
structure = 'headings';
|
|
455
|
+
headings = declared;
|
|
456
|
+
}
|
|
457
|
+
else if (detect !== false && units.length > 0) {
|
|
458
|
+
const found = detectHeadings(units.map((unit) => unit.text), detect === 'markdown');
|
|
459
|
+
if (found.length >= minHeadings) {
|
|
460
|
+
structure = 'headings';
|
|
461
|
+
headings = found;
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
if (structure === 'flat' && llmStructure && llmEnabled && units.length > 1) {
|
|
465
|
+
try {
|
|
466
|
+
const found = await llmHeadings(budget, units, {
|
|
467
|
+
windowChars: options.structureWindowChars ?? 24_000,
|
|
468
|
+
maxCalls: Math.max(1, options.maxStructureCalls ?? 12),
|
|
469
|
+
});
|
|
470
|
+
if (found.length >= Math.min(minHeadings, 2)) {
|
|
471
|
+
structure = 'llm';
|
|
472
|
+
headings = found;
|
|
473
|
+
}
|
|
474
|
+
else {
|
|
475
|
+
fallbackReason = 'LLM found no usable structure';
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
catch (error) {
|
|
479
|
+
fallbackReason =
|
|
480
|
+
error instanceof TreeBudgetExceededError
|
|
481
|
+
? `LLM structuring stopped: ${error.message}`
|
|
482
|
+
: `LLM structuring failed: ${error.message}`;
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
const draft = structure === 'flat'
|
|
486
|
+
? units.length === 0
|
|
487
|
+
? []
|
|
488
|
+
: groupNodes(units, 0, units.length - 1, groupSize, maxChildren, maxDepth, running)
|
|
489
|
+
: nestHeadings(headings, units.length);
|
|
490
|
+
const shaped = shape(draft, units, 1, {
|
|
491
|
+
maxDepth,
|
|
492
|
+
maxLeafUnits,
|
|
493
|
+
groupSize,
|
|
494
|
+
maxChildren,
|
|
495
|
+
running,
|
|
496
|
+
});
|
|
497
|
+
// Ids, pages and content hashes.
|
|
498
|
+
let counter = 0;
|
|
499
|
+
const finalize = (nodes) => nodes.map((node) => {
|
|
500
|
+
counter += 1;
|
|
501
|
+
const id = String(counter).padStart(4, '0');
|
|
502
|
+
const pageStart = units[node.unitStart]?.page;
|
|
503
|
+
const pageEnd = units[node.unitEnd]?.page;
|
|
504
|
+
return {
|
|
505
|
+
id,
|
|
506
|
+
title: node.title,
|
|
507
|
+
unitStart: node.unitStart,
|
|
508
|
+
unitEnd: node.unitEnd,
|
|
509
|
+
...(pageStart !== undefined ? { pageStart } : {}),
|
|
510
|
+
...(pageEnd !== undefined ? { pageEnd } : {}),
|
|
511
|
+
...(node.keyItems !== undefined ? { keyItems: node.keyItems } : {}),
|
|
512
|
+
contentHash: stableHash(`${node.title}\u0000${unitHashes.slice(node.unitStart, node.unitEnd + 1).join(',')}`),
|
|
513
|
+
children: finalize(node.children),
|
|
514
|
+
};
|
|
515
|
+
});
|
|
516
|
+
const nodes = finalize(shaped);
|
|
517
|
+
// Summaries: reuse, then top-down (breadth-first) within the budget, then the text itself.
|
|
518
|
+
const previousSummaries = new Map();
|
|
519
|
+
if (options.previous !== undefined) {
|
|
520
|
+
for (const { node } of walkTree(options.previous.nodes)) {
|
|
521
|
+
if (node.summary !== undefined &&
|
|
522
|
+
(node.summarySource === 'llm' || node.summarySource === 'reused')) {
|
|
523
|
+
previousSummaries.set(node.contentHash, node.summary);
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
const breadthFirst = [];
|
|
528
|
+
for (let queue = [...nodes]; queue.length > 0;) {
|
|
529
|
+
const node = queue.shift();
|
|
530
|
+
breadthFirst.push(node);
|
|
531
|
+
queue = queue.concat(node.children);
|
|
532
|
+
}
|
|
533
|
+
let reusedSummaries = 0;
|
|
534
|
+
const pending = [];
|
|
535
|
+
for (const node of breadthFirst) {
|
|
536
|
+
const reused = previousSummaries.get(node.contentHash);
|
|
537
|
+
const own = ownTexts(units, node).join('\n\n');
|
|
538
|
+
if (reused !== undefined) {
|
|
539
|
+
node.summary = reused;
|
|
540
|
+
node.summarySource = 'reused';
|
|
541
|
+
reusedSummaries += 1;
|
|
542
|
+
}
|
|
543
|
+
else if (own.trim().length < summaryMinChars && node.children.length === 0) {
|
|
544
|
+
const text = collapse(own, summaryMinChars);
|
|
545
|
+
if (text) {
|
|
546
|
+
node.summary = text;
|
|
547
|
+
node.summarySource = 'text';
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
else if (summaries && llmEnabled) {
|
|
551
|
+
pending.push(node);
|
|
552
|
+
}
|
|
553
|
+
else {
|
|
554
|
+
const text = collapse(own, 300);
|
|
555
|
+
if (text) {
|
|
556
|
+
node.summary = text;
|
|
557
|
+
node.summarySource = 'text';
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
let summariesCut = false;
|
|
562
|
+
await mapLimit(pending, Math.max(1, options.concurrency ?? 4), async (node) => {
|
|
563
|
+
const own = ownTexts(units, node);
|
|
564
|
+
const childTitles = [...node.children.map((child) => child.title), ...(node.keyItems ?? [])]
|
|
565
|
+
.map((title) => `- ${title}`)
|
|
566
|
+
.join('\n');
|
|
567
|
+
const excerpt = excerptOf(own, summaryInputChars);
|
|
568
|
+
const prompt = `Summarize this section of ${input.title ? `"${input.title}"` : 'a document'} in 2-4 sentences (at most 80 words). Name the specific topics, requirements, clauses, figures and defined terms it covers, so a reader can tell whether it answers their question.
|
|
569
|
+
|
|
570
|
+
Section title: ${node.title}
|
|
571
|
+
${childTitles ? `Subsections:\n${childTitles}\n` : ''}
|
|
572
|
+
Section text:
|
|
573
|
+
${excerpt}
|
|
574
|
+
|
|
575
|
+
Reply with the summary only.`;
|
|
576
|
+
if (!summariesCut && budget.fits(SUMMARY_SYSTEM + prompt, 300)) {
|
|
577
|
+
try {
|
|
578
|
+
const reply = await budget.call({ kind: 'summarize', title: node.title, text: excerpt }, SUMMARY_SYSTEM, prompt, 300);
|
|
579
|
+
const json = parseJsonReply(reply);
|
|
580
|
+
const summary = typeof json?.summary === 'string' ? json.summary : reply;
|
|
581
|
+
if (summary.trim()) {
|
|
582
|
+
node.summary = collapse(summary, 800);
|
|
583
|
+
node.summarySource = 'llm';
|
|
584
|
+
return;
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
catch (error) {
|
|
588
|
+
if (error instanceof TreeBudgetExceededError) {
|
|
589
|
+
summariesCut = true;
|
|
590
|
+
}
|
|
591
|
+
fallbackReason ??= `summary failed: ${error.message}`;
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
else {
|
|
595
|
+
summariesCut = true;
|
|
596
|
+
}
|
|
597
|
+
const text = collapse(own.join('\n\n'), 300);
|
|
598
|
+
if (text) {
|
|
599
|
+
node.summary = text;
|
|
600
|
+
node.summarySource = 'text';
|
|
601
|
+
}
|
|
602
|
+
});
|
|
603
|
+
if (summariesCut) {
|
|
604
|
+
fallbackReason ??= 'LLM budget exhausted before every node was summarized';
|
|
605
|
+
}
|
|
606
|
+
let description;
|
|
607
|
+
const outlineForDescription = nodes.slice(0, 40).map((node) => ({
|
|
608
|
+
title: node.title,
|
|
609
|
+
...(node.summary !== undefined ? { summary: node.summary } : {}),
|
|
610
|
+
}));
|
|
611
|
+
if (describe && llmEnabled && nodes.length > 0) {
|
|
612
|
+
const prompt = `Write a one-paragraph description (at most 80 words) of the document ${input.title ? `"${input.title}"` : ''} from its table of contents below, so a reader can tell which questions it can answer.
|
|
613
|
+
|
|
614
|
+
${outlineForDescription.map((entry) => `- ${entry.title}${entry.summary ? `: ${entry.summary}` : ''}`).join('\n')}
|
|
615
|
+
|
|
616
|
+
Reply with the description only.`;
|
|
617
|
+
if (budget.fits(SUMMARY_SYSTEM + prompt, 300)) {
|
|
618
|
+
try {
|
|
619
|
+
description = collapse(await budget.call({
|
|
620
|
+
kind: 'describe',
|
|
621
|
+
...(input.title !== undefined ? { title: input.title } : {}),
|
|
622
|
+
outline: outlineForDescription,
|
|
623
|
+
}, SUMMARY_SYSTEM, prompt, 300), 1_000);
|
|
624
|
+
}
|
|
625
|
+
catch (error) {
|
|
626
|
+
fallbackReason ??= `description failed: ${error.message}`;
|
|
627
|
+
}
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
if (description === undefined &&
|
|
631
|
+
options.previous?.description !== undefined &&
|
|
632
|
+
reusedSummaries > 0) {
|
|
633
|
+
description = options.previous.description;
|
|
634
|
+
}
|
|
635
|
+
const tree = {
|
|
636
|
+
documentId: options.documentId,
|
|
637
|
+
version: 1,
|
|
638
|
+
...(input.title !== undefined ? { title: input.title } : {}),
|
|
639
|
+
...(description ? { description } : {}),
|
|
640
|
+
structure,
|
|
641
|
+
fingerprint,
|
|
642
|
+
unitCount: units.length,
|
|
643
|
+
nodes,
|
|
644
|
+
stats: {
|
|
645
|
+
llmCalls: budget.calls,
|
|
646
|
+
inputTokens: budget.inputTokens,
|
|
647
|
+
outputTokens: budget.outputTokens,
|
|
648
|
+
durationMs: Date.now() - started,
|
|
649
|
+
...(fallbackReason !== undefined ? { fallbackReason } : {}),
|
|
650
|
+
budgetExhausted: budget.exhausted,
|
|
651
|
+
reusedSummaries,
|
|
652
|
+
},
|
|
653
|
+
...(options.metadata !== undefined ? { metadata: options.metadata } : {}),
|
|
654
|
+
...(options.source !== undefined ? { source: options.source } : {}),
|
|
655
|
+
builtAt: new Date().toISOString(),
|
|
656
|
+
};
|
|
657
|
+
return { tree, units, unchanged: false };
|
|
658
|
+
}
|
|
659
|
+
//# sourceMappingURL=build.js.map
|