nexorum 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +9 -0
- package/README.md +144 -2
- package/bin/nexorum.js +12 -0
- package/package.json +46 -4
- package/src/agent-dream.js +277 -0
- package/src/agents-md.js +80 -0
- package/src/args.js +71 -0
- package/src/cli.js +278 -0
- package/src/core/basics.js +88 -0
- package/src/core/diff.js +63 -0
- package/src/core/dream/agent.js +109 -0
- package/src/core/dream/engine.js +223 -0
- package/src/core/dream/prompts.js +112 -0
- package/src/core/dream/tools.js +348 -0
- package/src/core/estimate.js +57 -0
- package/src/core/index.js +13 -0
- package/src/core/providers/http.js +143 -0
- package/src/core/providers/index.js +179 -0
- package/src/core/scrub.js +82 -0
- package/src/core/search.js +107 -0
- package/src/core/sessions.js +171 -0
- package/src/core/text.js +72 -0
- package/src/core/wiki.js +102 -0
- package/src/doctor.js +177 -0
- package/src/dream.js +309 -0
- package/src/hook.js +101 -0
- package/src/import.js +137 -0
- package/src/init.js +266 -0
- package/src/mcp.js +376 -0
- package/src/memory.js +188 -0
- package/src/proposals.js +204 -0
- package/src/store.js +445 -0
- package/src/transcript.js +40 -0
- package/src/version.js +4 -0
|
@@ -0,0 +1,348 @@
|
|
|
1
|
+
// Document tools for the dream agents. One toolbox per agent conversation. It remembers the hash of every document
|
|
2
|
+
// the agent has read completely: replacing an existing document with write_doc requires such a read, and that hash
|
|
3
|
+
// is the write's precondition, so an agent never overwrites a parallel agent's change unseen.
|
|
4
|
+
// All handlers are synchronous, so a check and the write after it cannot interleave with another agent.
|
|
5
|
+
import { INDEX_PATH } from "../text.js";
|
|
6
|
+
import { MAX_DOC_BYTES } from "../wiki.js";
|
|
7
|
+
|
|
8
|
+
export const MAX_TOOL_OUTPUT_CHARS = 30_000;
|
|
9
|
+
export const MAX_READ_CHARS = 28_000;
|
|
10
|
+
export const MAX_GREP_RESULTS = 200;
|
|
11
|
+
export const DEFAULT_GREP_RESULTS = 40;
|
|
12
|
+
export const INDEX_MAX_CHARS = 4_096;
|
|
13
|
+
const GREP_SCAN_LIMIT = 5_000_000;
|
|
14
|
+
const GREP_LINE_LIMIT = 2_000;
|
|
15
|
+
const NESTED_QUANTIFIER = /\([^()]*[+*][^()]*\)\s*[+*{]/;
|
|
16
|
+
const MAX_INDEX_REJECTIONS = 2;
|
|
17
|
+
|
|
18
|
+
const pathProp = { type: "string", description: 'Document path such as "/deploys.md". Must start with "/".' };
|
|
19
|
+
const schema = (properties, required) => ({ type: "object", properties, ...(required ? { required } : {}) });
|
|
20
|
+
|
|
21
|
+
const DEFS = {
|
|
22
|
+
list_docs: {
|
|
23
|
+
name: "list_docs",
|
|
24
|
+
description: "List documents: path, size, short sha and the frontmatter name and description. Optional path prefix (a prefix ending in / selects a folder).",
|
|
25
|
+
input_schema: schema({ prefix: { type: "string", description: 'Optional path prefix starting with "/".' } }),
|
|
26
|
+
},
|
|
27
|
+
read_doc: {
|
|
28
|
+
name: "read_doc",
|
|
29
|
+
description: `Read one document, up to ${MAX_READ_CHARS} characters per call; page through longer ones with offset. Only a complete read in this conversation allows write_doc to replace an existing document.`,
|
|
30
|
+
input_schema: schema({ path: pathProp, offset: { type: "integer", minimum: 0, description: "Character offset (default 0)." } }, ["path"]),
|
|
31
|
+
},
|
|
32
|
+
grep_docs: {
|
|
33
|
+
name: "grep_docs",
|
|
34
|
+
description: `Search every document line by line with a JavaScript regular expression (case-insensitive unless ignore_case is false). Returns "path:line: text", at most ${MAX_GREP_RESULTS} matches.`,
|
|
35
|
+
input_schema: schema(
|
|
36
|
+
{
|
|
37
|
+
pattern: { type: "string", description: "Regular expression source, without slashes or flags." },
|
|
38
|
+
prefix: { type: "string", description: 'Only paths with this prefix (starting with "/").' },
|
|
39
|
+
ignore_case: { type: "boolean", description: "Default true." },
|
|
40
|
+
max_results: { type: "integer", minimum: 1, maximum: MAX_GREP_RESULTS, description: `Default ${DEFAULT_GREP_RESULTS}.` },
|
|
41
|
+
},
|
|
42
|
+
["pattern"],
|
|
43
|
+
),
|
|
44
|
+
},
|
|
45
|
+
write_doc: {
|
|
46
|
+
name: "write_doc",
|
|
47
|
+
description:
|
|
48
|
+
"Create a document, or replace the whole content of one. Replacing needs a complete read_doc of it earlier in this conversation; if it changed since, the write fails with a conflict: re-read, merge, write again. Prefer edit_doc for small changes.",
|
|
49
|
+
input_schema: schema({ path: pathProp, content: { type: "string", description: "The complete new content." } }, ["path", "content"]),
|
|
50
|
+
},
|
|
51
|
+
edit_doc: {
|
|
52
|
+
name: "edit_doc",
|
|
53
|
+
description:
|
|
54
|
+
"Replace exactly one occurrence of old_string with new_string in the document's CURRENT content. Fails when old_string is missing or occurs more than once (add context to make it unique). Best for shared files such as /_index.md.",
|
|
55
|
+
input_schema: schema(
|
|
56
|
+
{
|
|
57
|
+
path: pathProp,
|
|
58
|
+
old_string: { type: "string", description: "The exact text to replace, whitespace included." },
|
|
59
|
+
new_string: { type: "string", description: "The replacement (empty deletes)." },
|
|
60
|
+
},
|
|
61
|
+
["path", "old_string", "new_string"],
|
|
62
|
+
),
|
|
63
|
+
},
|
|
64
|
+
delete_doc: {
|
|
65
|
+
name: "delete_doc",
|
|
66
|
+
description: "Delete a document, only once its content lives elsewhere or it is an exact duplicate. Every change is versioned and reviewed.",
|
|
67
|
+
input_schema: schema({ path: pathProp }, ["path"]),
|
|
68
|
+
},
|
|
69
|
+
rename_doc: {
|
|
70
|
+
name: "rename_doc",
|
|
71
|
+
description: "Rename or move a document. Fails when the destination exists. Fix [[links]] and /_index.md yourself afterwards.",
|
|
72
|
+
input_schema: schema({ from: pathProp, to: pathProp }, ["from", "to"]),
|
|
73
|
+
},
|
|
74
|
+
};
|
|
75
|
+
|
|
76
|
+
const SUBMIT = {
|
|
77
|
+
analyzer: {
|
|
78
|
+
name: "submit",
|
|
79
|
+
description: "Finish. Call it exactly once, after your last write has been confirmed; never in the same turn as writes whose results you have not seen.",
|
|
80
|
+
input_schema: schema(
|
|
81
|
+
{
|
|
82
|
+
summary: { type: "string", description: "What you verified, corrected and added, and what is contradictory or uncertain for the organizer." },
|
|
83
|
+
touched: { type: "array", items: { type: "string" }, description: "Paths you created or changed." },
|
|
84
|
+
},
|
|
85
|
+
["summary"],
|
|
86
|
+
),
|
|
87
|
+
},
|
|
88
|
+
organizer: {
|
|
89
|
+
name: "submit",
|
|
90
|
+
description: `Finish. Call it exactly once, when the wiki is consistent. ${INDEX_PATH} must exist.`,
|
|
91
|
+
input_schema: schema(
|
|
92
|
+
{
|
|
93
|
+
summary: { type: "string", description: "What you merged, fixed and found, in a few sentences." },
|
|
94
|
+
changes: { type: "array", items: { type: "string" }, description: "One short line per notable change (merged, split, deleted, conflict marked, link fixed ...)." },
|
|
95
|
+
},
|
|
96
|
+
["summary", "changes"],
|
|
97
|
+
),
|
|
98
|
+
},
|
|
99
|
+
};
|
|
100
|
+
|
|
101
|
+
const ANALYZER_TOOLS = ["list_docs", "read_doc", "grep_docs", "write_doc", "edit_doc"];
|
|
102
|
+
const ORGANIZER_TOOLS = [...ANALYZER_TOOLS, "delete_doc", "rename_doc"];
|
|
103
|
+
|
|
104
|
+
export function toolDefinitions(role) {
|
|
105
|
+
if (role === "analyzer") return [...ANALYZER_TOOLS.map((n) => DEFS[n]), SUBMIT.analyzer];
|
|
106
|
+
if (role === "organizer") return [...ORGANIZER_TOOLS.map((n) => DEFS[n]), SUBMIT.organizer];
|
|
107
|
+
throw new Error(`unknown role ${role}`);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
export function clip(text, max = MAX_TOOL_OUTPUT_CHARS) {
|
|
111
|
+
const s = String(text);
|
|
112
|
+
return s.length <= max ? s : `${s.slice(0, max)}\n[output truncated: ${s.length - max} more characters]`;
|
|
113
|
+
}
|
|
114
|
+
const short = (s, n = 80) => clip(String(s), n).replace(/\n\[output truncated.*$/s, "...");
|
|
115
|
+
const fail = (text) => ({ text: clip(text), isError: true });
|
|
116
|
+
const done = (text) => ({ text: clip(text), isError: false });
|
|
117
|
+
const bytesOf = (s) => Buffer.byteLength(s, "utf8");
|
|
118
|
+
|
|
119
|
+
function explain(e, p) {
|
|
120
|
+
switch (e?.code) {
|
|
121
|
+
case "conflict":
|
|
122
|
+
return `Conflict: ${p} changed since you last read it (another agent updated it, or it already exists or was moved). Re-read it with read_doc, merge your change into the current content and write again, or use edit_doc.`;
|
|
123
|
+
case "not_found":
|
|
124
|
+
return `Not found: ${e.message}`;
|
|
125
|
+
case "invalid_path":
|
|
126
|
+
return e.message;
|
|
127
|
+
case "too_large":
|
|
128
|
+
return `Too large: ${e.message} Split the content into smaller linked files (the hard limit is ${MAX_DOC_BYTES} bytes; aim far lower).`;
|
|
129
|
+
case "quota":
|
|
130
|
+
return `Document quota reached: ${e.message} Merge or delete documents before creating more.`;
|
|
131
|
+
default:
|
|
132
|
+
return `Document error: ${e?.message ?? e}`;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* @param {{docs:object, role:"analyzer"|"organizer", scrub:(s:string)=>{text:string,count:number,kinds:string[]}, stats:{mutations:number, redactions:number}}} o
|
|
138
|
+
*/
|
|
139
|
+
export function createToolbox({ docs, role, scrub, stats }) {
|
|
140
|
+
const seen = new Map(); // path -> sha256 of the content this agent has read completely
|
|
141
|
+
let indexRejections = 0;
|
|
142
|
+
const definitions = toolDefinitions(role);
|
|
143
|
+
const allowed = new Set(definitions.map((d) => d.name));
|
|
144
|
+
|
|
145
|
+
const clean = (content) => {
|
|
146
|
+
const r = scrub(content);
|
|
147
|
+
stats.redactions += r.count;
|
|
148
|
+
const note = r.count
|
|
149
|
+
? `\nNote: ${r.count} secret-looking value(s) (${r.kinds.join(", ")}) were replaced with [REDACTED:...]. Never write secrets; record only the credential's name and where it is kept.`
|
|
150
|
+
: "";
|
|
151
|
+
return { text: r.text, note };
|
|
152
|
+
};
|
|
153
|
+
const lookup = (p) => {
|
|
154
|
+
try {
|
|
155
|
+
return { doc: docs.get(p) };
|
|
156
|
+
} catch (e) {
|
|
157
|
+
return { error: fail(explain(e, p)) };
|
|
158
|
+
}
|
|
159
|
+
};
|
|
160
|
+
|
|
161
|
+
const handlers = {
|
|
162
|
+
list_docs({ prefix }) {
|
|
163
|
+
if (prefix !== undefined && typeof prefix !== "string") return fail("prefix must be a string.");
|
|
164
|
+
if (prefix && !prefix.startsWith("/")) return fail('prefix must start with "/".');
|
|
165
|
+
const items = docs.list(prefix ? { prefix } : {});
|
|
166
|
+
if (!items.length) return done(prefix ? `No documents under ${prefix}.` : "No documents yet.");
|
|
167
|
+
const lines = items.map((d) => {
|
|
168
|
+
const label = [d.name, d.description].filter(Boolean).join(": ");
|
|
169
|
+
return `- ${d.path} (${d.bytes} bytes, sha ${d.sha256.slice(0, 8)})${label ? ` ${short(label, 160)}` : ""}`;
|
|
170
|
+
});
|
|
171
|
+
return done(`${items.length} document(s):\n${lines.join("\n")}`);
|
|
172
|
+
},
|
|
173
|
+
|
|
174
|
+
read_doc({ path: p, offset = 0 }) {
|
|
175
|
+
if (typeof p !== "string") return fail("path must be a string.");
|
|
176
|
+
if (!Number.isInteger(offset) || offset < 0) return fail("offset must be a non-negative integer.");
|
|
177
|
+
const { doc, error } = lookup(p);
|
|
178
|
+
if (error) return error;
|
|
179
|
+
if (!doc) return fail(`Not found: ${p} does not exist.`);
|
|
180
|
+
const total = doc.content.length;
|
|
181
|
+
if (total === 0) {
|
|
182
|
+
seen.set(p, doc.sha256);
|
|
183
|
+
return done("[document is empty]");
|
|
184
|
+
}
|
|
185
|
+
if (offset >= total) return fail(`offset ${offset} is past the end of the document (${total} characters).`);
|
|
186
|
+
const end = Math.min(total, offset + MAX_READ_CHARS);
|
|
187
|
+
const part = doc.content.slice(offset, end);
|
|
188
|
+
if (offset === 0 && end === total) {
|
|
189
|
+
seen.set(p, doc.sha256);
|
|
190
|
+
return done(part);
|
|
191
|
+
}
|
|
192
|
+
const next = end < total ? ` Call read_doc with offset=${end} for the next part.` : "";
|
|
193
|
+
return done(`${part}\n[partial read: characters ${offset}-${end} of ${total}.${next} A partial read does not allow write_doc to overwrite this document; use edit_doc for targeted changes.]`);
|
|
194
|
+
},
|
|
195
|
+
|
|
196
|
+
grep_docs({ pattern, prefix, ignore_case, max_results }) {
|
|
197
|
+
if (typeof pattern !== "string" || pattern === "") return fail("pattern must be a non-empty string.");
|
|
198
|
+
if (pattern.length > 300) return fail("pattern is too long (max 300 characters).");
|
|
199
|
+
if (NESTED_QUANTIFIER.test(pattern)) return fail("pattern rejected: nested quantifiers like (a+)+ can run catastrophically slow. Simplify it.");
|
|
200
|
+
if (prefix !== undefined && (typeof prefix !== "string" || !prefix.startsWith("/"))) return fail('prefix must be a string starting with "/".');
|
|
201
|
+
let max = max_results === undefined ? DEFAULT_GREP_RESULTS : max_results;
|
|
202
|
+
if (!Number.isInteger(max) || max < 1) return fail("max_results must be a positive integer.");
|
|
203
|
+
max = Math.min(max, MAX_GREP_RESULTS);
|
|
204
|
+
let re;
|
|
205
|
+
try {
|
|
206
|
+
re = new RegExp(pattern, ignore_case === false ? "" : "i");
|
|
207
|
+
} catch (e) {
|
|
208
|
+
return fail(`Invalid regular expression: ${e.message}`);
|
|
209
|
+
}
|
|
210
|
+
const hits = [];
|
|
211
|
+
let scanned = 0;
|
|
212
|
+
let stop = null;
|
|
213
|
+
scan: for (const entry of docs.list(prefix ? { prefix } : {})) {
|
|
214
|
+
const doc = docs.get(entry.path);
|
|
215
|
+
if (!doc) continue;
|
|
216
|
+
scanned += doc.content.length;
|
|
217
|
+
if (scanned > GREP_SCAN_LIMIT) {
|
|
218
|
+
stop = "scan limit reached; narrow the search with prefix";
|
|
219
|
+
break;
|
|
220
|
+
}
|
|
221
|
+
const lines = doc.content.split("\n");
|
|
222
|
+
for (let i = 0; i < lines.length; i++) {
|
|
223
|
+
if (!re.test(lines[i].slice(0, GREP_LINE_LIMIT))) continue;
|
|
224
|
+
if (hits.length >= max) {
|
|
225
|
+
stop = `more matches exist; showing the first ${max}`;
|
|
226
|
+
break scan;
|
|
227
|
+
}
|
|
228
|
+
hits.push(`${doc.path}:${i + 1}: ${lines[i].slice(0, 300)}`);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
if (!hits.length) return done(stop ? `No matches (${stop}).` : "No matches.");
|
|
232
|
+
return done(`${hits.join("\n")}${stop ? `\n[${stop}]` : ""}`);
|
|
233
|
+
},
|
|
234
|
+
|
|
235
|
+
write_doc({ path: p, content }) {
|
|
236
|
+
if (typeof p !== "string") return fail("path must be a string.");
|
|
237
|
+
if (typeof content !== "string") return fail("content must be a string.");
|
|
238
|
+
if (content.trim() === "") return fail("content is empty. Write the document's content, or use delete_doc (organizer only) to remove a document.");
|
|
239
|
+
const { doc: existing, error } = lookup(p);
|
|
240
|
+
if (error) return error;
|
|
241
|
+
const { text, note } = clean(content);
|
|
242
|
+
try {
|
|
243
|
+
if (existing) {
|
|
244
|
+
const known = seen.get(p);
|
|
245
|
+
if (!known) {
|
|
246
|
+
return fail(`${p} already exists and you have not read it completely in this session. Read it with read_doc first, then write the merged result (or use edit_doc for a targeted change).`);
|
|
247
|
+
}
|
|
248
|
+
if (existing.content === text) return done(`No change: ${p} already has exactly this content.`);
|
|
249
|
+
seen.set(p, docs.write(p, text, { expectSha256: known }).sha256);
|
|
250
|
+
stats.mutations++;
|
|
251
|
+
return done(`Updated ${p} (${bytesOf(text)} bytes).${note}`);
|
|
252
|
+
}
|
|
253
|
+
seen.set(p, docs.write(p, text).sha256);
|
|
254
|
+
stats.mutations++;
|
|
255
|
+
return done(`Created ${p} (${bytesOf(text)} bytes).${note}`);
|
|
256
|
+
} catch (e) {
|
|
257
|
+
return fail(explain(e, p));
|
|
258
|
+
}
|
|
259
|
+
},
|
|
260
|
+
|
|
261
|
+
edit_doc(input) {
|
|
262
|
+
for (const k of ["path", "old_string", "new_string"]) if (typeof input[k] !== "string") return fail(`${k} must be a string.`);
|
|
263
|
+
const { path: p, old_string: from, new_string: to } = input;
|
|
264
|
+
if (from === "") return fail("old_string must not be empty.");
|
|
265
|
+
const { doc, error } = lookup(p);
|
|
266
|
+
if (error) return error;
|
|
267
|
+
if (!doc) return fail(`Not found: ${p} does not exist. Create it with write_doc.`);
|
|
268
|
+
const at = doc.content.indexOf(from);
|
|
269
|
+
if (at === -1) return fail(`old_string was not found in ${p}. It must match the current content exactly, including whitespace and newlines. Re-read the document with read_doc and try again.`);
|
|
270
|
+
let count = 1;
|
|
271
|
+
for (let i = doc.content.indexOf(from, at + 1); i !== -1; i = doc.content.indexOf(from, i + 1)) count++;
|
|
272
|
+
if (count > 1) return fail(`old_string matches ${count} places in ${p}. Include more surrounding text so that it matches exactly one place.`);
|
|
273
|
+
const { text, note } = clean(doc.content.slice(0, at) + to + doc.content.slice(at + from.length));
|
|
274
|
+
if (text === doc.content) return done(`No change: ${p} is already as requested.`);
|
|
275
|
+
if (text.trim() === "") return fail("The edit would leave the document empty. Use delete_doc (organizer only) or keep some content.");
|
|
276
|
+
try {
|
|
277
|
+
const res = docs.write(p, text, { expectSha256: doc.sha256 });
|
|
278
|
+
// The agent has seen the whole new content only if its earlier complete read matched what it edited.
|
|
279
|
+
if (seen.get(p) === doc.sha256) seen.set(p, res.sha256);
|
|
280
|
+
else seen.delete(p);
|
|
281
|
+
stats.mutations++;
|
|
282
|
+
return done(`Edited ${p} (${bytesOf(text)} bytes).${note}`);
|
|
283
|
+
} catch (e) {
|
|
284
|
+
return fail(explain(e, p));
|
|
285
|
+
}
|
|
286
|
+
},
|
|
287
|
+
|
|
288
|
+
delete_doc({ path: p }) {
|
|
289
|
+
if (typeof p !== "string") return fail("path must be a string.");
|
|
290
|
+
try {
|
|
291
|
+
docs.delete(p);
|
|
292
|
+
} catch (e) {
|
|
293
|
+
return fail(explain(e, p));
|
|
294
|
+
}
|
|
295
|
+
seen.delete(p);
|
|
296
|
+
stats.mutations++;
|
|
297
|
+
return done(`Deleted ${p}.`);
|
|
298
|
+
},
|
|
299
|
+
|
|
300
|
+
rename_doc({ from, to }) {
|
|
301
|
+
if (typeof from !== "string" || typeof to !== "string") return fail("from and to must be strings.");
|
|
302
|
+
try {
|
|
303
|
+
docs.rename(from, to);
|
|
304
|
+
} catch (e) {
|
|
305
|
+
return fail(explain(e, e?.code === "conflict" ? to : from));
|
|
306
|
+
}
|
|
307
|
+
seen.delete(from);
|
|
308
|
+
seen.delete(to);
|
|
309
|
+
stats.mutations++;
|
|
310
|
+
return done(`Renamed ${from} to ${to}. Update [[links]] and ${INDEX_PATH} that referenced the old name.`);
|
|
311
|
+
},
|
|
312
|
+
|
|
313
|
+
submit(input) {
|
|
314
|
+
if (typeof input.summary !== "string" || input.summary.trim() === "") return fail("summary must be a non-empty string.");
|
|
315
|
+
const summary = clip(input.summary.trim(), 4_000);
|
|
316
|
+
if (role === "analyzer") {
|
|
317
|
+
const touched = Array.isArray(input.touched) ? input.touched.filter((p) => typeof p === "string").slice(0, 100) : [];
|
|
318
|
+
return { text: "Submitted.", isError: false, submission: { summary, touched } };
|
|
319
|
+
}
|
|
320
|
+
if (!Array.isArray(input.changes) || input.changes.some((c) => typeof c !== "string")) return fail("changes must be an array of strings.");
|
|
321
|
+
if (docs.list().length > 0) {
|
|
322
|
+
const index = docs.get(INDEX_PATH);
|
|
323
|
+
if (!index) return fail(`${INDEX_PATH} does not exist. Write the index (one "[[slug]] - one sentence" line per document, grouped under ## domain headings), then submit.`);
|
|
324
|
+
// Refused twice at most: a model that cannot shorten it must still be able to finish.
|
|
325
|
+
if (index.content.length > INDEX_MAX_CHARS && indexRejections++ < MAX_INDEX_REJECTIONS) {
|
|
326
|
+
return fail(`${INDEX_PATH} is ${index.content.length} characters; it is injected into every session and must stay under ${INDEX_MAX_CHARS}. Shorten entries, group or drop low-value ones, then submit again.`);
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
return { text: "Submitted.", isError: false, submission: { summary, changes: input.changes.slice(0, 100).map((c) => clip(c.trim(), 300)) } };
|
|
330
|
+
},
|
|
331
|
+
};
|
|
332
|
+
|
|
333
|
+
return {
|
|
334
|
+
definitions,
|
|
335
|
+
/** @returns {{text:string, isError:boolean, submission?:object}} */
|
|
336
|
+
execute(name, input) {
|
|
337
|
+
if (typeof name !== "string" || !allowed.has(name)) {
|
|
338
|
+
return fail(`Unknown tool "${short(name, 64)}". Available tools: ${definitions.map((d) => d.name).join(", ")}. There is no shell, network or file access.`);
|
|
339
|
+
}
|
|
340
|
+
const args = input && typeof input === "object" && !Array.isArray(input) ? input : {};
|
|
341
|
+
try {
|
|
342
|
+
return handlers[name](args);
|
|
343
|
+
} catch (e) {
|
|
344
|
+
return fail(`Tool ${name} failed: ${e?.message ?? e}`);
|
|
345
|
+
}
|
|
346
|
+
},
|
|
347
|
+
};
|
|
348
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
// Rough planning estimate of a dream's token use (order of magnitude, can be off by 2x either way).
|
|
2
|
+
// Model: ~4 characters per token; an analyzer keeps its input in the conversation and re-reads it every turn
|
|
3
|
+
// (cache reads count against the budget); ~4 turns per analyzer, ~6 for the organizer; each turn appends ~1,000
|
|
4
|
+
// tokens of tool results and writes ~600 tokens. The real usage is reported once a dream has run.
|
|
5
|
+
import { MAX_PART_CHARS } from "./basics.js";
|
|
6
|
+
|
|
7
|
+
const CHARS_PER_TOKEN = 4;
|
|
8
|
+
const FIXED_PROMPT_TOKENS = 2_500;
|
|
9
|
+
const ANALYZER_TURNS = 4;
|
|
10
|
+
const ORGANIZER_TURNS = 6;
|
|
11
|
+
const APPEND_PER_TURN = 1_000;
|
|
12
|
+
const OUTPUT_PER_TURN = 600;
|
|
13
|
+
const DOCS_READ_BY_ANALYZER = 0.25;
|
|
14
|
+
const DOCS_GROWTH_PER_INPUT = 0.15;
|
|
15
|
+
|
|
16
|
+
const toTokens = (chars) => Math.ceil(chars / CHARS_PER_TOKEN);
|
|
17
|
+
const triangle = (n) => (n * (n + 1)) / 2;
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* @param {{transcripts:{chars:number}[], existingDocsBytes?:number}} args
|
|
21
|
+
* @returns {{transcripts:number, chars:number, est_total_tokens:number, notes:string[]}}
|
|
22
|
+
*/
|
|
23
|
+
export function estimateDream({ transcripts, existingDocsBytes = 0 }) {
|
|
24
|
+
const list = Array.isArray(transcripts) ? transcripts : [];
|
|
25
|
+
const docTokens = toTokens(Math.max(0, existingDocsBytes));
|
|
26
|
+
let chars = 0;
|
|
27
|
+
let total = 0;
|
|
28
|
+
for (const t of list) {
|
|
29
|
+
const c = Math.max(0, Number(t?.chars) || 0);
|
|
30
|
+
chars += c;
|
|
31
|
+
const parts = Math.max(1, Math.ceil(c / MAX_PART_CHARS));
|
|
32
|
+
const partTokens = toTokens(c / parts);
|
|
33
|
+
const perPart =
|
|
34
|
+
ANALYZER_TURNS * (FIXED_PROMPT_TOKENS + partTokens) +
|
|
35
|
+
triangle(ANALYZER_TURNS) * APPEND_PER_TURN +
|
|
36
|
+
docTokens * DOCS_READ_BY_ANALYZER * (ANALYZER_TURNS / 2) +
|
|
37
|
+
ANALYZER_TURNS * OUTPUT_PER_TURN;
|
|
38
|
+
total += parts * perPart;
|
|
39
|
+
}
|
|
40
|
+
if (list.length) {
|
|
41
|
+
const docsAfter = docTokens + toTokens(chars) * DOCS_GROWTH_PER_INPUT;
|
|
42
|
+
total +=
|
|
43
|
+
ORGANIZER_TURNS * FIXED_PROMPT_TOKENS +
|
|
44
|
+
docsAfter * ORGANIZER_TURNS * 0.7 +
|
|
45
|
+
triangle(ORGANIZER_TURNS) * APPEND_PER_TURN * 0.5 +
|
|
46
|
+
ORGANIZER_TURNS * OUTPUT_PER_TURN;
|
|
47
|
+
}
|
|
48
|
+
const notes = [
|
|
49
|
+
"Rough estimate (order of magnitude, can be off by 2x): about 4 characters per token, ~4 turns per analyzer, ~6 for the organizer.",
|
|
50
|
+
"Cache reads count against the budget, so large inputs cost more than their size: every turn re-reads the input.",
|
|
51
|
+
"The real usage is reported on the dream once it has run.",
|
|
52
|
+
];
|
|
53
|
+
if (list.some((t) => (Number(t?.chars) || 0) > MAX_PART_CHARS)) {
|
|
54
|
+
notes.push(`Inputs over ${MAX_PART_CHARS.toLocaleString("en-US")} characters are split into parts that are analyzed one after another.`);
|
|
55
|
+
}
|
|
56
|
+
return { transcripts: list.length, chars, est_total_tokens: Math.round(total / 1000) * 1000, notes };
|
|
57
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
// The core library: no CLI concerns, no reads or writes of the memory folder. The CLI imports it from here.
|
|
2
|
+
export { sha256Hex, newId, parseFrontmatter, splitText, MAX_PART_CHARS } from "./basics.js";
|
|
3
|
+
export { scrubSecrets } from "./scrub.js";
|
|
4
|
+
export { MemoryWiki, validatePath, docError, MAX_DOC_BYTES, MAX_DOCS, MAX_PATH_CHARS } from "./wiki.js";
|
|
5
|
+
export { diffDocs, lineDiff } from "./diff.js";
|
|
6
|
+
export { buildContext, renderTranscript, renderNotes, INDEX_PATH } from "./text.js";
|
|
7
|
+
export { buildCorpus, searchCorpus, tokenize, stem } from "./search.js";
|
|
8
|
+
export { claudeCodeJsonlToEvents, ompJsonlToEvents } from "./sessions.js";
|
|
9
|
+
export { estimateDream } from "./estimate.js";
|
|
10
|
+
export { runDream, DEFAULT_CONCURRENCY, DEFAULT_MAX_TURNS, DEFAULT_MAX_TOKENS_PER_CALL } from "./dream/engine.js";
|
|
11
|
+
export { BudgetTracker } from "./dream/agent.js";
|
|
12
|
+
export { createToolbox, toolDefinitions, INDEX_MAX_CHARS } from "./dream/tools.js";
|
|
13
|
+
export { createProvider, createAnthropicProvider, createOpenAIProvider, LlmError, BudgetExhaustedError, AbortedError, normalizeUsage, zeroUsage } from "./providers/index.js";
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
// Shared plumbing for the LLM adapters: error types, usage accounting and a POST-JSON helper with bounded,
|
|
2
|
+
// abortable retries that honour retry-after.
|
|
3
|
+
import { scrubSecrets } from "../scrub.js";
|
|
4
|
+
|
|
5
|
+
export class LlmError extends Error {
|
|
6
|
+
constructor(message, { status = 0, type = "error", retryable = false, requestId = null } = {}) {
|
|
7
|
+
super(message);
|
|
8
|
+
this.name = "LlmError";
|
|
9
|
+
Object.assign(this, { status, type, retryable, requestId });
|
|
10
|
+
}
|
|
11
|
+
/** Bad key, no permission, unknown model or endpoint: no agent can succeed, so the whole dream stops. */
|
|
12
|
+
get fatal() {
|
|
13
|
+
return this.status === 401 || this.status === 403 || this.status === 404;
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export class BudgetExhaustedError extends Error {
|
|
18
|
+
constructor() {
|
|
19
|
+
super("Token budget exhausted.");
|
|
20
|
+
this.name = "BudgetExhaustedError";
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export class AbortedError extends Error {
|
|
25
|
+
constructor() {
|
|
26
|
+
super("Canceled.");
|
|
27
|
+
this.name = "AbortError";
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const USAGE_KEYS = ["input_tokens", "output_tokens", "cache_creation_input_tokens", "cache_read_input_tokens"];
|
|
32
|
+
|
|
33
|
+
export function zeroUsage() {
|
|
34
|
+
return { input_tokens: 0, output_tokens: 0, cache_creation_input_tokens: 0, cache_read_input_tokens: 0, total_tokens: 0 };
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function normalizeUsage(u) {
|
|
38
|
+
const out = zeroUsage();
|
|
39
|
+
for (const k of USAGE_KEYS) {
|
|
40
|
+
const v = u?.[k];
|
|
41
|
+
out[k] = Number.isFinite(v) && v > 0 ? Math.floor(v) : 0;
|
|
42
|
+
out.total_tokens += out[k];
|
|
43
|
+
}
|
|
44
|
+
return out;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const RETRY_STATUS = new Set([408, 409, 429, 500, 502, 503, 504, 529]);
|
|
48
|
+
|
|
49
|
+
function retryAfterMs(header) {
|
|
50
|
+
if (header == null || header === "") return null;
|
|
51
|
+
const secs = Number(header);
|
|
52
|
+
if (Number.isFinite(secs)) return Math.max(0, secs * 1000);
|
|
53
|
+
const at = Date.parse(header);
|
|
54
|
+
return Number.isNaN(at) ? null : Math.max(0, at - Date.now());
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function sleep(ms, signal) {
|
|
58
|
+
return new Promise((resolve, reject) => {
|
|
59
|
+
if (signal.aborted) return reject(new AbortedError());
|
|
60
|
+
const timer = setTimeout(() => {
|
|
61
|
+
signal.removeEventListener("abort", onAbort);
|
|
62
|
+
resolve();
|
|
63
|
+
}, ms);
|
|
64
|
+
function onAbort() {
|
|
65
|
+
clearTimeout(timer);
|
|
66
|
+
reject(new AbortedError());
|
|
67
|
+
}
|
|
68
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Removes the API key and anything key-shaped from text that may reach an error or an event. */
|
|
73
|
+
export function makeRedactor(apiKey) {
|
|
74
|
+
return (text) => {
|
|
75
|
+
let s = String(text);
|
|
76
|
+
if (apiKey) s = s.split(apiKey).join("[REDACTED:api_key]");
|
|
77
|
+
return scrubSecrets(s).text;
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* POST a JSON body and return the parsed JSON response, retrying transient failures with exponential backoff.
|
|
83
|
+
* Defaults: 5 attempts, 1 s base, 30 s cap, retry-after honoured up to 60 s, 300 s per request.
|
|
84
|
+
*/
|
|
85
|
+
export async function postJson({ url, headers, body, label, describeError, isBadBody, fetchImpl, signal, redact, retry = {} }) {
|
|
86
|
+
const { maxAttempts = 5, baseMs = 1000, maxMs = 30_000, maxRetryAfterMs = 60_000, requestTimeoutMs = 300_000 } = retry;
|
|
87
|
+
const payload = JSON.stringify(body);
|
|
88
|
+
|
|
89
|
+
async function once() {
|
|
90
|
+
const ctl = new AbortController();
|
|
91
|
+
const relay = () => ctl.abort();
|
|
92
|
+
signal.addEventListener("abort", relay, { once: true });
|
|
93
|
+
const timer = setTimeout(() => ctl.abort(), requestTimeoutMs);
|
|
94
|
+
let res;
|
|
95
|
+
let text;
|
|
96
|
+
try {
|
|
97
|
+
res = await fetchImpl(url, { method: "POST", headers: { ...headers, "content-type": "application/json" }, body: payload, signal: ctl.signal });
|
|
98
|
+
text = await res.text();
|
|
99
|
+
} catch (e) {
|
|
100
|
+
if (signal.aborted) throw new AbortedError();
|
|
101
|
+
throw new LlmError(redact(`Could not reach the ${label} API: ${e?.cause?.message ?? e?.message ?? e}`), { type: "network_error", retryable: true });
|
|
102
|
+
} finally {
|
|
103
|
+
clearTimeout(timer);
|
|
104
|
+
signal.removeEventListener("abort", relay);
|
|
105
|
+
}
|
|
106
|
+
let data = null;
|
|
107
|
+
try {
|
|
108
|
+
data = text ? JSON.parse(text) : null;
|
|
109
|
+
} catch {}
|
|
110
|
+
const requestId = res.headers?.get?.("request-id") ?? res.headers?.get?.("x-request-id") ?? null;
|
|
111
|
+
if (res.ok) {
|
|
112
|
+
if (!data || typeof data !== "object" || isBadBody(data)) {
|
|
113
|
+
throw new LlmError(`The ${label} API returned an unreadable response.`, { status: res.status, type: "bad_response", retryable: true, requestId });
|
|
114
|
+
}
|
|
115
|
+
return data;
|
|
116
|
+
}
|
|
117
|
+
const { type, message } = describeError(data, text ?? "");
|
|
118
|
+
const err = new LlmError(redact(`${label} API error ${res.status} (${type}): ${message}${res.status === 401 ? " Check the API key." : ""}`), {
|
|
119
|
+
status: res.status,
|
|
120
|
+
type,
|
|
121
|
+
retryable: RETRY_STATUS.has(res.status) || type === "overloaded_error",
|
|
122
|
+
requestId,
|
|
123
|
+
});
|
|
124
|
+
err.retryAfterMs = retryAfterMs(res.headers?.get?.("retry-after"));
|
|
125
|
+
throw err;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
if (signal.aborted) throw new AbortedError();
|
|
129
|
+
let last;
|
|
130
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
131
|
+
try {
|
|
132
|
+
return await once();
|
|
133
|
+
} catch (e) {
|
|
134
|
+
if (!(e instanceof LlmError) || !e.retryable) throw e;
|
|
135
|
+
last = e;
|
|
136
|
+
if (attempt === maxAttempts) break;
|
|
137
|
+
const backoff = Math.min(maxMs, baseMs * 2 ** (attempt - 1)) * (0.5 + Math.random() * 0.5);
|
|
138
|
+
await sleep(e.retryAfterMs != null ? Math.min(e.retryAfterMs, maxRetryAfterMs) : backoff, signal);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
last.message = `${last.message} (gave up after ${maxAttempts} attempts)`;
|
|
142
|
+
throw last;
|
|
143
|
+
}
|