nexorum 0.0.0-stage → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,179 @@
1
+ // LLM adapters. The engine speaks the Anthropic Messages block format internally (text, tool_use, tool_result);
2
+ // the OpenAI-compatible adapter converts to and from chat-completions with tool calling.
3
+ import { LlmError, makeRedactor, normalizeUsage, postJson } from "./http.js";
4
+
5
+ export { AbortedError, BudgetExhaustedError, LlmError, normalizeUsage, zeroUsage } from "./http.js";
6
+
7
+ const EPHEMERAL = { type: "ephemeral" };
8
+ export const ANTHROPIC_VERSION = "2023-06-01";
9
+ /** Tool errors travel in-band on the chat-completions side, because the "tool" role has no error flag. */
10
+ export const TOOL_ERROR_PREFIX = "ERROR: ";
11
+
12
+ const asBlocks = (content) => (typeof content === "string" ? [{ type: "text", text: content }] : content);
13
+
14
+ /** Moves a cache breakpoint onto the last block of the latest message (without touching the stored history). */
15
+ function cacheTail(messages) {
16
+ if (messages.length < 2) return messages;
17
+ const last = messages.at(-1);
18
+ if (!Array.isArray(last.content) || !last.content.length) return messages;
19
+ const blocks = [...last.content];
20
+ blocks[blocks.length - 1] = { ...blocks.at(-1), cache_control: EPHEMERAL };
21
+ return [...messages.slice(0, -1), { ...last, content: blocks }];
22
+ }
23
+
24
+ function anthropicStop(reason, content) {
25
+ if (reason === "tool_use" || reason === "max_tokens" || reason === "refusal") return reason;
26
+ if (reason === "end_turn" || reason === "stop_sequence" || reason === "pause_turn") return "end_turn";
27
+ return content.some((b) => b.type === "tool_use") ? "tool_use" : "end_turn";
28
+ }
29
+
30
+ /** Anthropic Messages API: POST {baseUrl}/v1/messages. */
31
+ export function createAnthropicProvider(llm, { fetchImpl, retry } = {}) {
32
+ const baseUrl = String(llm.baseUrl || "https://api.anthropic.com").replace(/\/+$/, "");
33
+ const redact = makeRedactor(llm.apiKey);
34
+ return {
35
+ name: "anthropic",
36
+ redact,
37
+ async callModel({ model, system, messages, tools, maxTokens, signal }) {
38
+ const data = await postJson({
39
+ url: `${baseUrl}/v1/messages`,
40
+ headers: { "x-api-key": llm.apiKey, "anthropic-version": llm.anthropicVersion || ANTHROPIC_VERSION },
41
+ body: {
42
+ model,
43
+ max_tokens: maxTokens,
44
+ system: [{ type: "text", text: system, cache_control: EPHEMERAL }],
45
+ tools,
46
+ messages: cacheTail(messages.map((m) => ({ ...m, content: asBlocks(m.content) }))),
47
+ },
48
+ label: "Anthropic",
49
+ describeError: (j, text) => ({ type: j?.error?.type ?? "error", message: j?.error?.message ?? text.slice(0, 300) }),
50
+ isBadBody: (j) => j.type === "error",
51
+ fetchImpl,
52
+ signal,
53
+ redact,
54
+ retry,
55
+ });
56
+ if (!Array.isArray(data.content)) throw new LlmError("The Anthropic API returned a response without content.", { type: "bad_response", retryable: true });
57
+ const content = data.content.filter(
58
+ (b) => b && typeof b === "object" && ((b.type === "text" && typeof b.text === "string") || (b.type === "tool_use" && typeof b.name === "string")),
59
+ );
60
+ return { content, stopReason: anthropicStop(data.stop_reason, content), usage: normalizeUsage(data.usage) };
61
+ },
62
+ };
63
+ }
64
+
65
+ const textOfBlocks = (content) =>
66
+ typeof content === "string"
67
+ ? content
68
+ : (content ?? [])
69
+ .filter((b) => b?.type === "text" && typeof b.text === "string")
70
+ .map((b) => b.text)
71
+ .join("\n");
72
+
73
+ function toChat(system, messages) {
74
+ const out = [{ role: "system", content: system }];
75
+ for (const m of messages) {
76
+ if (typeof m.content === "string") {
77
+ out.push({ role: m.role, content: m.content });
78
+ } else if (m.role === "assistant") {
79
+ const text = textOfBlocks(m.content);
80
+ const calls = m.content
81
+ .filter((b) => b?.type === "tool_use")
82
+ .map((b) => ({ id: b.id, type: "function", function: { name: b.name, arguments: JSON.stringify(b.input ?? {}) } }));
83
+ const msg = { role: "assistant", content: text || (calls.length ? null : "") };
84
+ if (calls.length) msg.tool_calls = calls;
85
+ out.push(msg);
86
+ } else {
87
+ for (const b of m.content) {
88
+ if (b?.type !== "tool_result") continue;
89
+ const body = typeof b.content === "string" ? b.content : textOfBlocks(b.content);
90
+ out.push({ role: "tool", tool_call_id: b.tool_use_id, content: b.is_error ? `${TOOL_ERROR_PREFIX}${body}` : body });
91
+ }
92
+ const text = textOfBlocks(m.content.filter((b) => b?.type === "text"));
93
+ if (text) out.push({ role: "user", content: text });
94
+ }
95
+ }
96
+ return out;
97
+ }
98
+
99
+ function parseArgs(raw) {
100
+ if (raw && typeof raw === "object") return { input: raw };
101
+ const s = typeof raw === "string" ? raw.trim() : "";
102
+ if (!s) return { input: {} };
103
+ try {
104
+ const v = JSON.parse(s);
105
+ return v && typeof v === "object" && !Array.isArray(v) ? { input: v } : { input: {}, parse_error: "arguments must be a JSON object" };
106
+ } catch (e) {
107
+ return { input: {}, parse_error: e.message };
108
+ }
109
+ }
110
+
111
+ function chatUsage(u) {
112
+ const num = (v) => (Number.isFinite(v) ? v : null);
113
+ const prompt = num(u?.prompt_tokens) ?? num(u?.input_tokens) ?? 0;
114
+ const completion = num(u?.completion_tokens) ?? num(u?.output_tokens) ?? 0;
115
+ const cached = u?.prompt_tokens_details?.cached_tokens ?? u?.cache_read_input_tokens ?? 0;
116
+ const cacheRead = Number.isFinite(cached) && cached > 0 ? Math.min(cached, prompt) : 0;
117
+ return normalizeUsage({
118
+ input_tokens: Math.max(0, prompt - cacheRead),
119
+ output_tokens: completion,
120
+ cache_read_input_tokens: cacheRead,
121
+ cache_creation_input_tokens: num(u?.cache_creation_input_tokens) ?? 0,
122
+ });
123
+ }
124
+
125
+ /** Any OpenAI-compatible endpoint: POST {baseUrl}/chat/completions (baseUrl includes the version, e.g. .../v1). */
126
+ export function createOpenAIProvider(llm, { fetchImpl, retry } = {}) {
127
+ const baseUrl = String(llm.baseUrl || "https://api.openai.com/v1").replace(/\/+$/, "");
128
+ const redact = makeRedactor(llm.apiKey);
129
+ let generated = 0;
130
+ return {
131
+ name: "openai",
132
+ redact,
133
+ async callModel({ model, system, messages, tools, maxTokens, signal }) {
134
+ const data = await postJson({
135
+ url: `${baseUrl}/chat/completions`,
136
+ headers: llm.apiKey ? { authorization: `Bearer ${llm.apiKey}` } : {},
137
+ body: {
138
+ model,
139
+ max_tokens: maxTokens,
140
+ messages: toChat(system, messages),
141
+ tools: tools.map((t) => ({ type: "function", function: { name: t.name, description: t.description, parameters: t.input_schema } })),
142
+ stream: false,
143
+ },
144
+ label: "OpenAI-compatible",
145
+ describeError: (j, text) => ({
146
+ type: String(j?.error?.type ?? j?.error?.code ?? "error"),
147
+ message: j?.error?.message ?? (typeof j?.error === "string" ? j.error : text.slice(0, 300)),
148
+ }),
149
+ isBadBody: (j) => !Array.isArray(j.choices) || j.choices.length === 0,
150
+ fetchImpl,
151
+ signal,
152
+ redact,
153
+ retry,
154
+ });
155
+ const choice = data.choices[0];
156
+ if (!choice?.message) throw new LlmError("The OpenAI-compatible API returned a choice without a message.", { type: "bad_response", retryable: true });
157
+ const msg = choice.message;
158
+ const content = [];
159
+ const text = typeof msg.content === "string" ? msg.content : Array.isArray(msg.content) ? textOfBlocks(msg.content) : "";
160
+ if (text) content.push({ type: "text", text });
161
+ for (const call of Array.isArray(msg.tool_calls) ? msg.tool_calls : []) {
162
+ const fn = call?.function;
163
+ if (!fn || typeof fn.name !== "string") continue;
164
+ content.push({ type: "tool_use", id: typeof call.id === "string" && call.id ? call.id : `call_${++generated}`, name: fn.name, ...parseArgs(fn.arguments) });
165
+ }
166
+ const hasTools = content.some((b) => b.type === "tool_use");
167
+ let stopReason = hasTools ? "tool_use" : "end_turn";
168
+ if (choice.finish_reason === "length") stopReason = "max_tokens";
169
+ else if (choice.finish_reason === "content_filter" && !hasTools) stopReason = "refusal";
170
+ return { content, stopReason, usage: chatUsage(data.usage) };
171
+ },
172
+ };
173
+ }
174
+
175
+ export function createProvider(llm, env) {
176
+ if (llm?.provider === "anthropic") return createAnthropicProvider(llm, env);
177
+ if (llm?.provider === "openai") return createOpenAIProvider(llm, env);
178
+ throw new Error(`Unknown LLM provider "${llm?.provider}". Use "anthropic" or "openai".`);
179
+ }
@@ -0,0 +1,82 @@
1
+ // Pattern-based secret scrubber. It runs on every transcript and note before it is stored and on every document a
2
+ // dream writes. Specific formats come before generic ones (sk-ant- before sk-, JWT before Bearer). The replacement
3
+ // "[REDACTED:<kind>]" matches no pattern, so scrubbing twice changes nothing.
4
+
5
+ // PEM private keys are scanned by hand: a lazy BEGIN...END regex rescans the remaining text for every BEGIN without
6
+ // an END, which is quadratic on hostile input.
7
+ const PEM_BEGIN = /-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----/g;
8
+ const PEM_END = /-----END [A-Z0-9 ]*PRIVATE KEY-----/g;
9
+ const PEM_LINES = /(?:\r?\n[A-Za-z0-9+/=]{16,})+/y;
10
+
11
+ function scrubPem(text, label) {
12
+ let out = "";
13
+ let from = 0;
14
+ let hits = 0;
15
+ let endMissing = false;
16
+ PEM_BEGIN.lastIndex = 0;
17
+ for (let m = PEM_BEGIN.exec(text); m; m = PEM_BEGIN.exec(text)) {
18
+ const bodyAt = m.index + m[0].length;
19
+ let stop = -1;
20
+ if (!endMissing) {
21
+ PEM_END.lastIndex = bodyAt;
22
+ const e = PEM_END.exec(text);
23
+ if (e) stop = e.index + e[0].length;
24
+ else endMissing = true;
25
+ }
26
+ if (stop < 0) {
27
+ // Truncated key: take the base64 lines that follow the header.
28
+ PEM_LINES.lastIndex = bodyAt;
29
+ const b = PEM_LINES.exec(text);
30
+ if (!b) continue;
31
+ stop = bodyAt + b[0].length;
32
+ }
33
+ out += text.slice(from, m.index) + label;
34
+ from = stop;
35
+ hits++;
36
+ PEM_BEGIN.lastIndex = stop;
37
+ }
38
+ return hits ? { text: out + text.slice(from), hits } : { text, hits: 0 };
39
+ }
40
+
41
+ const RULES = [
42
+ ["private_key", null],
43
+ ["anthropic_key", /sk-ant-[\w-]{20,}/g],
44
+ ["openai_key", /sk-[A-Za-z0-9_-]{20,}/g],
45
+ ["nexorum_key", /\bnxr_[A-Za-z0-9_-]{32,}/g],
46
+ ["aws_access_key", /\bAKIA[0-9A-Z]{16,}/g],
47
+ ["github_token", /\bgh[pousr]_[A-Za-z0-9]{36,}/g],
48
+ ["npm_token", /\bnpm_[A-Za-z0-9]{36}\b/g],
49
+ ["slack_token", /\bxox[baprs]-[A-Za-z0-9-]{10,}/g],
50
+ ["google_api_key", /\bAIza[0-9A-Za-z_-]{35}/g],
51
+ ["jwt", /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}/g],
52
+ ["bearer_token", /\bBearer\s+[A-Za-z0-9._~+/=-]{20,}/g],
53
+ ];
54
+
55
+ /**
56
+ * @param {string} text
57
+ * @returns {{text:string, count:number, kinds:string[]}}
58
+ */
59
+ export function scrubSecrets(text) {
60
+ let out = String(text);
61
+ let count = 0;
62
+ const kinds = [];
63
+ for (const [kind, re] of RULES) {
64
+ const label = `[REDACTED:${kind}]`;
65
+ let hits = 0;
66
+ if (re === null) {
67
+ const r = scrubPem(out, label);
68
+ out = r.text;
69
+ hits = r.hits;
70
+ } else {
71
+ out = out.replace(re, () => {
72
+ hits++;
73
+ return label;
74
+ });
75
+ }
76
+ if (hits) {
77
+ count += hits;
78
+ kinds.push(kind);
79
+ }
80
+ }
81
+ return { text: out, count, kinds };
82
+ }
@@ -0,0 +1,107 @@
1
+ // BM25 keyword search over notes and wiki pages. No embeddings: agents navigate through /_index.md, [[links]] and
2
+ // this search. Deterministic for the same input.
3
+ import { parseFrontmatter } from "./basics.js";
4
+
5
+ const K1 = 1.5;
6
+ const B = 0.75;
7
+ const TITLE_WEIGHT = 3;
8
+ const STOP_WORDS = new Set("a an and are as at be but by for from has have i in is it its of on or that the this to was were will with you".split(" "));
9
+
10
+ const dropDouble = (w) => (/([^aeiou])\1$/.test(w) && w.length > 3 ? w.slice(0, -1) : w);
11
+
12
+ /** Light suffix stemming, the same for documents and queries. */
13
+ export function stem(word) {
14
+ const w = word;
15
+ if (w.length > 4 && w.endsWith("ies")) return `${w.slice(0, -3)}y`;
16
+ if (w.length > 4 && w.endsWith("sses")) return w.slice(0, -2);
17
+ if (w.length > 5 && w.endsWith("ing")) return dropDouble(w.slice(0, -3));
18
+ if (w.length > 4 && w.endsWith("ed")) return dropDouble(w.slice(0, -2));
19
+ if (w.length > 3 && w.endsWith("s") && !/(ss|us|is)$/.test(w)) return w.slice(0, -1);
20
+ return w;
21
+ }
22
+
23
+ /** Lower-cased, NFKC-normalised, stop-word-free, stemmed unicode letter/digit runs. */
24
+ export function tokenize(text) {
25
+ const out = [];
26
+ for (const [w] of String(text ?? "").normalize("NFKC").toLowerCase().matchAll(/[\p{L}\p{N}]+/gu)) {
27
+ if (!STOP_WORDS.has(w)) out.push(stem(w));
28
+ }
29
+ return out;
30
+ }
31
+
32
+ const clip = (s, n) => (s.length <= n ? s : `${s.slice(0, n - 1).trimEnd()}…`);
33
+
34
+ /** The line with the most query terms (first line when none match), clipped to 240 characters. */
35
+ function snippet(text, terms) {
36
+ const lines = String(text).split("\n").map((l) => l.trim()).filter((l) => l && l !== "---");
37
+ let best = null;
38
+ let bestHits = 0;
39
+ for (const line of lines) {
40
+ const set = new Set(tokenize(line));
41
+ const hits = terms.reduce((n, t) => n + (set.has(t) ? 1 : 0), 0);
42
+ if (hits > bestHits) {
43
+ bestHits = hits;
44
+ best = line;
45
+ }
46
+ }
47
+ return clip(best ?? lines[0] ?? "", 240);
48
+ }
49
+
50
+ /**
51
+ * Notes are titled by their first line; wiki pages by frontmatter name (else the slug), and their text is the
52
+ * description plus the body.
53
+ * @returns {{id:string, kind:"note"|"wiki", title:string, text:string}[]}
54
+ */
55
+ export function buildCorpus({ notes = [], wiki = {} } = {}) {
56
+ const docs = [];
57
+ for (const n of Array.isArray(notes) ? notes : []) {
58
+ if (!n || typeof n.content !== "string") continue;
59
+ docs.push({ id: n.id, kind: "note", title: clip(n.content.trim().split("\n")[0] ?? "", 80), text: n.content });
60
+ }
61
+ const snap = wiki && typeof wiki.snapshot === "function" ? wiki.snapshot() : (wiki ?? {}); // a MemoryWiki works too
62
+ for (const [p, content] of Object.entries(snap).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) {
63
+ const fm = parseFrontmatter(content);
64
+ const slug = p.replace(/^.*\//, "").replace(/\.[^.]+$/, "");
65
+ docs.push({ id: p, kind: "wiki", title: fm.name ?? slug, text: `${fm.description ? `${fm.description}\n` : ""}${fm.body}` });
66
+ }
67
+ return docs;
68
+ }
69
+
70
+ /**
71
+ * BM25 (k1 1.5, b 0.75); title tokens count three times in term frequency and length.
72
+ * @returns {{id:string, kind:string, score:number, title:string, snippet:string}[]} best first, ties by id
73
+ */
74
+ export function searchCorpus(docs, query, { topK = 5 } = {}) {
75
+ const terms = [...new Set(tokenize(query))];
76
+ if (!terms.length || !Array.isArray(docs) || !docs.length) return [];
77
+ const rows = docs.map((d) => {
78
+ const tf = new Map();
79
+ let len = 0;
80
+ for (const t of tokenize(d.text)) {
81
+ tf.set(t, (tf.get(t) ?? 0) + 1);
82
+ len++;
83
+ }
84
+ for (const t of tokenize(d.title)) {
85
+ tf.set(t, (tf.get(t) ?? 0) + TITLE_WEIGHT);
86
+ len += TITLE_WEIGHT;
87
+ }
88
+ return { d, tf, len };
89
+ });
90
+ const N = rows.length;
91
+ const avg = rows.reduce((s, r) => s + r.len, 0) / N || 1;
92
+ const idf = new Map(terms.map((t) => {
93
+ const df = rows.filter((r) => r.tf.has(t)).length;
94
+ return [t, Math.log(1 + (N - df + 0.5) / (df + 0.5))];
95
+ }));
96
+ const hits = [];
97
+ for (const r of rows) {
98
+ let score = 0;
99
+ for (const t of terms) {
100
+ const f = r.tf.get(t);
101
+ if (f) score += idf.get(t) * ((f * (K1 + 1)) / (f + K1 * (1 - B + B * (r.len / avg))));
102
+ }
103
+ if (score > 0) hits.push({ id: r.d.id, kind: r.d.kind, score, title: r.d.title, snippet: snippet(r.d.text, terms) });
104
+ }
105
+ hits.sort((a, b) => b.score - a.score || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
106
+ return hits.slice(0, Math.max(0, topK));
107
+ }
@@ -0,0 +1,171 @@
1
+ // Converters from other agents' session files to Nexorum transcript events ({role, content, name?, ts?}).
2
+ // Both are tolerant: lines they cannot use are skipped. Thinking blocks are dropped, tool output is clipped
3
+ // (it dominates real sessions and rarely holds the lesson), and only the most recent `maxEvents` are kept.
4
+
5
+ const isObj = (v) => v !== null && typeof v === "object" && !Array.isArray(v);
6
+
7
+ function json(v) {
8
+ try {
9
+ return typeof v === "string" ? v : (JSON.stringify(v) ?? "");
10
+ } catch {
11
+ return "";
12
+ }
13
+ }
14
+
15
+ function blockText(content, { images = false } = {}) {
16
+ if (typeof content === "string") return content;
17
+ if (!Array.isArray(content)) return "";
18
+ return content
19
+ .map((b) => {
20
+ if (typeof b === "string") return b;
21
+ if (!isObj(b)) return "";
22
+ if (b.type === "text" && typeof b.text === "string") return b.text;
23
+ if (images && b.type === "image") return "[image]";
24
+ return "";
25
+ })
26
+ .filter(Boolean)
27
+ .join("\n");
28
+ }
29
+
30
+ function collector({ maxContentChars, maxToolChars }) {
31
+ const events = [];
32
+ const push = (role, content, name, ts) => {
33
+ if (typeof content !== "string" || content.trim() === "") return;
34
+ const limit = role === "tool" ? Math.min(maxToolChars, maxContentChars) : maxContentChars;
35
+ let text = content.length > limit ? content.slice(0, limit) : content;
36
+ if (role === "tool" && text.length < content.length) text = `${text}\n… [clipped ${content.length - text.length} more characters]`;
37
+ const ev = { role, content: text };
38
+ if (name) ev.name = name.slice(0, 64);
39
+ if (ts) ev.ts = ts;
40
+ events.push(ev);
41
+ };
42
+ return { events, push };
43
+ }
44
+
45
+ const isoOf = (v) => (typeof v === "string" && !Number.isNaN(Date.parse(v)) ? new Date(v).toISOString() : undefined);
46
+
47
+ /**
48
+ * Claude Code session JSONL. Sidechain (subagent) and meta lines are skipped; tool_use becomes a "tool" event
49
+ * "call: <json>", tool_result a "tool" event "result: ..." named after its call.
50
+ * @returns {{events:object[], skipped:number}}
51
+ */
52
+ export function claudeCodeJsonlToEvents(jsonl, { maxEvents = 5000, maxContentChars = 50_000, maxToolChars = 4_000 } = {}) {
53
+ const { events, push } = collector({ maxContentChars, maxToolChars });
54
+ const toolNames = new Map();
55
+ let skipped = 0;
56
+ for (const line of String(jsonl ?? "").split("\n")) {
57
+ if (!line.trim()) continue;
58
+ const before = events.length;
59
+ readLine(line);
60
+ if (events.length === before) skipped++;
61
+ }
62
+ return { events: events.length > maxEvents ? events.slice(events.length - maxEvents) : events, skipped };
63
+
64
+ function readLine(line) {
65
+ let entry;
66
+ try {
67
+ entry = JSON.parse(line);
68
+ } catch {
69
+ return;
70
+ }
71
+ if (!isObj(entry) || entry.isSidechain === true || entry.isMeta === true) return;
72
+ const msg = entry.message;
73
+ if (!isObj(msg) || (msg.role !== "user" && msg.role !== "assistant")) return;
74
+ const ts = isoOf(entry.timestamp);
75
+ if (typeof msg.content === "string") return push(msg.role, msg.content, undefined, ts);
76
+ if (!Array.isArray(msg.content)) return;
77
+ let texts = [];
78
+ const flush = () => {
79
+ if (texts.length) push(msg.role, texts.join("\n"), undefined, ts);
80
+ texts = [];
81
+ };
82
+ for (const b of msg.content) {
83
+ if (typeof b === "string") texts.push(b);
84
+ else if (!isObj(b)) continue;
85
+ else if (b.type === "text" && typeof b.text === "string") texts.push(b.text);
86
+ else if (b.type === "tool_use") {
87
+ flush();
88
+ const name = typeof b.name === "string" && b.name ? b.name : "tool";
89
+ if (typeof b.id === "string") toolNames.set(b.id, name);
90
+ push("tool", `call: ${json(b.input)}`, name, ts);
91
+ } else if (b.type === "tool_result") {
92
+ flush();
93
+ push("tool", `result: ${blockText(b.content) || json(b.content)}`, toolNames.get(b.tool_use_id) ?? "tool_result", ts);
94
+ }
95
+ }
96
+ flush();
97
+ }
98
+ }
99
+
100
+ /**
101
+ * omp (oh-my-pi) session JSONL: a `session` header line (id, cwd, timestamp), an optional `title` line and
102
+ * `message` lines with roles user / assistant (text, thinking, toolCall blocks) / toolResult.
103
+ * @returns {{events:object[], skipped:number, meta:{id:string|null, cwd:string|null, startedAt:string|null, title:string|null}}}
104
+ */
105
+ export function ompJsonlToEvents(jsonl, { maxEvents = 5000, maxContentChars = 50_000, maxToolChars = 4_000 } = {}) {
106
+ const { events, push } = collector({ maxContentChars, maxToolChars });
107
+ const toolNames = new Map();
108
+ const meta = { id: null, cwd: null, startedAt: null, title: null };
109
+ let skipped = 0;
110
+ for (const line of String(jsonl ?? "").split("\n")) {
111
+ if (!line.trim()) continue;
112
+ const before = events.length;
113
+ if (!readLine(line) && events.length === before) skipped++;
114
+ }
115
+ return { events: events.length > maxEvents ? events.slice(events.length - maxEvents) : events, skipped, meta };
116
+
117
+ function readLine(line) {
118
+ let entry;
119
+ try {
120
+ entry = JSON.parse(line);
121
+ } catch {
122
+ return false;
123
+ }
124
+ if (!isObj(entry)) return false;
125
+ if (entry.type === "session") {
126
+ if (typeof entry.id === "string") meta.id = entry.id;
127
+ if (typeof entry.cwd === "string") meta.cwd = entry.cwd;
128
+ if (typeof entry.timestamp === "string") meta.startedAt = entry.timestamp;
129
+ return true;
130
+ }
131
+ if (entry.type === "title") {
132
+ if (typeof entry.title === "string") meta.title = entry.title.trim();
133
+ return true;
134
+ }
135
+ if (entry.type !== "message" || !isObj(entry.message)) return false;
136
+ const msg = entry.message;
137
+ const ts = isoOf(entry.timestamp);
138
+ if (msg.role === "user") {
139
+ push("user", blockText(msg.content, { images: true }), undefined, ts);
140
+ return true;
141
+ }
142
+ if (msg.role === "toolResult") {
143
+ const name = typeof msg.toolName === "string" && msg.toolName ? msg.toolName : (toolNames.get(msg.toolCallId) ?? "tool");
144
+ push("tool", `${msg.isError === true ? "error" : "result"}: ${blockText(msg.content, { images: true }) || json(msg.content)}`, name, ts);
145
+ return true;
146
+ }
147
+ if (msg.role !== "assistant") return false;
148
+ if (typeof msg.content === "string") {
149
+ push("assistant", msg.content, undefined, ts);
150
+ return true;
151
+ }
152
+ if (!Array.isArray(msg.content)) return true;
153
+ let texts = [];
154
+ const flush = () => {
155
+ if (texts.length) push("assistant", texts.join("\n"), undefined, ts);
156
+ texts = [];
157
+ };
158
+ for (const b of msg.content) {
159
+ if (!isObj(b)) continue;
160
+ if (b.type === "text" && typeof b.text === "string") texts.push(b.text);
161
+ else if (b.type === "toolCall") {
162
+ flush();
163
+ const name = typeof b.name === "string" && b.name ? b.name : "tool";
164
+ if (typeof b.id === "string") toolNames.set(b.id, name);
165
+ push("tool", `call: ${json(b.arguments)}`, name, ts);
166
+ }
167
+ }
168
+ flush();
169
+ return true;
170
+ }
171
+ }
@@ -0,0 +1,72 @@
1
+ // How inputs are shown to the dream agents (labelled plain text), and the context digest injected at session start.
2
+
3
+ export const INDEX_PATH = "/_index.md";
4
+
5
+ // A content line that looks like one of our labels gets a leading space, so data cannot fake a new turn.
6
+ const LOOKS_LIKE_LABEL = /^(?:USER:|ASSISTANT:|SYSTEM:|TOOL .*:|TRANSCRIPT\s|NOTES\s)/;
7
+ const LABEL = { user: "USER", assistant: "ASSISTANT", system: "SYSTEM" };
8
+
9
+ function body(text) {
10
+ return String(text ?? "")
11
+ .replace(/\r\n?/g, "\n")
12
+ .replace(/\n+$/, "")
13
+ .split("\n")
14
+ .map((line) => (LOOKS_LIKE_LABEL.test(line) ? ` ${line}` : line))
15
+ .join("\n");
16
+ }
17
+
18
+ /** @param {{id:string, source:string, events:{role:string, content:string, name?:string}[]}} t */
19
+ export function renderTranscript({ id, source, events }) {
20
+ const list = Array.isArray(events) ? events : [];
21
+ const blocks = list.map((e) => {
22
+ if (e?.role === "tool") {
23
+ const name = String(e.name || "tool").replace(/[\r\n:]+/g, " ").trim() || "tool";
24
+ return `TOOL ${name}:\n${body(e.content)}`;
25
+ }
26
+ return `${LABEL[e?.role] ?? "SYSTEM"}:\n${body(e?.content)}`;
27
+ });
28
+ return [`TRANSCRIPT ${id} source=${source} events=${list.length}`, ...blocks].join("\n\n");
29
+ }
30
+
31
+ /** @param {{id:string, ts:string, content:string}[]} notes */
32
+ export function renderNotes(notes) {
33
+ const list = Array.isArray(notes) ? notes : [];
34
+ const head = `NOTES source=notes count=${list.length}`;
35
+ if (!list.length) return head;
36
+ const lines = list.map((n) => `- [${n.ts}] (${n.id}) ${String(n.content ?? "").replace(/\r\n?/g, "\n").replace(/\n+$/, "").replace(/\n/g, "\n ")}`);
37
+ return `${head}\n\n${lines.join("\n")}`;
38
+ }
39
+
40
+ /**
41
+ * The digest: /_index.md cut at the last whole line that fits, or else the newest notes ("Recent notes:\n- ...").
42
+ * @param {Record<string,string>|{get:(p:string)=>({content:string}|null)}|null} wiki a snapshot or a document API such as
43
+ * MemoryWiki (only the index is read)
44
+ * @returns {{content:string, source:"index"|"notes"|"empty", truncated:boolean}}
45
+ */
46
+ export function buildContext(wiki, notes = [], { maxChars = 4096 } = {}) {
47
+ let index = null;
48
+ if (wiki && typeof wiki.get === "function") index = wiki.get(INDEX_PATH)?.content ?? null;
49
+ else if (wiki && typeof wiki === "object" && Object.hasOwn(wiki, INDEX_PATH)) index = String(wiki[INDEX_PATH]);
50
+ if (index !== null && index.trim() !== "") {
51
+ if (index.length <= maxChars) return { content: index, source: "index", truncated: false };
52
+ let cut = index.lastIndexOf("\n", maxChars);
53
+ if (cut <= 0) cut = maxChars;
54
+ return { content: index.slice(0, cut), source: "index", truncated: true };
55
+ }
56
+ const newest = (Array.isArray(notes) ? notes : [])
57
+ .map((n, i) => ({ n, i }))
58
+ .filter(({ n }) => n && typeof n.content === "string" && n.content.trim() !== "")
59
+ .sort((a, b) => (a.n.ts === b.n.ts ? b.i - a.i : a.n.ts < b.n.ts ? 1 : -1))
60
+ .map(({ n }) => n);
61
+ if (!newest.length) return { content: "", source: "empty", truncated: false };
62
+ let content = "Recent notes:";
63
+ let used = 0;
64
+ for (const n of newest) {
65
+ const line = `\n- ${n.content.replace(/\s+/g, " ").trim()}`;
66
+ if (content.length + line.length > maxChars) break;
67
+ content += line;
68
+ used++;
69
+ }
70
+ if (!used) return { content: "", source: "empty", truncated: true };
71
+ return { content, source: "notes", truncated: used < newest.length };
72
+ }