nexorum 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +9 -0
- package/README.md +144 -2
- package/bin/nexorum.js +12 -0
- package/package.json +46 -4
- package/src/agent-dream.js +277 -0
- package/src/agents-md.js +80 -0
- package/src/args.js +71 -0
- package/src/cli.js +278 -0
- package/src/core/basics.js +88 -0
- package/src/core/diff.js +63 -0
- package/src/core/dream/agent.js +109 -0
- package/src/core/dream/engine.js +223 -0
- package/src/core/dream/prompts.js +112 -0
- package/src/core/dream/tools.js +348 -0
- package/src/core/estimate.js +57 -0
- package/src/core/index.js +13 -0
- package/src/core/providers/http.js +143 -0
- package/src/core/providers/index.js +179 -0
- package/src/core/scrub.js +82 -0
- package/src/core/search.js +107 -0
- package/src/core/sessions.js +171 -0
- package/src/core/text.js +72 -0
- package/src/core/wiki.js +102 -0
- package/src/doctor.js +177 -0
- package/src/dream.js +309 -0
- package/src/hook.js +101 -0
- package/src/import.js +137 -0
- package/src/init.js +266 -0
- package/src/mcp.js +376 -0
- package/src/memory.js +188 -0
- package/src/proposals.js +204 -0
- package/src/store.js +445 -0
- package/src/transcript.js +40 -0
- package/src/version.js +4 -0
package/src/core/wiki.js
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
// In-memory wiki with the synchronous document API the dream engine works against (the staging wiki).
|
|
2
|
+
// Errors carry `.code`: "conflict" | "not_found" | "invalid_path" | "too_large" | "quota".
|
|
3
|
+
import { parseFrontmatter, sha256Hex } from "./basics.js";
|
|
4
|
+
|
|
5
|
+
export const MAX_PATH_CHARS = 200;
|
|
6
|
+
export const MAX_DOC_BYTES = 100_000;
|
|
7
|
+
export const MAX_DOCS = 2000;
|
|
8
|
+
|
|
9
|
+
const PATH_RE = /^\/[A-Za-z0-9._-]+(\/[A-Za-z0-9._-]+)*$/;
|
|
10
|
+
|
|
11
|
+
export function docError(code, message) {
|
|
12
|
+
return Object.assign(new Error(message), { code });
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/** Same rule as the CLI's isDocPath: whatever a dream may write, `apply` must be able to write to disk. */
|
|
16
|
+
export function validatePath(p) {
|
|
17
|
+
const okay = typeof p === "string" && p.length <= MAX_PATH_CHARS && PATH_RE.test(p) && !p.includes("..") && !p.split("/").includes(".");
|
|
18
|
+
if (!okay) {
|
|
19
|
+
throw docError(
|
|
20
|
+
"invalid_path",
|
|
21
|
+
`Invalid path: it must start with "/", use segments of letters, digits, ".", "_" or "-", contain no ".." and no "." segment, and be at most ${MAX_PATH_CHARS} characters.`,
|
|
22
|
+
);
|
|
23
|
+
}
|
|
24
|
+
return p;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const bytesOf = (s) => Buffer.byteLength(s, "utf8");
|
|
28
|
+
const byPath = ([a], [b]) => (a < b ? -1 : a > b ? 1 : 0);
|
|
29
|
+
|
|
30
|
+
export class MemoryWiki {
|
|
31
|
+
constructor(initial = {}, { maxDocs = MAX_DOCS, maxBytes = MAX_DOC_BYTES } = {}) {
|
|
32
|
+
this.maxDocs = maxDocs;
|
|
33
|
+
this.maxBytes = maxBytes;
|
|
34
|
+
this.docs = new Map();
|
|
35
|
+
for (const [p, content] of Object.entries(initial ?? {})) this.#put(validatePath(p), String(content));
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
#put(p, content) {
|
|
39
|
+
const sha256 = sha256Hex(content);
|
|
40
|
+
this.docs.set(p, { content, sha256, bytes: bytesOf(content) });
|
|
41
|
+
return { path: p, sha256 };
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
list({ prefix } = {}) {
|
|
45
|
+
if (prefix && !String(prefix).startsWith("/")) throw docError("invalid_path", 'prefix must start with "/"');
|
|
46
|
+
return [...this.docs.entries()]
|
|
47
|
+
.filter(([p]) => !prefix || p.startsWith(prefix))
|
|
48
|
+
.sort(byPath)
|
|
49
|
+
.map(([p, d]) => {
|
|
50
|
+
const fm = parseFrontmatter(d.content);
|
|
51
|
+
return { path: p, bytes: d.bytes, sha256: d.sha256, name: fm.name, description: fm.description };
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
get(p) {
|
|
56
|
+
validatePath(p);
|
|
57
|
+
const d = this.docs.get(p);
|
|
58
|
+
return d ? { path: p, content: d.content, sha256: d.sha256 } : null;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
has(p) {
|
|
62
|
+
return this.docs.has(p);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** Create or replace. With `expectSha256` the current content must have that hash (a missing doc never matches). */
|
|
66
|
+
write(p, content, { expectSha256 } = {}) {
|
|
67
|
+
validatePath(p);
|
|
68
|
+
if (typeof content !== "string") throw docError("too_large", "content must be a string");
|
|
69
|
+
if (bytesOf(content) > this.maxBytes) throw docError("too_large", `Document exceeds ${this.maxBytes} bytes.`);
|
|
70
|
+
const cur = this.docs.get(p);
|
|
71
|
+
if (expectSha256 != null && (cur?.sha256 ?? null) !== expectSha256) throw docError("conflict", `${p} changed since it was read.`);
|
|
72
|
+
if (!cur && this.docs.size >= this.maxDocs) throw docError("quota", `Document limit of ${this.maxDocs} reached.`);
|
|
73
|
+
return this.#put(p, content);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
delete(p) {
|
|
77
|
+
validatePath(p);
|
|
78
|
+
if (!this.docs.delete(p)) throw docError("not_found", `${p} does not exist.`);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
rename(from, to) {
|
|
82
|
+
validatePath(from);
|
|
83
|
+
validatePath(to);
|
|
84
|
+
const d = this.docs.get(from);
|
|
85
|
+
if (!d) throw docError("not_found", `${from} does not exist.`);
|
|
86
|
+
if (this.docs.has(to)) throw docError("conflict", `${to} already exists.`);
|
|
87
|
+
this.docs.delete(from);
|
|
88
|
+
this.docs.set(to, d);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** @returns {Record<string,string>} path -> content, sorted by path */
|
|
92
|
+
snapshot() {
|
|
93
|
+
return Object.fromEntries([...this.docs.entries()].sort(byPath).map(([p, d]) => [p, d.content]));
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** An independent copy with the same limits. */
|
|
97
|
+
clone() {
|
|
98
|
+
const c = new MemoryWiki({}, { maxDocs: this.maxDocs, maxBytes: this.maxBytes });
|
|
99
|
+
for (const [p, d] of this.docs) c.docs.set(p, { ...d });
|
|
100
|
+
return c;
|
|
101
|
+
}
|
|
102
|
+
}
|
package/src/doctor.js
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
// `nexorum doctor [--json]`: read-only health check. One PASS/WARN/FAIL line per check; exit 1 when any check fails.
|
|
2
|
+
// No network call; the API key is only ever reported as set or missing.
|
|
3
|
+
import fs from "node:fs";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { spawnSync } from "node:child_process";
|
|
6
|
+
import { nexorumRoot, userHome, isSafeId } from "./store.js";
|
|
7
|
+
import { findBinary, hasHook } from "./init.js";
|
|
8
|
+
import { DEFAULT_ANTHROPIC_MODEL, DEFAULT_OPENAI_BASE_URL } from "./dream.js";
|
|
9
|
+
|
|
10
|
+
const out = (s = "") => process.stdout.write(`${s}\n`);
|
|
11
|
+
const check = (name, status, detail, hint = "") => ({ name, status, detail, hint });
|
|
12
|
+
|
|
13
|
+
function checkNode() {
|
|
14
|
+
const major = Number(process.versions.node.split(".")[0]);
|
|
15
|
+
return major >= 22 ? check("node", "PASS", `Node ${process.versions.node}`) : check("node", "FAIL", `Node ${process.versions.node} is too old`, "Install Node 22 or newer.");
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function checkHome(env) {
|
|
19
|
+
const root = nexorumRoot(env);
|
|
20
|
+
let probe = root;
|
|
21
|
+
while (!fs.existsSync(probe)) {
|
|
22
|
+
const up = path.dirname(probe);
|
|
23
|
+
if (up === probe) break;
|
|
24
|
+
probe = up;
|
|
25
|
+
}
|
|
26
|
+
const exists = probe === root;
|
|
27
|
+
try {
|
|
28
|
+
if (!fs.statSync(probe).isDirectory()) throw new Error("not a directory");
|
|
29
|
+
fs.accessSync(probe, fs.constants.W_OK);
|
|
30
|
+
} catch {
|
|
31
|
+
const why = exists ? "is not a writable directory" : `cannot be created (${probe} is not a writable directory)`;
|
|
32
|
+
return check("home", "FAIL", `${root} ${why}`, "Set NEXORUM_HOME to a writable folder or fix its permissions.");
|
|
33
|
+
}
|
|
34
|
+
return exists
|
|
35
|
+
? check("home", "PASS", `${root} exists and is writable`)
|
|
36
|
+
: check("home", "PASS", `${root} does not exist yet but can be created`, "It is created on first use.");
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Parses every file of the space and counts what cannot be read. Never writes. */
|
|
40
|
+
function checkSpaceFiles(store) {
|
|
41
|
+
if (!fs.existsSync(store.dir)) return check("space files", "PASS", `space "${store.space}" has no data yet`, 'Run: nexorum remember "..." or nexorum log.');
|
|
42
|
+
let total = 0;
|
|
43
|
+
const bad = [];
|
|
44
|
+
const rel = (f) => path.relative(store.dir, f);
|
|
45
|
+
const list = (dir) => {
|
|
46
|
+
try {
|
|
47
|
+
return fs.readdirSync(dir);
|
|
48
|
+
} catch {
|
|
49
|
+
return [];
|
|
50
|
+
}
|
|
51
|
+
};
|
|
52
|
+
const tryJson = (file, valid) => {
|
|
53
|
+
total++;
|
|
54
|
+
try {
|
|
55
|
+
if (!valid(JSON.parse(fs.readFileSync(file, "utf8")))) throw new Error("unexpected shape");
|
|
56
|
+
} catch {
|
|
57
|
+
bad.push(rel(file));
|
|
58
|
+
}
|
|
59
|
+
};
|
|
60
|
+
const tryLines = (file) => {
|
|
61
|
+
let text;
|
|
62
|
+
try {
|
|
63
|
+
text = fs.readFileSync(file, "utf8");
|
|
64
|
+
} catch (e) {
|
|
65
|
+
if (e.code === "ENOENT") return;
|
|
66
|
+
total++;
|
|
67
|
+
bad.push(rel(file));
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
let n = 0;
|
|
71
|
+
for (const line of text.split("\n")) {
|
|
72
|
+
if (!line.trim()) continue;
|
|
73
|
+
n++;
|
|
74
|
+
total++;
|
|
75
|
+
try {
|
|
76
|
+
JSON.parse(line);
|
|
77
|
+
} catch {
|
|
78
|
+
bad.push(`${rel(file)} line ${n}`);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
const isObject = (v) => v && typeof v === "object" && !Array.isArray(v);
|
|
83
|
+
for (const n of list(store.transcriptsDir)) if (n.endsWith(".json")) tryJson(path.join(store.transcriptsDir, n), (t) => t && typeof t.id === "string" && Array.isArray(t.events));
|
|
84
|
+
tryLines(store.notesFile);
|
|
85
|
+
if (fs.existsSync(store.stateFile)) tryJson(store.stateFile, isObject);
|
|
86
|
+
tryLines(store.historyLog);
|
|
87
|
+
for (const n of list(store.proposalsDir)) {
|
|
88
|
+
const f = path.join(store.proposalsDir, n, "proposal.json");
|
|
89
|
+
if (isSafeId(n) && fs.existsSync(f)) tryJson(f, isObject);
|
|
90
|
+
}
|
|
91
|
+
if (!bad.length) return check("space files", "PASS", `${total} file(s)/record(s) parse cleanly in space "${store.space}"`);
|
|
92
|
+
const shown = bad.slice(0, 5).join(", ") + (bad.length > 5 ? `, and ${bad.length - 5} more` : "");
|
|
93
|
+
return check("space files", "FAIL", `${bad.length} unreadable of ${total} in space "${store.space}": ${shown}`, "Restore the file from a backup or delete it; nexorum skips unreadable records but cannot repair them.");
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function checkClaudeHooks(env) {
|
|
97
|
+
const dir = path.join(userHome(env), ".claude");
|
|
98
|
+
if (!fs.existsSync(dir)) return check("claude hooks", "PASS", "~/.claude not found; Claude Code not in use", "Run `nexorum init --claude-code` if you use Claude Code.");
|
|
99
|
+
const file = path.join(dir, "settings.json");
|
|
100
|
+
let settings;
|
|
101
|
+
try {
|
|
102
|
+
settings = JSON.parse(fs.readFileSync(file, "utf8"));
|
|
103
|
+
} catch (e) {
|
|
104
|
+
return e.code === "ENOENT"
|
|
105
|
+
? check("claude hooks", "WARN", `${file} does not exist`, "Run: nexorum init --claude-code")
|
|
106
|
+
: check("claude hooks", "WARN", `${file} is not valid JSON`, "Fix the JSON by hand, then run: nexorum init --claude-code");
|
|
107
|
+
}
|
|
108
|
+
const hooks = settings && typeof settings === "object" ? settings.hooks : undefined;
|
|
109
|
+
const missing = ["SessionStart", "Stop"].filter((ev) => !hasHook(hooks?.[ev], /nexorum hook/));
|
|
110
|
+
if (!missing.length) return check("claude hooks", "PASS", "SessionStart and Stop hooks call `nexorum hook`");
|
|
111
|
+
return check("claude hooks", "WARN", `missing ${missing.join(" and ")} hook(s) in ${file}`, "Run: nexorum init --claude-code");
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function checkMcp(bin, env) {
|
|
115
|
+
const name = `mcp (${bin})`;
|
|
116
|
+
if (!findBinary(bin, env)) return check(name, "PASS", `\`${bin}\` not found on PATH; skipped`);
|
|
117
|
+
// Windows needs a shell for .cmd shims; the arguments are fixed constants, so pass one command string.
|
|
118
|
+
const win = process.platform === "win32";
|
|
119
|
+
const r = win ? spawnSync(`${bin} mcp list`, { encoding: "utf8", env, shell: true, timeout: 5000 }) : spawnSync(bin, ["mcp", "list"], { encoding: "utf8", env, timeout: 5000 });
|
|
120
|
+
const hint = `Register it: ${bin} mcp add nexorum -- npx -y nexorum mcp (or run: nexorum init)`;
|
|
121
|
+
if (r.error || r.status !== 0) {
|
|
122
|
+
const why = r.error?.code === "ETIMEDOUT" ? "timed out after 5 s" : r.error ? r.error.message : `exit ${r.status}`;
|
|
123
|
+
return check(name, "WARN", `\`${bin} mcp list\` failed (${why})`, hint);
|
|
124
|
+
}
|
|
125
|
+
return /nexorum/i.test(`${r.stdout}${r.stderr}`) ? check(name, "PASS", `nexorum is registered in ${bin}`) : check(name, "WARN", `nexorum is not listed by \`${bin} mcp list\``, hint);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function checkLlm(env) {
|
|
129
|
+
const provider = (env.NEXORUM_LLM_PROVIDER || "anthropic").toLowerCase();
|
|
130
|
+
if (provider !== "anthropic" && provider !== "openai") return check("llm config", "FAIL", `NEXORUM_LLM_PROVIDER "${provider}" is not valid`, 'Use "anthropic" or "openai".');
|
|
131
|
+
const fallbackKey = provider === "anthropic" ? "ANTHROPIC_API_KEY" : "OPENAI_API_KEY";
|
|
132
|
+
const key = env.NEXORUM_LLM_API_KEY || env[fallbackKey];
|
|
133
|
+
const parts = [`provider ${provider}`, `api key ${key ? "set" : "missing"}`];
|
|
134
|
+
const problems = key ? [] : [`Set NEXORUM_LLM_API_KEY (or ${fallbackKey}).`];
|
|
135
|
+
if (env.NEXORUM_LLM_BASE_URL) parts.push("base URL set");
|
|
136
|
+
else parts.push(provider === "openai" ? `base URL default ${DEFAULT_OPENAI_BASE_URL}` : "base URL default (Anthropic API)");
|
|
137
|
+
const fallback = provider === "anthropic" ? DEFAULT_ANTHROPIC_MODEL : undefined;
|
|
138
|
+
for (const [role, varName] of [["analyzer", "NEXORUM_ANALYZER_MODEL"], ["organizer", "NEXORUM_ORGANIZER_MODEL"]]) {
|
|
139
|
+
const model = env[varName] || fallback;
|
|
140
|
+
parts.push(`${role} model ${model ?? "missing"}${!env[varName] && fallback ? " (default)" : ""}`);
|
|
141
|
+
if (!model) problems.push(`Set ${varName}.`);
|
|
142
|
+
}
|
|
143
|
+
return problems.length
|
|
144
|
+
? check("llm config", "WARN", parts.join("; "), `${problems.join(" ")} Needed only for \`nexorum dream\`.`)
|
|
145
|
+
: check("llm config", "PASS", parts.join("; "), "Not tested against the network.");
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function checkPending(store) {
|
|
149
|
+
let pending = 0;
|
|
150
|
+
let review = 0;
|
|
151
|
+
try {
|
|
152
|
+
pending = store.listTranscripts().filter((t) => t.status === "pending" && t.events.length > 0).length;
|
|
153
|
+
review = store.listProposals().filter((p) => p.status === "review").length;
|
|
154
|
+
} catch {}
|
|
155
|
+
const hint = review ? "Run: nexorum proposals, then nexorum diff / apply / reject." : pending ? "Run: nexorum dream" : "";
|
|
156
|
+
return check("pending work", "PASS", `${pending} pending transcript(s), ${review} proposal(s) awaiting review`, hint);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
export function runChecks(store, env) {
|
|
160
|
+
return [checkNode(), checkHome(env), checkSpaceFiles(store), checkClaudeHooks(env), checkMcp("claude", env), checkMcp("codex", env), checkLlm(env), checkPending(store)];
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
export function doctorCommand(store, flags, env) {
|
|
164
|
+
const results = runChecks(store, env);
|
|
165
|
+
const failed = results.some((r) => r.status === "FAIL");
|
|
166
|
+
if (flags.json) out(JSON.stringify(results, null, 2));
|
|
167
|
+
else {
|
|
168
|
+
out(`nexorum doctor (space "${store.space}")`);
|
|
169
|
+
for (const r of results) {
|
|
170
|
+
out(`${r.status.padEnd(4)} ${r.name}: ${r.detail}`);
|
|
171
|
+
if (r.hint) out(` hint: ${r.hint}`);
|
|
172
|
+
}
|
|
173
|
+
const warns = results.filter((r) => r.status === "WARN").length;
|
|
174
|
+
out(failed ? "Result: FAIL" : warns ? `Result: OK with ${warns} warning(s)` : "Result: OK");
|
|
175
|
+
}
|
|
176
|
+
return failed ? 1 : 0;
|
|
177
|
+
}
|
package/src/dream.js
ADDED
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
// `nexorum dream`: select inputs, estimate, confirm, stage the wiki, run the analyzer/organizer engine with your own
|
|
2
|
+
// key, and write a proposal for review.
|
|
3
|
+
import fs from "node:fs";
|
|
4
|
+
import readline from "node:readline";
|
|
5
|
+
import { MemoryWiki, scrubSecrets, sha256Hex, renderTranscript, renderNotes, runDream, estimateDream } from "./core/index.js";
|
|
6
|
+
import { UsageError, intFlag } from "./args.js";
|
|
7
|
+
import { applyProposal, consumeInputs, proposalChanges, runRetention } from "./proposals.js";
|
|
8
|
+
|
|
9
|
+
export const MAX_TRANSCRIPTS = 50;
|
|
10
|
+
export const MAX_TRANSCRIPT_CHARS = 6_000_000;
|
|
11
|
+
export const MAX_NOTES = 300;
|
|
12
|
+
export const DEFAULT_BUDGET = 3_000_000;
|
|
13
|
+
export const DEFAULT_ANTHROPIC_MODEL = "claude-haiku-5-5";
|
|
14
|
+
export const DEFAULT_OPENAI_BASE_URL = "http://localhost:20128/v1";
|
|
15
|
+
|
|
16
|
+
const out = (s = "") => process.stdout.write(`${s}\n`);
|
|
17
|
+
const fmt = (n) => Number(n).toLocaleString("en-US");
|
|
18
|
+
|
|
19
|
+
/** LLM settings from the environment. A missing key or model is a UsageError (exit 2). */
|
|
20
|
+
export function resolveLlmConfig(env) {
|
|
21
|
+
const provider = (env.NEXORUM_LLM_PROVIDER || "anthropic").toLowerCase();
|
|
22
|
+
if (provider !== "anthropic" && provider !== "openai") throw new UsageError(`NEXORUM_LLM_PROVIDER must be "anthropic" or "openai" (got "${provider}").`);
|
|
23
|
+
const fallbackKey = provider === "anthropic" ? "ANTHROPIC_API_KEY" : "OPENAI_API_KEY";
|
|
24
|
+
const apiKey = env.NEXORUM_LLM_API_KEY || env[fallbackKey];
|
|
25
|
+
if (!apiKey) throw new UsageError(`No API key for provider "${provider}". Set NEXORUM_LLM_API_KEY (or ${fallbackKey}).`);
|
|
26
|
+
const llm = { provider, apiKey };
|
|
27
|
+
if (env.NEXORUM_LLM_BASE_URL) llm.baseUrl = env.NEXORUM_LLM_BASE_URL;
|
|
28
|
+
else if (provider === "openai") llm.baseUrl = DEFAULT_OPENAI_BASE_URL;
|
|
29
|
+
const fallbackModel = provider === "anthropic" ? DEFAULT_ANTHROPIC_MODEL : undefined;
|
|
30
|
+
const analyzer = env.NEXORUM_ANALYZER_MODEL || fallbackModel;
|
|
31
|
+
const organizer = env.NEXORUM_ORGANIZER_MODEL || fallbackModel;
|
|
32
|
+
if (!analyzer || !organizer) {
|
|
33
|
+
throw new UsageError('Provider "openai" needs models: set NEXORUM_ANALYZER_MODEL and NEXORUM_ORGANIZER_MODEL to model names your endpoint serves.');
|
|
34
|
+
}
|
|
35
|
+
return { llm, models: { analyzer, organizer } };
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** Every document the agents write is scrubbed before it reaches the staging wiki. */
|
|
39
|
+
class ScrubbingDocs {
|
|
40
|
+
constructor(wiki) {
|
|
41
|
+
this.wiki = wiki;
|
|
42
|
+
this.redactions = 0;
|
|
43
|
+
}
|
|
44
|
+
list(opts) {
|
|
45
|
+
return this.wiki.list(opts);
|
|
46
|
+
}
|
|
47
|
+
get(p) {
|
|
48
|
+
return this.wiki.get(p);
|
|
49
|
+
}
|
|
50
|
+
write(p, content, opts) {
|
|
51
|
+
const r = scrubSecrets(content);
|
|
52
|
+
this.redactions += r.count;
|
|
53
|
+
return this.wiki.write(p, r.text, opts);
|
|
54
|
+
}
|
|
55
|
+
delete(p) {
|
|
56
|
+
return this.wiki.delete(p);
|
|
57
|
+
}
|
|
58
|
+
rename(from, to) {
|
|
59
|
+
return this.wiki.rename(from, to);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** --since: YYYY-MM-DD or a full ISO timestamp, as epoch ms; null when absent. Anything else is a UsageError. */
|
|
64
|
+
export function parseSince(raw) {
|
|
65
|
+
if (raw === undefined) return null;
|
|
66
|
+
const s = String(raw).trim();
|
|
67
|
+
const m = /^(\d{4})-(\d{2})-(\d{2})(?:[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:?\d{2})?)?$/.exec(s);
|
|
68
|
+
const ms = m ? Date.parse(m[0].replace(" ", "T")) : NaN;
|
|
69
|
+
const realDay = m ? new Date(Date.UTC(+m[1], +m[2] - 1, +m[3])).getUTCDate() === +m[3] : false;
|
|
70
|
+
if (!m || Number.isNaN(ms) || !realDay) throw new UsageError(`--since must be an ISO date such as 2026-11-01 or 2026-11-01T08:00:00Z (got "${s}").`);
|
|
71
|
+
return ms;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const pendingOf = (store, since) =>
|
|
75
|
+
store.listTranscripts().filter((t) => t.status === "pending" && t.events.length > 0 && (since === null || Date.parse(t.updated_at) >= since));
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Why a scheduled dream should not run now, or null when it is due: enough pending transcripts, the newest
|
|
79
|
+
* transcript idle long enough, and the newest proposal older than the cooldown.
|
|
80
|
+
*/
|
|
81
|
+
export function notDueReason(store, { minPending, idleMinutes, cooldownHours }, nowMs, since = null) {
|
|
82
|
+
const pending = pendingOf(store, since);
|
|
83
|
+
if (pending.length < minPending) return `${pending.length} pending transcript(s), need at least ${minPending}`;
|
|
84
|
+
const newest = Math.max(...store.listTranscripts().map((t) => Date.parse(t.updated_at)).filter((n) => !Number.isNaN(n)));
|
|
85
|
+
if (Number.isFinite(newest)) {
|
|
86
|
+
const idle = (nowMs - newest) / 60_000;
|
|
87
|
+
if (idle < idleMinutes) return `newest transcript changed ${Math.floor(idle)} minute(s) ago, wait until it is ${idleMinutes} minute(s) idle`;
|
|
88
|
+
}
|
|
89
|
+
const last = Math.max(...store.listProposals().map((p) => Date.parse(p.created_at)).filter((n) => !Number.isNaN(n)));
|
|
90
|
+
if (Number.isFinite(last)) {
|
|
91
|
+
const hours = (nowMs - last) / 3_600_000;
|
|
92
|
+
if (hours < cooldownHours) return `last dream was ${hours.toFixed(1)} hour(s) ago, cooldown is ${cooldownHours} hour(s)`;
|
|
93
|
+
}
|
|
94
|
+
return null;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** Oldest pending transcripts first (at most maxInputs and 6,000,000 characters), plus up to 300 unconsolidated notes. */
|
|
98
|
+
export function selectInputs(store, { since = null, maxInputs = MAX_TRANSCRIPTS } = {}) {
|
|
99
|
+
const pending = pendingOf(store, since).sort((a, b) => (a.created_at < b.created_at ? -1 : a.created_at > b.created_at ? 1 : a.id < b.id ? -1 : 1));
|
|
100
|
+
const transcripts = [];
|
|
101
|
+
let chars = 0;
|
|
102
|
+
for (const t of pending) {
|
|
103
|
+
if (transcripts.length >= maxInputs) break;
|
|
104
|
+
const text = renderTranscript({ id: t.id, source: t.source, events: t.events });
|
|
105
|
+
if (transcripts.length > 0 && chars + text.length > MAX_TRANSCRIPT_CHARS) break;
|
|
106
|
+
chars += text.length;
|
|
107
|
+
transcripts.push({ id: t.id, source: t.source, text, updated_at: t.updated_at });
|
|
108
|
+
}
|
|
109
|
+
const through = store.readState().notes_consolidated_through;
|
|
110
|
+
const notes = store
|
|
111
|
+
.readNotes()
|
|
112
|
+
.filter((n) => !through || n.ts > through)
|
|
113
|
+
.sort((a, b) => (a.ts < b.ts ? -1 : a.ts > b.ts ? 1 : 0))
|
|
114
|
+
.slice(0, MAX_NOTES);
|
|
115
|
+
return { transcripts, notes };
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
async function confirm(question) {
|
|
119
|
+
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
|
|
120
|
+
try {
|
|
121
|
+
const answer = await new Promise((resolve) => rl.question(question, resolve));
|
|
122
|
+
return /^y(es)?$/i.test(String(answer).trim());
|
|
123
|
+
} finally {
|
|
124
|
+
rl.close();
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function progress() {
|
|
129
|
+
let usage = null;
|
|
130
|
+
const print = (e) => {
|
|
131
|
+
if (!e || typeof e !== "object") return;
|
|
132
|
+
switch (e.type) {
|
|
133
|
+
case "phase":
|
|
134
|
+
return out(`* ${e.message}`);
|
|
135
|
+
case "analyzer_started":
|
|
136
|
+
case "analyzer_done":
|
|
137
|
+
case "organizer_started":
|
|
138
|
+
case "organizer_done":
|
|
139
|
+
return out(`* [${e.agent}] ${e.message ?? e.type}`);
|
|
140
|
+
case "agent_text":
|
|
141
|
+
return out(` [${e.agent}] ${String(e.message ?? "").replace(/\s+/g, " ").slice(0, 160)}`);
|
|
142
|
+
case "tool": {
|
|
143
|
+
const why = !e.ok && e.message ? `: ${String(e.message).slice(0, 120)}` : "";
|
|
144
|
+
return out(` [${e.agent}] ${e.tool}${e.path ? ` ${e.path}` : ""} ${e.ok ? "ok" : "FAILED"}${why}`);
|
|
145
|
+
}
|
|
146
|
+
case "usage":
|
|
147
|
+
usage = e.usage;
|
|
148
|
+
return;
|
|
149
|
+
case "budget":
|
|
150
|
+
return out(`! ${e.message}`);
|
|
151
|
+
case "error":
|
|
152
|
+
return out(`! error: ${e.message}`);
|
|
153
|
+
default:
|
|
154
|
+
}
|
|
155
|
+
};
|
|
156
|
+
return { print, usage: () => usage };
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
export async function dreamCommand(store, flags, env) {
|
|
160
|
+
const budget = intFlag(flags, "budget", { fallback: DEFAULT_BUDGET });
|
|
161
|
+
const instructions = typeof flags.instructions === "string" && flags.instructions.trim() ? flags.instructions.trim() : null;
|
|
162
|
+
const since = parseSince(flags.since);
|
|
163
|
+
const maxInputs = intFlag(flags, "max-inputs", { min: 1, max: 200, fallback: MAX_TRANSCRIPTS });
|
|
164
|
+
if (flags["if-due"]) {
|
|
165
|
+
const reason = notDueReason(
|
|
166
|
+
store,
|
|
167
|
+
{
|
|
168
|
+
minPending: intFlag(flags, "min-pending", { min: 1, fallback: 3 }),
|
|
169
|
+
idleMinutes: intFlag(flags, "idle-minutes", { min: 0, fallback: 10 }),
|
|
170
|
+
cooldownHours: intFlag(flags, "cooldown-hours", { min: 0, fallback: 6 }),
|
|
171
|
+
},
|
|
172
|
+
Date.now(),
|
|
173
|
+
since,
|
|
174
|
+
);
|
|
175
|
+
if (reason) {
|
|
176
|
+
out(`not due: ${reason}`);
|
|
177
|
+
return 0;
|
|
178
|
+
}
|
|
179
|
+
} else {
|
|
180
|
+
for (const f of ["min-pending", "idle-minutes", "cooldown-hours"]) {
|
|
181
|
+
if (flags[f] !== undefined) throw new UsageError(`--${f} only applies together with --if-due.`);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
const { transcripts, notes } = selectInputs(store, { since, maxInputs });
|
|
186
|
+
const inputs = transcripts.map((t) => ({ id: t.id, source: t.source, text: t.text }));
|
|
187
|
+
if (notes.length) inputs.push({ id: "notes", source: "notes", text: renderNotes(notes) });
|
|
188
|
+
out(`Space: ${store.space}`);
|
|
189
|
+
out(`Pending transcripts: ${transcripts.length}`);
|
|
190
|
+
out(`Unconsolidated notes: ${notes.length}`);
|
|
191
|
+
if (!inputs.length) {
|
|
192
|
+
out('Nothing to dream about yet. Log a session or run `nexorum remember "..."` first.');
|
|
193
|
+
return 0;
|
|
194
|
+
}
|
|
195
|
+
const live = store.wikiSnapshot();
|
|
196
|
+
const docsBytes = Object.values(live).reduce((n, c) => n + Buffer.byteLength(c), 0);
|
|
197
|
+
const est = estimateDream({ transcripts: inputs.map((i) => ({ chars: i.text.length })), existingDocsBytes: docsBytes });
|
|
198
|
+
out(`Input size: ${fmt(est.chars)} characters`);
|
|
199
|
+
out(`Rough estimate: ~${fmt(est.est_total_tokens)} tokens (budget ${fmt(budget)}). This is billed by your LLM provider.`);
|
|
200
|
+
for (const n of est.notes) out(` ${n}`);
|
|
201
|
+
if (flags["dry-run"]) {
|
|
202
|
+
out("Dry run: nothing was sent.");
|
|
203
|
+
return 0;
|
|
204
|
+
}
|
|
205
|
+
if (est.est_total_tokens > budget) {
|
|
206
|
+
throw new UsageError(
|
|
207
|
+
`The estimate (${fmt(est.est_total_tokens)} tokens) is above the budget (${fmt(budget)}). ` +
|
|
208
|
+
"Dream less: pass --since DATE (only newer transcripts) or --max-inputs N (fewer transcripts), or allow more with a larger --budget TOKENS.",
|
|
209
|
+
);
|
|
210
|
+
}
|
|
211
|
+
const config = resolveLlmConfig(env);
|
|
212
|
+
if (!flags.yes) {
|
|
213
|
+
if (!process.stdin.isTTY) throw new UsageError("Not a terminal: pass --yes to confirm that this dream may call your LLM provider.");
|
|
214
|
+
if (!(await confirm("Proceed? [y/N] "))) {
|
|
215
|
+
out("Canceled.");
|
|
216
|
+
return 0;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// Stage: the live wiki is copied into the proposal; the agents work on that copy only.
|
|
221
|
+
const id = store.newProposalId();
|
|
222
|
+
store.writeBase(id, Object.fromEntries(Object.entries(live).map(([p, c]) => [p, sha256Hex(c)])));
|
|
223
|
+
store.writeProposalWiki(id, live);
|
|
224
|
+
const staging = new MemoryWiki(store.readProposalWiki(id));
|
|
225
|
+
const docs = new ScrubbingDocs(staging);
|
|
226
|
+
const createdAt = new Date().toISOString();
|
|
227
|
+
const ac = new AbortController();
|
|
228
|
+
const onSigint = () => ac.abort();
|
|
229
|
+
process.once("SIGINT", onSigint);
|
|
230
|
+
const prog = progress();
|
|
231
|
+
out(`Dreaming (proposal ${id}) ...`);
|
|
232
|
+
let result;
|
|
233
|
+
try {
|
|
234
|
+
result = await runDream({ llm: config.llm, models: config.models, docs, transcripts: inputs, instructions, budgetTokens: budget, signal: ac.signal, onEvent: prog.print });
|
|
235
|
+
} catch (e) {
|
|
236
|
+
result = { status: "failed", summary: "", usage: prog.usage(), analyzed: [], organizer: { ok: false, changes: [] }, error: e?.message ?? String(e) };
|
|
237
|
+
} finally {
|
|
238
|
+
process.removeListener("SIGINT", onSigint);
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
store.writeProposalWiki(id, staging.snapshot());
|
|
242
|
+
const proposal = {
|
|
243
|
+
id,
|
|
244
|
+
status: "review",
|
|
245
|
+
created_at: createdAt,
|
|
246
|
+
dream_status: result.status,
|
|
247
|
+
partial: false,
|
|
248
|
+
summary: result.summary ?? "",
|
|
249
|
+
error: result.error ?? null,
|
|
250
|
+
usage: result.usage ?? null,
|
|
251
|
+
analyzed: result.analyzed ?? [],
|
|
252
|
+
organizer: result.organizer ?? null,
|
|
253
|
+
models: config.models,
|
|
254
|
+
provider: config.llm.provider,
|
|
255
|
+
transcripts: transcripts.map((t) => t.id),
|
|
256
|
+
transcript_versions: Object.fromEntries(transcripts.map((t) => [t.id, t.updated_at])),
|
|
257
|
+
notes_through: notes.length ? notes.at(-1).ts : null,
|
|
258
|
+
// The dream tools scrub before they write, so most redactions are counted by the engine, the rest here.
|
|
259
|
+
redactions: docs.redactions + (Number.isFinite(result.redactions) ? result.redactions : 0),
|
|
260
|
+
};
|
|
261
|
+
const changes = proposalChanges(store, id);
|
|
262
|
+
const complete = result.status === "completed" && result.organizer?.ok !== false && (result.analyzed ?? []).every((a) => a.ok !== false);
|
|
263
|
+
proposal.partial = !complete && changes.length > 0;
|
|
264
|
+
|
|
265
|
+
if (result.usage) {
|
|
266
|
+
const u = result.usage;
|
|
267
|
+
out(`Tokens used: ${fmt(u.total_tokens ?? 0)} (input ${u.input_tokens ?? 0}, output ${u.output_tokens ?? 0}, cache read ${u.cache_read_input_tokens ?? 0}, cache write ${u.cache_creation_input_tokens ?? 0})`);
|
|
268
|
+
}
|
|
269
|
+
if (result.summary) out(`Summary: ${String(result.summary).trim()}`);
|
|
270
|
+
for (const c of result.organizer?.changes ?? []) out(` - ${c}`);
|
|
271
|
+
|
|
272
|
+
if (!changes.length) {
|
|
273
|
+
if (complete) {
|
|
274
|
+
Object.assign(proposal, { status: "applied", no_changes: true, applied_at: new Date().toISOString() });
|
|
275
|
+
store.writeProposal(proposal);
|
|
276
|
+
consumeInputs(store, proposal);
|
|
277
|
+
out("The dream made no changes to the wiki; inputs are marked as dreamed.");
|
|
278
|
+
runRetention(store);
|
|
279
|
+
return 0;
|
|
280
|
+
}
|
|
281
|
+
fs.rmSync(store.proposalDir(id), { recursive: true, force: true });
|
|
282
|
+
out(`The dream ended with status "${result.status}" and wrote nothing${result.error ? `: ${result.error}` : ""}. Inputs stay pending.`);
|
|
283
|
+
return 1;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
store.writeProposal(proposal);
|
|
287
|
+
const counts = { added: 0, modified: 0, removed: 0 };
|
|
288
|
+
for (const c of changes) counts[c.kind]++;
|
|
289
|
+
out(`Proposal ${id} (${proposal.partial ? "review, PARTIAL" : "review"}): ${counts.added} added, ${counts.modified} modified, ${counts.removed} removed`);
|
|
290
|
+
for (const c of changes) out(` ${c.kind.padEnd(8)} ${c.path}`);
|
|
291
|
+
if (proposal.redactions) out(` (${proposal.redactions} secret-like value(s) were redacted from the written documents)`);
|
|
292
|
+
if (proposal.partial) {
|
|
293
|
+
out(`WARNING: the dream did not finish cleanly (status ${result.status}${result.error ? `: ${result.error}` : ""}). Review the diff carefully; some inputs may not be covered and the index may be incomplete.`);
|
|
294
|
+
}
|
|
295
|
+
if (flags.apply && !proposal.partial) {
|
|
296
|
+
const r = applyProposal(store, proposal);
|
|
297
|
+
out(`Applied: ${r.applied.length} path(s) written${r.skipped.length ? `, ${r.skipped.length} skipped` : ""}.`);
|
|
298
|
+
for (const s of r.skipped) out(` skipped ${s.path}: ${s.reason}`);
|
|
299
|
+
runRetention(store);
|
|
300
|
+
return 0;
|
|
301
|
+
}
|
|
302
|
+
if (flags.apply) out("--apply was not honoured because the run was partial.");
|
|
303
|
+
out("Next steps:");
|
|
304
|
+
out(` nexorum diff ${id} show what would change`);
|
|
305
|
+
out(` nexorum apply ${id} write it to the live wiki`);
|
|
306
|
+
out(` nexorum reject ${id} discard it`);
|
|
307
|
+
runRetention(store);
|
|
308
|
+
return complete ? 0 : 1;
|
|
309
|
+
}
|