@gamaze/hicortex 0.18.2 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +60 -0
- package/assets/dashboard.html +103 -7
- package/assets/identity.html +48 -1
- package/assets/viz.html +52 -1
- package/dist/backup.d.ts +107 -0
- package/dist/backup.js +343 -0
- package/dist/capture.d.ts +20 -0
- package/dist/capture.js +9 -1
- package/dist/cli.js +30 -0
- package/dist/config-read.d.ts +28 -2
- package/dist/config-read.js +32 -2
- package/dist/consolidate.js +2 -4
- package/dist/dashboard.d.ts +71 -6
- package/dist/dashboard.js +61 -7
- package/dist/distiller.d.ts +4 -2
- package/dist/distiller.js +13 -5
- package/dist/hosted-boot.d.ts +61 -0
- package/dist/hosted-boot.js +72 -0
- package/dist/init.d.ts +15 -0
- package/dist/init.js +99 -8
- package/dist/localhost-bypass.d.ts +27 -0
- package/dist/localhost-bypass.js +71 -0
- package/dist/mcp-server.d.ts +18 -0
- package/dist/mcp-server.js +147 -4
- package/dist/memory-instructions.js +1 -1
- package/dist/nightly.js +104 -2
- package/dist/prompts.js +19 -4
- package/dist/telemetry.d.ts +14 -0
- package/dist/token-budget.d.ts +34 -0
- package/dist/token-budget.js +139 -0
- package/dist/type-classify.js +7 -4
- package/dist/types.d.ts +49 -0
- package/dist/viz.d.ts +9 -1
- package/dist/viz.js +11 -2
- package/package.json +6 -4
package/dist/backup.js
ADDED
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Backup mechanism — a transactionally-consistent snapshot of the irreplaceable
|
|
4
|
+
* data (#6, Phase 0B).
|
|
5
|
+
*
|
|
6
|
+
* For a product whose value is accumulated memory, silent backup loss is
|
|
7
|
+
* existential (spec §8). The live DB runs with WAL on, so a plain `cp`/`tar` of
|
|
8
|
+
* `hicortex.db` is torn (the `-wal` file holds uncommitted pages → a copy that
|
|
9
|
+
* looks fine until you restore it). This module uses better-sqlite3's native
|
|
10
|
+
* `db.backup(path)` — the SQLite online-backup API, already proven in `dedup.ts`
|
|
11
|
+
* — which folds the WAL into a single self-contained, consistent snapshot, then
|
|
12
|
+
* packages it with the hand-edited identity layer + capture state into one
|
|
13
|
+
* `tar.gz` artifact the operator ships offsite.
|
|
14
|
+
*
|
|
15
|
+
* Backup set (the irreplaceable data under HICORTEX_HOME):
|
|
16
|
+
* - `hicortex.db` — via db.backup() to a temp snapshot (WAL-safe)
|
|
17
|
+
* - `identity/` (whole tree) — global `*.md` AND per-agent `agents/<id>/*.md`
|
|
18
|
+
* - `context/` (legacy) — additive fallback (identity-store.ts migrate)
|
|
19
|
+
* - `state.json` — cursors/tier/timestamps (state.ts)
|
|
20
|
+
* - `capture-cursors.json` — per-session capture positions (capture-cursors.ts)
|
|
21
|
+
*
|
|
22
|
+
* Excluded by design:
|
|
23
|
+
* - `config.json` — secrets (authToken/llmApiKey); re-creatable via init
|
|
24
|
+
* - `backups/` — output dir (never back up the backups)
|
|
25
|
+
* - logs (`nightly.log`, `server*.log`) — re-creatable, noisy, not data
|
|
26
|
+
* - `models/` — embedder cache (re-downloadable)
|
|
27
|
+
* - `capture.lock` — a transient single-flight lock, not state
|
|
28
|
+
* - `.allow-localhost-bypass` — a hosted fail-closed marker, not data
|
|
29
|
+
*
|
|
30
|
+
* Vectors: `db.backup()` copies the vec0 shadow tables (they're real tables).
|
|
31
|
+
* If a restore ever finds them missing, vectors regenerate from
|
|
32
|
+
* `memories.content` via the embedder — non-destructive either way.
|
|
33
|
+
*/
|
|
34
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
35
|
+
exports.createBackup = createBackup;
|
|
36
|
+
exports.runBackupHook = runBackupHook;
|
|
37
|
+
exports.runBackupCli = runBackupCli;
|
|
38
|
+
const node_child_process_1 = require("node:child_process");
|
|
39
|
+
const node_crypto_1 = require("node:crypto");
|
|
40
|
+
const node_fs_1 = require("node:fs");
|
|
41
|
+
const node_os_1 = require("node:os");
|
|
42
|
+
const node_path_1 = require("node:path");
|
|
43
|
+
const node_zlib_1 = require("node:zlib");
|
|
44
|
+
const tar_stream_1 = require("tar-stream");
|
|
45
|
+
const paths_js_1 = require("./paths.js");
|
|
46
|
+
const db_js_1 = require("./db.js");
|
|
47
|
+
const init_js_1 = require("./init.js");
|
|
48
|
+
/**
|
|
49
|
+
* Hook timeout — a stuck offsite upload (rclone/aws/B2 hung on a dead network)
|
|
50
|
+
* must NOT hang the nightly. 5 min is generous for typical tarball uploads; the
|
|
51
|
+
* operator's wrapper can re-queue on timeout. Mirrors the capture-lock "don't
|
|
52
|
+
* block forever" discipline.
|
|
53
|
+
*/
|
|
54
|
+
const HOOK_TIMEOUT_MS = 5 * 60 * 1000;
|
|
55
|
+
/**
|
|
56
|
+
* Create a backup artifact. Snapshots the live DB via the online-backup API,
|
|
57
|
+
* packages it with the identity tree + state into a single `tar.gz`, and writes
|
|
58
|
+
* it to `outFile` (default `<home>/backups/hicortex-<ISO>.tar.gz`) or streams
|
|
59
|
+
* to stdout. Returns `{ path?, bytes, files }`.
|
|
60
|
+
*
|
|
61
|
+
* Never partially writes a file artifact on failure: if the tar pipeline errors
|
|
62
|
+
* mid-stream the (likely-truncated) file is removed before the error propagates,
|
|
63
|
+
* so a stale half-backup is never left on disk to masquerade as a good one.
|
|
64
|
+
*/
|
|
65
|
+
async function createBackup(opts) {
|
|
66
|
+
if (opts.stdout && (opts.outFile || opts.outDir)) {
|
|
67
|
+
throw new Error("[hicortex] backup: --stdout is mutually exclusive with --out / backupDir");
|
|
68
|
+
}
|
|
69
|
+
// 1. Snapshot the live DB via the WAL-safe online-backup API (proven at
|
|
70
|
+
// dedup.ts:499). Runs concurrently with the live DB — no writer blocking,
|
|
71
|
+
// no torn copy. The temp file is cleaned up in `finally` below.
|
|
72
|
+
const snapshotPath = (0, node_path_1.join)((0, node_os_1.tmpdir)(), `hicortex-backup-${(0, node_crypto_1.randomBytes)(6).toString("hex")}.db`);
|
|
73
|
+
await opts.db.backup(snapshotPath);
|
|
74
|
+
let totalBytes = 0;
|
|
75
|
+
let fileCount = 0;
|
|
76
|
+
// Resolve the output target (file path or stdout). The ISO stamp is filename-
|
|
77
|
+
// safe (colons/dots → dashes) so it survives every filesystem.
|
|
78
|
+
const iso = new Date().toISOString().replace(/[:.]/g, "-");
|
|
79
|
+
const outFile = opts.stdout
|
|
80
|
+
? undefined
|
|
81
|
+
: opts.outFile ?? (0, node_path_1.join)(opts.outDir ?? (0, node_path_1.join)(opts.home, "backups"), `hicortex-${iso}.tar.gz`);
|
|
82
|
+
// Prepare the output dir up front so a later mkdirSync failure (permissions,
|
|
83
|
+
// read-only mount) surfaces BEFORE we spend time tarring — fail loud, early.
|
|
84
|
+
if (outFile)
|
|
85
|
+
(0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(outFile), { recursive: true });
|
|
86
|
+
// 2. Build the entry list (deterministic order for diffable tarballs).
|
|
87
|
+
const entries = [{ name: "hicortex.db", file: snapshotPath }];
|
|
88
|
+
appendTree(entries, (0, node_path_1.join)(opts.home, "identity"), "identity");
|
|
89
|
+
appendTree(entries, (0, node_path_1.join)(opts.home, "context"), "context");
|
|
90
|
+
pushIfExists(entries, (0, node_path_1.join)(opts.home, "state.json"), "state.json");
|
|
91
|
+
pushIfExists(entries, (0, node_path_1.join)(opts.home, "capture-cursors.json"), "capture-cursors.json");
|
|
92
|
+
// 3. Pack → gzip → sink (file or stdout). Entries are added SEQUENTIALLY:
|
|
93
|
+
// tar-stream writes headers in call order, and concurrent entry() calls
|
|
94
|
+
// would interleave headers into a corrupt tar. Each entry streams from disk
|
|
95
|
+
// so the DB snapshot (potentially large) is never fully in memory.
|
|
96
|
+
const packStream = (0, tar_stream_1.pack)();
|
|
97
|
+
const gzip = (0, node_zlib_1.createGzip)();
|
|
98
|
+
packStream.pipe(gzip);
|
|
99
|
+
const sink = opts.stdout ? process.stdout : (0, node_fs_1.createWriteStream)(outFile);
|
|
100
|
+
gzip.pipe(sink);
|
|
101
|
+
gzip.on("data", (chunk) => {
|
|
102
|
+
totalBytes += chunk.length;
|
|
103
|
+
});
|
|
104
|
+
// Completion signal — DIFFERENT per sink type:
|
|
105
|
+
// - File: await the file stream's 'finish' (all bytes flushed to disk) so
|
|
106
|
+
// the offsite hook reads a complete, closed file.
|
|
107
|
+
// - stdout: process.stdout is a special non-closable stream that NEVER
|
|
108
|
+
// emits 'finish', so awaiting it deadlocks. Instead await gzip's 'end'
|
|
109
|
+
// (the last compressed byte has been emitted and handed to stdout); the
|
|
110
|
+
// OS drains process.stdout asynchronously and Node won't exit while it
|
|
111
|
+
// has pending buffered bytes.
|
|
112
|
+
const sinkFinished = opts.stdout
|
|
113
|
+
? new Promise((resolve, reject) => {
|
|
114
|
+
gzip.on("end", resolve);
|
|
115
|
+
gzip.on("error", reject);
|
|
116
|
+
packStream.on("error", reject);
|
|
117
|
+
// A downstream pipe closing early (--stdout | head, or rclone dying)
|
|
118
|
+
// emits 'error' (EPIPE) on process.stdout — listen so it rejects cleanly
|
|
119
|
+
// instead of becoming an uncaughtException (and so we never hang waiting
|
|
120
|
+
// for gzip 'end' that will never come once the sink has errored).
|
|
121
|
+
sink.on("error", reject);
|
|
122
|
+
})
|
|
123
|
+
: new Promise((resolve, reject) => {
|
|
124
|
+
sink.on("finish", resolve);
|
|
125
|
+
sink.on("error", reject);
|
|
126
|
+
gzip.on("error", reject);
|
|
127
|
+
packStream.on("error", reject);
|
|
128
|
+
});
|
|
129
|
+
try {
|
|
130
|
+
for (const entry of entries) {
|
|
131
|
+
const size = (0, node_fs_1.statSync)(entry.file).size;
|
|
132
|
+
await new Promise((resolve, reject) => {
|
|
133
|
+
const writable = packStream.entry({ name: entry.name, size }, (err) => {
|
|
134
|
+
if (err)
|
|
135
|
+
reject(err);
|
|
136
|
+
else
|
|
137
|
+
resolve();
|
|
138
|
+
});
|
|
139
|
+
const rs = (0, node_fs_1.createReadStream)(entry.file);
|
|
140
|
+
rs.on("error", reject);
|
|
141
|
+
rs.pipe(writable);
|
|
142
|
+
});
|
|
143
|
+
fileCount++;
|
|
144
|
+
}
|
|
145
|
+
packStream.finalize();
|
|
146
|
+
await sinkFinished;
|
|
147
|
+
return { path: outFile, bytes: totalBytes, files: fileCount };
|
|
148
|
+
}
|
|
149
|
+
catch (err) {
|
|
150
|
+
// Never leave a truncated/half-written artifact on disk masquerading as a
|
|
151
|
+
// good backup — a restore drill against it would fail at the worst time.
|
|
152
|
+
if (!opts.stdout && outFile) {
|
|
153
|
+
try {
|
|
154
|
+
(0, node_fs_1.rmSync)(outFile, { force: true });
|
|
155
|
+
}
|
|
156
|
+
catch {
|
|
157
|
+
// Best-effort cleanup; the real error is the one we re-throw.
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
throw err;
|
|
161
|
+
}
|
|
162
|
+
finally {
|
|
163
|
+
// Always remove the temp DB snapshot (best-effort, never masks a real error).
|
|
164
|
+
try {
|
|
165
|
+
(0, node_fs_1.rmSync)(snapshotPath, { force: true });
|
|
166
|
+
}
|
|
167
|
+
catch {
|
|
168
|
+
// ignore
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* Run the operator's post-backup offsite hook. The configured `command` is split
|
|
174
|
+
* on whitespace and the artifact path appended as the LAST arg (e.g.
|
|
175
|
+
* `rclone copyto <path> remote:hicortex/` → `["rclone","copyto",path,...]`).
|
|
176
|
+
* Cloud creds + active alerting stay in the operator's wrapper, out of the
|
|
177
|
+
* product. NEVER throws — a hook failure is reported as `{ ok:false }` so the
|
|
178
|
+
* nightly continues (capture/consolidation already succeeded; the backup itself
|
|
179
|
+
* is on disk). 5 min timeout so a hung upload can't hang the nightly.
|
|
180
|
+
*
|
|
181
|
+
* Whitespace-split caveat: commands with quoted args containing spaces should
|
|
182
|
+
* be a wrapper script (`/etc/hicortex/offsite.sh`), not a one-liner — the split
|
|
183
|
+
* here is deliberately dumb to avoid re-implementing a shell parser.
|
|
184
|
+
*/
|
|
185
|
+
async function runBackupHook(artifactPath, command) {
|
|
186
|
+
// Missing/empty command = nothing configured. Not an error (the nightly only
|
|
187
|
+
// calls this when a command IS set), but the defensive contract is { ok:false }
|
|
188
|
+
// so a caller that forgets the guard still gets an honest "no hook ran".
|
|
189
|
+
if (!command || !command.trim()) {
|
|
190
|
+
return { ok: false };
|
|
191
|
+
}
|
|
192
|
+
const parts = command.trim().split(/\s+/);
|
|
193
|
+
const [cmd, ...baseArgs] = parts;
|
|
194
|
+
const args = [...baseArgs, artifactPath];
|
|
195
|
+
return new Promise((resolve) => {
|
|
196
|
+
try {
|
|
197
|
+
(0, node_child_process_1.execFile)(cmd, args, { timeout: HOOK_TIMEOUT_MS, maxBuffer: 1024 * 1024 }, (err, stdout, stderr) => {
|
|
198
|
+
if (err) {
|
|
199
|
+
const execErr = err;
|
|
200
|
+
// err.code: number = process exit code; "ENOENT" = binary not found.
|
|
201
|
+
// err.signal: set when killed (timeout → SIGTERM). Extract a numeric
|
|
202
|
+
// exit code only (the alerting signal); spawn/timeout failures are
|
|
203
|
+
// reported with undefined exitCode.
|
|
204
|
+
const exitCode = typeof execErr.code === "number" ? execErr.code : undefined;
|
|
205
|
+
const reason = execErr.signal != null
|
|
206
|
+
? `killed by signal ${execErr.signal} (timeout?)`
|
|
207
|
+
: execErr.code === "ENOENT"
|
|
208
|
+
? `command not found: ${cmd}`
|
|
209
|
+
: execErr.message;
|
|
210
|
+
console.error(`[hicortex] backup hook failed: ${reason}`);
|
|
211
|
+
if (stderr) {
|
|
212
|
+
console.error(`[hicortex] hook stderr: ${stderr.toString().trim().slice(0, 2000)}`);
|
|
213
|
+
}
|
|
214
|
+
resolve({ ok: false, exitCode });
|
|
215
|
+
return;
|
|
216
|
+
}
|
|
217
|
+
if (stdout) {
|
|
218
|
+
console.log(`[hicortex] backup hook stdout: ${stdout.toString().trim().slice(0, 2000)}`);
|
|
219
|
+
}
|
|
220
|
+
resolve({ ok: true, exitCode: 0 });
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
catch (err) {
|
|
224
|
+
// Synchronous spawn failure (shouldn't happen with execFile, but defensive
|
|
225
|
+
// — the contract is "never throws").
|
|
226
|
+
console.error(`[hicortex] backup hook could not run: ${err instanceof Error ? err.message : String(err)}`);
|
|
227
|
+
resolve({ ok: false });
|
|
228
|
+
}
|
|
229
|
+
});
|
|
230
|
+
}
|
|
231
|
+
// ---------------------------------------------------------------------------
|
|
232
|
+
// File-set helpers
|
|
233
|
+
// ---------------------------------------------------------------------------
|
|
234
|
+
/**
|
|
235
|
+
* Recursively append every regular file under `absDir` to `entries`, with
|
|
236
|
+
* in-tar paths rooted at `relRoot`. Symlinks are SKIPPED (security — a symlink
|
|
237
|
+
* in identity/ could escape home; the identity store already lstat-skips them
|
|
238
|
+
* on read, and we do the same on backup so a restored tree can't point outside
|
|
239
|
+
* itself). Missing dir = no-op (fresh install may have no identity/ yet).
|
|
240
|
+
*/
|
|
241
|
+
function appendTree(entries, absDir, relRoot) {
|
|
242
|
+
if (!(0, node_fs_1.existsSync)(absDir))
|
|
243
|
+
return;
|
|
244
|
+
const walk = (dir, rel) => {
|
|
245
|
+
let ents;
|
|
246
|
+
try {
|
|
247
|
+
ents = (0, node_fs_1.readdirSync)(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name));
|
|
248
|
+
}
|
|
249
|
+
catch {
|
|
250
|
+
// EACCES / transient — skip this subtree rather than aborting the backup.
|
|
251
|
+
return;
|
|
252
|
+
}
|
|
253
|
+
for (const ent of ents) {
|
|
254
|
+
// withFileTypes uses the entry's own type (not the target's), so a symlink
|
|
255
|
+
// is correctly identified without an extra lstat — skip symlinks so a
|
|
256
|
+
// link in identity/ can't escape home into the tarball.
|
|
257
|
+
const abs = (0, node_path_1.join)(dir, ent.name);
|
|
258
|
+
const relPath = (0, node_path_1.join)(rel, ent.name);
|
|
259
|
+
if (ent.isSymbolicLink())
|
|
260
|
+
continue;
|
|
261
|
+
if (ent.isDirectory()) {
|
|
262
|
+
walk(abs, relPath);
|
|
263
|
+
}
|
|
264
|
+
else if (ent.isFile()) {
|
|
265
|
+
// Normalize to forward slashes for tar (posix convention), even though
|
|
266
|
+
// join already produces them on mac/linux.
|
|
267
|
+
entries.push({ name: relPath.split("\\").join("/"), file: abs });
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
};
|
|
271
|
+
walk(absDir, relRoot);
|
|
272
|
+
}
|
|
273
|
+
// (Extraction of a backup artifact is intentionally NOT in this module: the
|
|
274
|
+
// restore path is manual for Phase 0 (spec §8) and tested directly via
|
|
275
|
+
// tar-stream's extract() in tests/backup.test.ts. A future `hicortex restore`
|
|
276
|
+
// command would grow its own module; packaging + extraction don't belong
|
|
277
|
+
// together — one writes artifacts, the other consumes them.)
|
|
278
|
+
/**
|
|
279
|
+
* Push a single regular file onto `entries` if it exists. Skips symlinks and
|
|
280
|
+
* non-files (the state files are plain files; a symlink here would be suspect).
|
|
281
|
+
*/
|
|
282
|
+
function pushIfExists(entries, abs, name) {
|
|
283
|
+
try {
|
|
284
|
+
const st = (0, node_fs_1.lstatSync)(abs);
|
|
285
|
+
if (st.isFile())
|
|
286
|
+
entries.push({ name, file: abs });
|
|
287
|
+
}
|
|
288
|
+
catch {
|
|
289
|
+
// ENOENT or unreadable — skip. A fresh install may lack capture-cursors.json.
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
/**
|
|
293
|
+
* Run the `hicortex backup` CLI command. Loads config (for `backupDir` /
|
|
294
|
+
* `backupCommand`), opens the DB, writes (or streams) the artifact, runs the
|
|
295
|
+
* offsite hook when configured, prints the artifact path, and exits non-zero on
|
|
296
|
+
* any failure (a failed backup must be visible — `cron`/launchd surfaces it).
|
|
297
|
+
*/
|
|
298
|
+
async function runBackupCli(opts) {
|
|
299
|
+
const home = (0, paths_js_1.hicortexHome)();
|
|
300
|
+
const { config } = (0, init_js_1.loadConfigStrict)((0, node_path_1.join)(home, "config.json"));
|
|
301
|
+
const backupDir = typeof config.backupDir === "string" && config.backupDir.trim()
|
|
302
|
+
? config.backupDir
|
|
303
|
+
: undefined;
|
|
304
|
+
const backupCommand = typeof config.backupCommand === "string" && config.backupCommand.trim()
|
|
305
|
+
? config.backupCommand
|
|
306
|
+
: undefined;
|
|
307
|
+
const db = (0, db_js_1.initDb)((0, db_js_1.resolveDbPath)());
|
|
308
|
+
try {
|
|
309
|
+
const result = await createBackup({
|
|
310
|
+
db,
|
|
311
|
+
home,
|
|
312
|
+
// CLI `--out <dir>` takes precedence over the config `backupDir`. When
|
|
313
|
+
// --stdout is set, don't forward backupDir at all — a configured backupDir
|
|
314
|
+
// (for the nightly) must not block a manual --stdout stream (the mutual-
|
|
315
|
+
// exclusivity check would otherwise reject it).
|
|
316
|
+
outDir: opts.stdout ? undefined : (opts.outDir ?? backupDir),
|
|
317
|
+
stdout: opts.stdout,
|
|
318
|
+
});
|
|
319
|
+
if (result.path) {
|
|
320
|
+
console.log(`[hicortex] Backup written: ${result.path} ` +
|
|
321
|
+
`(${result.files} files, ${result.bytes.toLocaleString()} bytes)`);
|
|
322
|
+
}
|
|
323
|
+
else {
|
|
324
|
+
console.error(`[hicortex] Backup streamed to stdout (${result.files} files, ${result.bytes.toLocaleString()} bytes compressed)`);
|
|
325
|
+
}
|
|
326
|
+
// Only run the hook when we have an on-disk artifact path to hand it. The
|
|
327
|
+
// --stdout path is already an offsite transport (piped to rclone/aws/B2),
|
|
328
|
+
// so a second hook would double-ship.
|
|
329
|
+
if (result.path && backupCommand) {
|
|
330
|
+
const hook = await runBackupHook(result.path, backupCommand);
|
|
331
|
+
if (!hook.ok) {
|
|
332
|
+
console.error(`[hicortex] Backup hook failed (exit ${hook.exitCode ?? "n/a"}). ` +
|
|
333
|
+
`Artifact is on disk; offsite copy did NOT complete.`);
|
|
334
|
+
process.exitCode = 1;
|
|
335
|
+
return;
|
|
336
|
+
}
|
|
337
|
+
console.log(`[hicortex] Backup hook ok (exit 0).`);
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
finally {
|
|
341
|
+
db.close();
|
|
342
|
+
}
|
|
343
|
+
}
|
package/dist/capture.d.ts
CHANGED
|
@@ -71,6 +71,15 @@ export interface PostResult {
|
|
|
71
71
|
dropped?: string[];
|
|
72
72
|
skipped?: boolean;
|
|
73
73
|
error?: string;
|
|
74
|
+
/**
|
|
75
|
+
* The segment's metered LLM usage from the 201 body (#287). Absent on a
|
|
76
|
+
* pre-#287 daemon (or any non-201) — callers must treat absent as zero.
|
|
77
|
+
*/
|
|
78
|
+
usage?: {
|
|
79
|
+
prompt: number;
|
|
80
|
+
completion: number;
|
|
81
|
+
total: number;
|
|
82
|
+
};
|
|
74
83
|
}
|
|
75
84
|
export type PostFn = (body: DistillBody) => Promise<PostResult>;
|
|
76
85
|
export interface CaptureOptions {
|
|
@@ -100,6 +109,17 @@ export interface CaptureResult {
|
|
|
100
109
|
* (429 memory cap) or "auth" (401). The caller decides watermark handling.
|
|
101
110
|
*/
|
|
102
111
|
stopped?: "limit" | "auth";
|
|
112
|
+
/**
|
|
113
|
+
* Sum of the successful segments' reported LLM usage (#287) — the run's
|
|
114
|
+
* distill spend, for the dashboard snapshot's token totals. Zero-filled when
|
|
115
|
+
* the daemon predates the usage field (an old server) or nothing distilled,
|
|
116
|
+
* so callers gate on `total > 0`, not on presence.
|
|
117
|
+
*/
|
|
118
|
+
distillUsage: {
|
|
119
|
+
prompt: number;
|
|
120
|
+
completion: number;
|
|
121
|
+
total: number;
|
|
122
|
+
};
|
|
103
123
|
}
|
|
104
124
|
/**
|
|
105
125
|
* Split an already-denoised string into ≤maxChars pieces (A2 hard-split).
|
package/dist/capture.js
CHANGED
|
@@ -139,6 +139,9 @@ async function captureBatches(batches, opts) {
|
|
|
139
139
|
let sessionsSent = 0;
|
|
140
140
|
let hadTransientFailure = false;
|
|
141
141
|
let stopped;
|
|
142
|
+
// #287: distill tokens reported by successful 201s only — skips (200) and
|
|
143
|
+
// failures metered nothing client-side.
|
|
144
|
+
const distillUsage = { prompt: 0, completion: 0, total: 0 };
|
|
142
145
|
for (const batch of batches) {
|
|
143
146
|
const short = batch.sessionId.slice(0, 8);
|
|
144
147
|
const segments = packSegments(batch.entries, batch.startCursor, batch.entryCursors, segmentMaxChars);
|
|
@@ -205,6 +208,11 @@ async function captureBatches(batches, opts) {
|
|
|
205
208
|
if (result.status === 201) {
|
|
206
209
|
memoriesIngested += result.distilled ?? 0;
|
|
207
210
|
sessionPosted = true;
|
|
211
|
+
if (result.usage) {
|
|
212
|
+
distillUsage.prompt += result.usage.prompt;
|
|
213
|
+
distillUsage.completion += result.usage.completion;
|
|
214
|
+
distillUsage.total += result.usage.total;
|
|
215
|
+
}
|
|
208
216
|
if (advancesBoundary)
|
|
209
217
|
lastConfirmedEnd = seg.segEnd;
|
|
210
218
|
console.log(`[hicortex] → ${result.distilled ?? 0} memories (segment ${body.segment_id})`);
|
|
@@ -258,7 +266,7 @@ async function captureBatches(batches, opts) {
|
|
|
258
266
|
if (stopped)
|
|
259
267
|
break;
|
|
260
268
|
}
|
|
261
|
-
return { memoriesIngested, sessionsSent, hadTransientFailure, stopped };
|
|
269
|
+
return { memoriesIngested, sessionsSent, hadTransientFailure, stopped, distillUsage };
|
|
262
270
|
}
|
|
263
271
|
// ---------------------------------------------------------------------------
|
|
264
272
|
// Single-flight guard (A5)
|
package/dist/cli.js
CHANGED
|
@@ -184,6 +184,33 @@ switch (command) {
|
|
|
184
184
|
});
|
|
185
185
|
break;
|
|
186
186
|
}
|
|
187
|
+
case "backup": {
|
|
188
|
+
// Backup — transactionally-consistent snapshot of the irreplaceable data
|
|
189
|
+
// (#6, Phase 0B). Mirrors `dedup`: flags parsed here, the runner (config
|
|
190
|
+
// load + DB open + createBackup + hook + close) lives in backup.ts so this
|
|
191
|
+
// switch stays thin and the heavy module is lazily imported.
|
|
192
|
+
const args = process.argv.slice(3);
|
|
193
|
+
let outDir;
|
|
194
|
+
try {
|
|
195
|
+
outDir = (0, cli_args_js_1.readValueFlag)(args, "--out");
|
|
196
|
+
}
|
|
197
|
+
catch {
|
|
198
|
+
console.error("[hicortex] backup: --out requires a directory path");
|
|
199
|
+
process.exit(1);
|
|
200
|
+
}
|
|
201
|
+
const stdout = args.includes("--stdout");
|
|
202
|
+
// `--out` is the output DIRECTORY (the artifact is auto-named
|
|
203
|
+
// hicortex-<ISO>.tar.gz inside it) — matches the `backupDir` config and the
|
|
204
|
+
// natural "put the backup here" invocation. Omit for the default <home>/backups.
|
|
205
|
+
const backupOptions = { outDir, stdout };
|
|
206
|
+
import("./backup.js").then(({ runBackupCli }) => {
|
|
207
|
+
runBackupCli(backupOptions).catch((err) => {
|
|
208
|
+
console.error(err instanceof Error ? err.message : `[hicortex] backup failed: ${err}`);
|
|
209
|
+
process.exit(1);
|
|
210
|
+
});
|
|
211
|
+
});
|
|
212
|
+
break;
|
|
213
|
+
}
|
|
187
214
|
case "dedup": {
|
|
188
215
|
const args = process.argv.slice(3);
|
|
189
216
|
let threshold;
|
|
@@ -310,6 +337,7 @@ Commands:
|
|
|
310
337
|
nightly Run nightly denoise + capture + consolidate
|
|
311
338
|
relink Resumable link-discovery pass over the ENTIRE corpus (server mode)
|
|
312
339
|
dedup Cluster + merge near-duplicate memories (server mode; dry run by default)
|
|
340
|
+
backup Snapshot the DB + identity + state to a tar.gz (online, WAL-safe)
|
|
313
341
|
classify-domains Backfill content-based domain tags over the corpus (server mode, needs config.domains)
|
|
314
342
|
classify-types Backfill episode→fact/decision type tags over the corpus (server mode)
|
|
315
343
|
learnings-identity Fetch identity + lessons and print Markdown to stdout (CC SessionStart hook)
|
|
@@ -335,6 +363,8 @@ Options:
|
|
|
335
363
|
dedup --apply Execute the merge (default: dry run, report only)
|
|
336
364
|
dedup --threshold <t> Override config dedupMergeThreshold for one run
|
|
337
365
|
dedup --db <path> DB path override (defaults to the configured DB)
|
|
366
|
+
backup --out <dir> Write the artifact into <dir> as hicortex-<ISO>.tar.gz (default: <home>/backups)
|
|
367
|
+
backup --stdout Stream the tar.gz to stdout (offsite pipe: hicortex backup --stdout | rclone rcat …)
|
|
338
368
|
classify-domains --all Reclassify every memory (default: only NULL/stale-domain rows)
|
|
339
369
|
classify-domains --batch <n> Memories per batch (default: 200)
|
|
340
370
|
classify-domains --reset Restart from the beginning (ignore saved cursor)
|
package/dist/config-read.d.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Strict config readers for primitive values. Pure functions — no LlmConfig
|
|
3
|
-
* dependency — shared by the nightly preflight knobs
|
|
4
|
-
* overlay (llm.ts / mcp-server.ts)
|
|
3
|
+
* dependency — shared by the nightly preflight knobs, the distill-tier
|
|
4
|
+
* overlay (llm.ts / mcp-server.ts), and the dashboard account identity
|
|
5
|
+
* (dashboard.ts).
|
|
5
6
|
*
|
|
6
7
|
* The point is to reject wrong-typed config values AT THE BOUNDARY (disk →
|
|
7
8
|
* runtime) with a warn, rather than casting them straight through. The trap
|
|
@@ -42,6 +43,31 @@ export declare function readNonNegativeConfig(config: Record<string, unknown>, k
|
|
|
42
43
|
* posture, and the parseConfigDomains leniency contract for the absent case.
|
|
43
44
|
*/
|
|
44
45
|
export declare function parseHours(config: Record<string, unknown> | null | undefined, key: string): number[] | null;
|
|
46
|
+
/**
|
|
47
|
+
* Read an optional non-empty string from a config object. Returns the trimmed
|
|
48
|
+
* string when valid, `null` when the key is absent, blank, OR present-but-not-
|
|
49
|
+
* a-string (with a warn in the latter case). Used for display-only keys (e.g.
|
|
50
|
+
* dashboard account identity) where "not set" and "invalid" mean the same
|
|
51
|
+
* thing: render nothing.
|
|
52
|
+
*/
|
|
53
|
+
export declare function readStringConfig(config: Record<string, unknown>, key: string): string | null;
|
|
54
|
+
/**
|
|
55
|
+
* Account identity shown in the console nav (name + plan pill). The display
|
|
56
|
+
* keys are operator-set config (hosted installs); each field is null when
|
|
57
|
+
* absent, blank, or wrong-typed (readStringConfig) — the page renders nothing
|
|
58
|
+
* for an all-null account (the self-hosted default), never the string "null".
|
|
59
|
+
*
|
|
60
|
+
* ONE construction shared by the /dashboard/data payload and GET /account so
|
|
61
|
+
* the nav element cannot drift between them (same reason the digest reuses
|
|
62
|
+
* formatIndexLine).
|
|
63
|
+
*/
|
|
64
|
+
export interface AccountIdentity {
|
|
65
|
+
name: string | null;
|
|
66
|
+
org: string | null;
|
|
67
|
+
plan: string | null;
|
|
68
|
+
}
|
|
69
|
+
/** Read the account identity (displayName/orgName/planLabel) from config. */
|
|
70
|
+
export declare function readAccount(config: Record<string, unknown> | null | undefined): AccountIdentity;
|
|
45
71
|
/**
|
|
46
72
|
* Warn if the saved config carries keys that 0.16.8+ ignores. Call at every
|
|
47
73
|
* config read (daemon boot + nightly). The warning clears once the keys are
|
package/dist/config-read.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/**
|
|
3
3
|
* Strict config readers for primitive values. Pure functions — no LlmConfig
|
|
4
|
-
* dependency — shared by the nightly preflight knobs
|
|
5
|
-
* overlay (llm.ts / mcp-server.ts)
|
|
4
|
+
* dependency — shared by the nightly preflight knobs, the distill-tier
|
|
5
|
+
* overlay (llm.ts / mcp-server.ts), and the dashboard account identity
|
|
6
|
+
* (dashboard.ts).
|
|
6
7
|
*
|
|
7
8
|
* The point is to reject wrong-typed config values AT THE BOUNDARY (disk →
|
|
8
9
|
* runtime) with a warn, rather than casting them straight through. The trap
|
|
@@ -17,6 +18,8 @@ exports.readPositiveConfig = readPositiveConfig;
|
|
|
17
18
|
exports.readStrictBoolean = readStrictBoolean;
|
|
18
19
|
exports.readNonNegativeConfig = readNonNegativeConfig;
|
|
19
20
|
exports.parseHours = parseHours;
|
|
21
|
+
exports.readStringConfig = readStringConfig;
|
|
22
|
+
exports.readAccount = readAccount;
|
|
20
23
|
exports.warnIgnoredConfigKeys = warnIgnoredConfigKeys;
|
|
21
24
|
/**
|
|
22
25
|
* Read a positive finite number from a config object. Returns `def` when the
|
|
@@ -96,6 +99,33 @@ function parseHours(config, key) {
|
|
|
96
99
|
}
|
|
97
100
|
return out.length > 0 ? out.sort((a, b) => a - b) : null;
|
|
98
101
|
}
|
|
102
|
+
/**
|
|
103
|
+
* Read an optional non-empty string from a config object. Returns the trimmed
|
|
104
|
+
* string when valid, `null` when the key is absent, blank, OR present-but-not-
|
|
105
|
+
* a-string (with a warn in the latter case). Used for display-only keys (e.g.
|
|
106
|
+
* dashboard account identity) where "not set" and "invalid" mean the same
|
|
107
|
+
* thing: render nothing.
|
|
108
|
+
*/
|
|
109
|
+
function readStringConfig(config, key) {
|
|
110
|
+
const v = config[key];
|
|
111
|
+
if (v === undefined)
|
|
112
|
+
return null;
|
|
113
|
+
if (typeof v === "string") {
|
|
114
|
+
const t = v.trim();
|
|
115
|
+
return t.length > 0 ? t : null;
|
|
116
|
+
}
|
|
117
|
+
console.warn(`[hicortex] config "${key}" = ${String(v)} is not a string — ignored.`);
|
|
118
|
+
return null;
|
|
119
|
+
}
|
|
120
|
+
/** Read the account identity (displayName/orgName/planLabel) from config. */
|
|
121
|
+
function readAccount(config) {
|
|
122
|
+
const c = config ?? {};
|
|
123
|
+
return {
|
|
124
|
+
name: readStringConfig(c, "displayName"),
|
|
125
|
+
org: readStringConfig(c, "orgName"),
|
|
126
|
+
plan: readStringConfig(c, "planLabel"),
|
|
127
|
+
};
|
|
128
|
+
}
|
|
99
129
|
/**
|
|
100
130
|
* Config keys that ≤0.16.7 accepted and 0.16.8+ IGNORES. Two groups:
|
|
101
131
|
* - per-stage model keys + the nested `models` block (#231): one model now
|
package/dist/consolidate.js
CHANGED
|
@@ -985,8 +985,6 @@ exports.DEFAULT_SUPERSESSION_MAX_CALLS = 0;
|
|
|
985
985
|
const SUPERSESSION_NEIGHBOR_POOL = 15;
|
|
986
986
|
/** Older-neighbor pairs kept per candidate after filtering. */
|
|
987
987
|
const SUPERSESSION_NEIGHBOR_TOP_K = 5;
|
|
988
|
-
/** Candidate rows read per SQL page (call budget stops the loop well before this in practice). */
|
|
989
|
-
const SUPERSESSION_BATCH_SIZE = 500;
|
|
990
988
|
/**
|
|
991
989
|
* A memory whose content/type marks it as a SUPERSEDABLE claim — one a newer
|
|
992
990
|
* memory about the same subject can replace. Decisions and corrections were the
|
|
@@ -1122,8 +1120,8 @@ async function stageSupersession(db, llm, budget, embedFn, dryRun, stateDir, opt
|
|
|
1122
1120
|
OR content LIKE '%[Corrections & Rejections]%'
|
|
1123
1121
|
OR content LIKE '%[Facts Learned]%'
|
|
1124
1122
|
OR content LIKE '%[Project State Changes]%')
|
|
1125
|
-
ORDER BY rowid ASC
|
|
1126
|
-
.all(startCursor
|
|
1123
|
+
ORDER BY rowid ASC`)
|
|
1124
|
+
.all(startCursor);
|
|
1127
1125
|
let scanned = 0;
|
|
1128
1126
|
let evaluated = 0;
|
|
1129
1127
|
let superseded = 0;
|