@gamaze/hicortex 0.18.3 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +60 -0
- package/assets/dashboard.html +103 -7
- package/assets/identity.html +48 -1
- package/assets/viz.html +52 -1
- package/dist/backup.d.ts +107 -0
- package/dist/backup.js +343 -0
- package/dist/capture.d.ts +20 -0
- package/dist/capture.js +9 -1
- package/dist/cli.js +30 -0
- package/dist/config-read.d.ts +28 -2
- package/dist/config-read.js +32 -2
- package/dist/consolidate.js +2 -4
- package/dist/dashboard.d.ts +71 -6
- package/dist/dashboard.js +61 -7
- package/dist/mcp-server.d.ts +18 -0
- package/dist/mcp-server.js +60 -2
- package/dist/memory-instructions.js +1 -1
- package/dist/nightly.js +104 -2
- package/dist/prompts.js +19 -4
- package/dist/telemetry.d.ts +14 -0
- package/dist/token-budget.js +10 -2
- package/dist/type-classify.js +7 -4
- package/dist/types.d.ts +40 -0
- package/package.json +6 -4
package/dist/dashboard.d.ts
CHANGED
|
@@ -42,12 +42,21 @@ export interface DashboardMetrics {
|
|
|
42
42
|
* cap; undefined on backfill rows (a stage outcome, not reconstructable). */
|
|
43
43
|
evicted?: number;
|
|
44
44
|
/**
|
|
45
|
-
* Total LLM tokens consumed by this run
|
|
46
|
-
*
|
|
47
|
-
*
|
|
45
|
+
* Total LLM tokens consumed by this run (#246 consolidation meter; #287
|
|
46
|
+
* widened to the TRUE total — distill + consolidation). Undefined in
|
|
47
|
+
* lockstep with `tokens_by_stage` (and on backfill rows, which can't
|
|
48
|
+
* reconstruct a per-run meter). Older snapshots are consolidation-only:
|
|
49
|
+
* historical rows can't be reconstructed, which is accepted (#287).
|
|
50
|
+
* Inherent under-count, same acceptance: attribution is response-based,
|
|
51
|
+
* so tokens a FAILED distill already spent (500 after spend, response
|
|
52
|
+
* lost after commit) reach the monthly meter but never a run's total —
|
|
53
|
+
* after such a night, the month's bars sum slightly below the headline.
|
|
48
54
|
*/
|
|
49
55
|
tokens?: number;
|
|
50
|
-
/**
|
|
56
|
+
/**
|
|
57
|
+
* Per-stage breakdown of `tokens` (#246; #287 adds a `distill` entry for
|
|
58
|
+
* capture-time distillation). Undefined on backfill rows.
|
|
59
|
+
*/
|
|
51
60
|
tokens_by_stage?: Record<string, {
|
|
52
61
|
prompt: number;
|
|
53
62
|
completion: number;
|
|
@@ -79,6 +88,19 @@ export interface DashboardMetrics {
|
|
|
79
88
|
* shape is clean on healthy runs.
|
|
80
89
|
*/
|
|
81
90
|
budget_deferred_by_stage?: Record<string, number>;
|
|
91
|
+
/**
|
|
92
|
+
* #6 backup stage outcome (Phase 0B). Present whenever the backup stage
|
|
93
|
+
* ran (full nightly); absent on capture-only / dry-run / backfill rows.
|
|
94
|
+
* `ok` is false when the snapshot OR the operator's offsite hook failed —
|
|
95
|
+
* the page flags a night the offsite copy didn't land. `bytes` is the
|
|
96
|
+
* compressed artifact size; `path` is the on-disk artifact (for "where
|
|
97
|
+
* did the last backup land?" debugging — not a restore button).
|
|
98
|
+
*/
|
|
99
|
+
backup?: {
|
|
100
|
+
ok: boolean;
|
|
101
|
+
bytes: number;
|
|
102
|
+
path?: string;
|
|
103
|
+
};
|
|
82
104
|
};
|
|
83
105
|
/** Corpus capacity (#245). `memory_soft_cap` is the configured ceiling (0 =
|
|
84
106
|
* disabled); always present in real snapshots, undefined on backfilled
|
|
@@ -102,6 +124,17 @@ export interface DashboardSnapshot {
|
|
|
102
124
|
}
|
|
103
125
|
/** The /dashboard/data response — the full payload the page renders. */
|
|
104
126
|
export interface DashboardData {
|
|
127
|
+
/**
|
|
128
|
+
* Account identity (hosted): who the viewer is, so a user holding two
|
|
129
|
+
* tenant tokens can tell whose data the page shows. Each field is null when
|
|
130
|
+
* its config key (displayName/orgName/planLabel) is absent — the page
|
|
131
|
+
* renders nothing when ALL are null (the self-hosted default).
|
|
132
|
+
*/
|
|
133
|
+
account: {
|
|
134
|
+
name: string | null;
|
|
135
|
+
org: string | null;
|
|
136
|
+
plan: string | null;
|
|
137
|
+
};
|
|
105
138
|
headline: {
|
|
106
139
|
total_memories: number;
|
|
107
140
|
uses_per_showing: number | null;
|
|
@@ -149,9 +182,10 @@ export interface DashboardData {
|
|
|
149
182
|
supersession: number;
|
|
150
183
|
added: number;
|
|
151
184
|
evicted?: number;
|
|
152
|
-
/** Total tokens consumed that run (#246
|
|
185
|
+
/** Total tokens consumed that run (#246; #287: distill + consolidation).
|
|
186
|
+
* Undefined = no metered run. */
|
|
153
187
|
tokens?: number;
|
|
154
|
-
/** Per-stage breakdown of `tokens` (#246). */
|
|
188
|
+
/** Per-stage breakdown of `tokens` (#246; #287 adds `distill`). */
|
|
155
189
|
tokens_by_stage?: Record<string, {
|
|
156
190
|
prompt: number;
|
|
157
191
|
completion: number;
|
|
@@ -207,6 +241,19 @@ export interface NightlyDelta {
|
|
|
207
241
|
completion: number;
|
|
208
242
|
total: number;
|
|
209
243
|
}>;
|
|
244
|
+
/**
|
|
245
|
+
* Distill tokens metered by the daemon across this run's capture POSTs
|
|
246
|
+
* (#287) — summed from the /distill responses by the capture loop. Merged
|
|
247
|
+
* into the snapshot so `new_this_run.tokens` is the run's TRUE total
|
|
248
|
+
* (distill + consolidation) and `distill` joins `tokens_by_stage`. Zero
|
|
249
|
+
* (a daemon predating the usage field, or nothing distilled) is a no-op:
|
|
250
|
+
* the row keeps its consolidation-only shape.
|
|
251
|
+
*/
|
|
252
|
+
distillUsage?: {
|
|
253
|
+
prompt: number;
|
|
254
|
+
completion: number;
|
|
255
|
+
total: number;
|
|
256
|
+
};
|
|
210
257
|
/**
|
|
211
258
|
* Always-on consolidation-budget usage (#255 CR). Forwarded whenever
|
|
212
259
|
* consolidation ran so the dashboard renders a continuous used/max bar.
|
|
@@ -227,6 +274,15 @@ export interface NightlyDelta {
|
|
|
227
274
|
* with `budgetExhausted`.
|
|
228
275
|
*/
|
|
229
276
|
budgetDeferredByStage?: Record<string, number>;
|
|
277
|
+
/**
|
|
278
|
+
* #6 backup stage (Phase 0B). Hoisted from the nightly backup block. Present
|
|
279
|
+
* whenever the backup stage ran (full nightly); undefined on capture-only /
|
|
280
|
+
* dry-run. `backupOk` flips to false on snapshot OR hook failure so the
|
|
281
|
+
* digest can flag a night the offsite copy didn't land.
|
|
282
|
+
*/
|
|
283
|
+
backupPath?: string;
|
|
284
|
+
backupBytes?: number;
|
|
285
|
+
backupOk?: boolean;
|
|
230
286
|
}
|
|
231
287
|
/**
|
|
232
288
|
* Write one snapshot row for `runAt` (an ISO timestamp the caller chooses —
|
|
@@ -286,3 +342,12 @@ export declare function handleDashboardData(db: Database.Database, query: {
|
|
|
286
342
|
* Failures surface as a 500 with the usual {error} shape — no silent degrade.
|
|
287
343
|
*/
|
|
288
344
|
export declare function dashboardDataHandler(getDb: () => Database.Database, getConfig: () => Record<string, unknown> | null | undefined): express.RequestHandler;
|
|
345
|
+
/**
|
|
346
|
+
* Express adapter for GET /account — the account identity ONLY (name/org/plan
|
|
347
|
+
* from config), so the /viz and /identity/ui pages can render the nav account
|
|
348
|
+
* element without pulling the full /dashboard/data payload. Same readAccount()
|
|
349
|
+
* construction as the dashboard payload — one shape, two surfaces. Also the
|
|
350
|
+
* natural whoami for the future OAuth session (#292). Failures surface as a
|
|
351
|
+
* 500 with the usual {error} shape (same as dashboardDataHandler).
|
|
352
|
+
*/
|
|
353
|
+
export declare function accountHandler(getConfig: () => Record<string, unknown> | null | undefined): express.RequestHandler;
|
package/dist/dashboard.js
CHANGED
|
@@ -25,6 +25,7 @@ exports.writeSnapshot = writeSnapshot;
|
|
|
25
25
|
exports.backfillSnapshots = backfillSnapshots;
|
|
26
26
|
exports.handleDashboardData = handleDashboardData;
|
|
27
27
|
exports.dashboardDataHandler = dashboardDataHandler;
|
|
28
|
+
exports.accountHandler = accountHandler;
|
|
28
29
|
const recall_index_js_1 = require("./recall-index.js");
|
|
29
30
|
const config_read_js_1 = require("./config-read.js");
|
|
30
31
|
const consolidate_js_1 = require("./consolidate.js");
|
|
@@ -92,17 +93,36 @@ function computeDashboardMetrics(db) {
|
|
|
92
93
|
*/
|
|
93
94
|
function writeSnapshot(db, runAt, delta, memorySoftCap) {
|
|
94
95
|
const metrics = computeDashboardMetrics(db);
|
|
96
|
+
// #287: merge the run's two meters into the customer-facing total. `tokens`
|
|
97
|
+
// = consolidation (tokensThisRun) + distill (distillUsage.total); the distill
|
|
98
|
+
// share joins the stage map under its own key. Both fields stay in lockstep —
|
|
99
|
+
// emitted when EITHER phase metered, omitted when neither did (the page
|
|
100
|
+
// treats undefined as "no data for this day"). A zero/absent distillUsage
|
|
101
|
+
// (old daemon, nothing distilled) changes nothing: tokens/tokens_by_stage
|
|
102
|
+
// come through exactly as the consolidation report produced them.
|
|
103
|
+
const hasDistill = (delta.distillUsage?.total ?? 0) > 0;
|
|
104
|
+
const metered = delta.tokensThisRun !== undefined || hasDistill;
|
|
105
|
+
const mergedTokens = metered
|
|
106
|
+
? (delta.tokensThisRun ?? 0) + (hasDistill ? delta.distillUsage.total : 0)
|
|
107
|
+
: undefined;
|
|
108
|
+
const mergedStages = metered
|
|
109
|
+
? { ...(delta.tokensByStage ?? {}), ...(hasDistill ? { distill: delta.distillUsage } : {}) }
|
|
110
|
+
: undefined;
|
|
111
|
+
// Shape fidelity: `tokens_by_stage` with zero keys never existed pre-#287
|
|
112
|
+
// (the key was simply absent) — keep it that way so consumers that treat
|
|
113
|
+
// "present" as "has a breakdown" stay right.
|
|
114
|
+
const emitStages = mergedStages && Object.keys(mergedStages).length > 0 ? mergedStages : undefined;
|
|
95
115
|
metrics.new_this_run = {
|
|
96
116
|
added: delta.added,
|
|
97
117
|
lessonsGenerated: delta.lessonsGenerated,
|
|
98
118
|
dedup: delta.dedup,
|
|
99
119
|
supersession: delta.supersession,
|
|
100
120
|
evicted: delta.evicted,
|
|
101
|
-
// #246: forward only when
|
|
102
|
-
//
|
|
103
|
-
//
|
|
104
|
-
...(
|
|
105
|
-
...(
|
|
121
|
+
// #246: forward only when a phase actually metered tokens this run. Absent
|
|
122
|
+
// on capture-only / throttled / no-LLM / no-metered-call runs — the page
|
|
123
|
+
// treats undefined as "no data for this day", matching adoption.
|
|
124
|
+
...(mergedTokens !== undefined ? { tokens: mergedTokens } : {}),
|
|
125
|
+
...(emitStages !== undefined ? { tokens_by_stage: emitStages } : {}),
|
|
106
126
|
// #255 CR: always-on usage metric — forward calls_used + max_calls
|
|
107
127
|
// whenever consolidation ran (regardless of exhaustion) so the page can
|
|
108
128
|
// render a continuous used/max bar. Presence = a run happened; absence =
|
|
@@ -121,6 +141,17 @@ function writeSnapshot(db, runAt, delta, memorySoftCap) {
|
|
|
121
141
|
: {}),
|
|
122
142
|
}
|
|
123
143
|
: {}),
|
|
144
|
+
// #6 backup stage — forwarded as a nested object only when the stage ran
|
|
145
|
+
// (backupOk !== undefined). Absent on capture-only / dry-run / backfill.
|
|
146
|
+
...(delta.backupOk !== undefined
|
|
147
|
+
? {
|
|
148
|
+
backup: {
|
|
149
|
+
ok: delta.backupOk === true,
|
|
150
|
+
bytes: delta.backupBytes ?? 0,
|
|
151
|
+
...(delta.backupPath ? { path: delta.backupPath } : {}),
|
|
152
|
+
},
|
|
153
|
+
}
|
|
154
|
+
: {}),
|
|
124
155
|
};
|
|
125
156
|
if (memorySoftCap !== undefined) {
|
|
126
157
|
metrics.capacity = { memory_soft_cap: memorySoftCap };
|
|
@@ -418,8 +449,9 @@ function handleDashboardData(db, query, config) {
|
|
|
418
449
|
supersession: dayMetrics?.new_this_run?.supersession ?? supersessionCount,
|
|
419
450
|
added: dayMetrics?.new_this_run?.added ?? sampleRows.length,
|
|
420
451
|
evicted: dayMetrics?.new_this_run?.evicted,
|
|
421
|
-
// #246: only present when the day's nightly metered tokens
|
|
422
|
-
// are forwarded together — the page
|
|
452
|
+
// #246/#287: only present when the day's nightly metered tokens (distill
|
|
453
|
+
// or consolidation). Both fields are forwarded together — the page
|
|
454
|
+
// renders either the breakdown or nothing.
|
|
423
455
|
tokens: dayMetrics?.new_this_run?.tokens,
|
|
424
456
|
tokens_by_stage: dayMetrics?.new_this_run?.tokens_by_stage,
|
|
425
457
|
// #255 CR: always-on usage metric — present whenever consolidation ran
|
|
@@ -441,6 +473,10 @@ function handleDashboardData(db, query, config) {
|
|
|
441
473
|
return {
|
|
442
474
|
status: 200,
|
|
443
475
|
body: {
|
|
476
|
+
// Account identity — read defensively like the numeric knobs above:
|
|
477
|
+
// null when absent/not a string (page renders nothing, never "null").
|
|
478
|
+
// Shared readAccount() so GET /account renders the identical shape.
|
|
479
|
+
account: (0, config_read_js_1.readAccount)(config),
|
|
444
480
|
range: rangeParam,
|
|
445
481
|
headline,
|
|
446
482
|
series,
|
|
@@ -469,3 +505,21 @@ function dashboardDataHandler(getDb, getConfig) {
|
|
|
469
505
|
}
|
|
470
506
|
};
|
|
471
507
|
}
|
|
508
|
+
/**
|
|
509
|
+
* Express adapter for GET /account — the account identity ONLY (name/org/plan
|
|
510
|
+
* from config), so the /viz and /identity/ui pages can render the nav account
|
|
511
|
+
* element without pulling the full /dashboard/data payload. Same readAccount()
|
|
512
|
+
* construction as the dashboard payload — one shape, two surfaces. Also the
|
|
513
|
+
* natural whoami for the future OAuth session (#292). Failures surface as a
|
|
514
|
+
* 500 with the usual {error} shape (same as dashboardDataHandler).
|
|
515
|
+
*/
|
|
516
|
+
function accountHandler(getConfig) {
|
|
517
|
+
return (_req, res) => {
|
|
518
|
+
try {
|
|
519
|
+
res.status(200).json({ account: (0, config_read_js_1.readAccount)(getConfig()) });
|
|
520
|
+
}
|
|
521
|
+
catch (err) {
|
|
522
|
+
res.status(500).json({ error: err instanceof Error ? err.message : String(err) });
|
|
523
|
+
}
|
|
524
|
+
};
|
|
525
|
+
}
|
package/dist/mcp-server.d.ts
CHANGED
|
@@ -10,7 +10,25 @@
|
|
|
10
10
|
* GET /sse — SSE stream for MCP clients
|
|
11
11
|
* POST /messages — message endpoint for MCP clients
|
|
12
12
|
*/
|
|
13
|
+
import express from "express";
|
|
13
14
|
import type { MemorySearchResult } from "./types.js";
|
|
15
|
+
/**
|
|
16
|
+
* Resolve the request body-size limit in MB (#7). Pure — exported for tests.
|
|
17
|
+
* Precedence: an explicit config value > hosted-mode default (5) > self-hosted
|
|
18
|
+
* default (25, the historical fixed value → no regression). A finite positive
|
|
19
|
+
* config value wins; invalid/absent falls through.
|
|
20
|
+
*/
|
|
21
|
+
export declare function resolveBodyLimitMb(configVal: unknown, hostedMode: boolean): number;
|
|
22
|
+
/**
|
|
23
|
+
* Express error middleware (#7): translate express.json's default HTML 413
|
|
24
|
+
* (entity.too.large) into a consistent JSON response. Catches body-parser
|
|
25
|
+
* errors only — which express.json emits BEFORE any route runs — so by
|
|
26
|
+
* registration order (this sits ahead of the routes) it never intercepts an
|
|
27
|
+
* error thrown inside a route handler; those reach Express's default handler.
|
|
28
|
+
* The `status === 413 || type === "entity.too.large"` check is defense-in-depth
|
|
29
|
+
* on top of that ordering. Exported so tests exercise the real handler.
|
|
30
|
+
*/
|
|
31
|
+
export declare function makeBodyLimitErrorHandler(limitMb: number): express.ErrorRequestHandler;
|
|
14
32
|
export declare function startServer(options?: {
|
|
15
33
|
port?: number;
|
|
16
34
|
host?: string;
|
package/dist/mcp-server.js
CHANGED
|
@@ -48,6 +48,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
48
48
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
49
49
|
};
|
|
50
50
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
51
|
+
exports.resolveBodyLimitMb = resolveBodyLimitMb;
|
|
52
|
+
exports.makeBodyLimitErrorHandler = makeBodyLimitErrorHandler;
|
|
51
53
|
exports.startServer = startServer;
|
|
52
54
|
exports.formatResults = formatResults;
|
|
53
55
|
const express_1 = __importDefault(require("express"));
|
|
@@ -407,6 +409,38 @@ function createMcpServer() {
|
|
|
407
409
|
// ---------------------------------------------------------------------------
|
|
408
410
|
// HTTP server with SSE transport
|
|
409
411
|
// ---------------------------------------------------------------------------
|
|
412
|
+
/**
|
|
413
|
+
* Resolve the request body-size limit in MB (#7). Pure — exported for tests.
|
|
414
|
+
* Precedence: an explicit config value > hosted-mode default (5) > self-hosted
|
|
415
|
+
* default (25, the historical fixed value → no regression). A finite positive
|
|
416
|
+
* config value wins; invalid/absent falls through.
|
|
417
|
+
*/
|
|
418
|
+
function resolveBodyLimitMb(configVal, hostedMode) {
|
|
419
|
+
const cfg = Number(configVal);
|
|
420
|
+
if (Number.isFinite(cfg) && cfg > 0)
|
|
421
|
+
return cfg;
|
|
422
|
+
return hostedMode ? 5 : 25;
|
|
423
|
+
}
|
|
424
|
+
/**
|
|
425
|
+
* Express error middleware (#7): translate express.json's default HTML 413
|
|
426
|
+
* (entity.too.large) into a consistent JSON response. Catches body-parser
|
|
427
|
+
* errors only — which express.json emits BEFORE any route runs — so by
|
|
428
|
+
* registration order (this sits ahead of the routes) it never intercepts an
|
|
429
|
+
* error thrown inside a route handler; those reach Express's default handler.
|
|
430
|
+
* The `status === 413 || type === "entity.too.large"` check is defense-in-depth
|
|
431
|
+
* on top of that ordering. Exported so tests exercise the real handler.
|
|
432
|
+
*/
|
|
433
|
+
function makeBodyLimitErrorHandler(limitMb) {
|
|
434
|
+
return (err, _req, res, next) => {
|
|
435
|
+
const status = err.status;
|
|
436
|
+
const type = err.type;
|
|
437
|
+
if (status === 413 || type === "entity.too.large") {
|
|
438
|
+
res.status(413).json({ error: "request body too large", limit_mb: limitMb });
|
|
439
|
+
return;
|
|
440
|
+
}
|
|
441
|
+
next(err);
|
|
442
|
+
};
|
|
443
|
+
}
|
|
410
444
|
async function startServer(options = {}) {
|
|
411
445
|
const port = options.port ?? 8787;
|
|
412
446
|
const host = options.host ?? "0.0.0.0";
|
|
@@ -493,6 +527,12 @@ async function startServer(options = {}) {
|
|
|
493
527
|
// env (provider-set, tenant-immutable) which takes precedence. Initialised here
|
|
494
528
|
// (after stateDir + savedConfig are known) so the warn-dedup can seed from state.
|
|
495
529
|
(0, token_budget_js_1.initTokenBudget)(stateDir, savedConfig?.llmTokensPerMonth);
|
|
530
|
+
// #7: request body-size limit. Config key wins; else 5 MB hosted / 25 MB
|
|
531
|
+
// self-hosted (the prior fixed value → no regression). Guards the OOM vector
|
|
532
|
+
// (the body is fully parsed into memory before the distiller truncates to 80K
|
|
533
|
+
// chars). Legitimate capture segments are ≤60K chars (~200KB), so this never
|
|
534
|
+
// constrains real flow — it's an abuse/backstop. Oversized → 413.
|
|
535
|
+
const bodyLimitMb = resolveBodyLimitMb(savedConfig?.distillBodyLimitMb, hostedMode);
|
|
496
536
|
if (savedConfig?.llmBackend === "claude-cli") {
|
|
497
537
|
const claudePath = (0, llm_js_1.findClaudeBinary)();
|
|
498
538
|
if (claudePath) {
|
|
@@ -636,7 +676,11 @@ async function startServer(options = {}) {
|
|
|
636
676
|
// Express app
|
|
637
677
|
const app = (0, express_1.default)();
|
|
638
678
|
// Raise the body limit — whole-session denoised transcripts exceed the 100 kB default.
|
|
639
|
-
app.use(express_1.default.json({ limit:
|
|
679
|
+
app.use(express_1.default.json({ limit: `${bodyLimitMb}mb` }));
|
|
680
|
+
// #7: JSON 413 on body-limit exceed (see makeBodyLimitErrorHandler). Server-side
|
|
681
|
+
// only — the client capture loop treats 413 like any non-2xx (holds cursor);
|
|
682
|
+
// it never fires for legitimate capture (segments ≤200KB ≪ the limit).
|
|
683
|
+
app.use(makeBodyLimitErrorHandler(bodyLimitMb));
|
|
640
684
|
// CORS: reflect ONLY explicitly-allowlisted origins (config.corsAllowedOrigins),
|
|
641
685
|
// and never send Access-Control-Allow-Credentials. Reflecting any origin with
|
|
642
686
|
// credentials — combined with the localhost auth bypass and the default 0.0.0.0
|
|
@@ -997,7 +1041,8 @@ async function startServer(options = {}) {
|
|
|
997
1041
|
});
|
|
998
1042
|
// REST /distill — canonical capture endpoint (0.9.0+).
|
|
999
1043
|
// Every machine (including the server itself) POSTs denoised session text here.
|
|
1000
|
-
// The server distills, embeds, stores. Body limit:
|
|
1044
|
+
// The server distills, embeds, stores. Body limit: see `distillBodyLimitMb`
|
|
1045
|
+
// (default 25 MB self-hosted / 5 MB hosted); oversized → 413 (#7).
|
|
1001
1046
|
//
|
|
1002
1047
|
// Accepts text (string, preferred nightly path) OR messages (array, legacy).
|
|
1003
1048
|
// Performs session-level dedup when session_id is present without segment_id.
|
|
@@ -1161,6 +1206,12 @@ async function startServer(options = {}) {
|
|
|
1161
1206
|
ids,
|
|
1162
1207
|
distilled: ids.length,
|
|
1163
1208
|
dropped: dropped.map((d) => (d.length > 120 ? `${d.slice(0, 120)}…` : d)),
|
|
1209
|
+
// #287: this segment's metered usage — the same breakdown
|
|
1210
|
+
// recordDistillUsage accrues below. Lets the capturing nightly
|
|
1211
|
+
// attribute distill tokens in its dashboard snapshot
|
|
1212
|
+
// (new_this_run.tokens_by_stage.distill). Always present (zeros when
|
|
1213
|
+
// no chunk reached an LLM call); pre-#287 clients ignore it.
|
|
1214
|
+
usage: distillUsage,
|
|
1164
1215
|
});
|
|
1165
1216
|
}
|
|
1166
1217
|
catch (err) {
|
|
@@ -1452,6 +1503,13 @@ async function startServer(options = {}) {
|
|
|
1452
1503
|
// express adapter that injects the live db + config. STRICTLY view-only —
|
|
1453
1504
|
// no mutation endpoints on the dashboard surface.
|
|
1454
1505
|
app.get("/dashboard/data", (0, dashboard_js_1.dashboardDataHandler)(() => db, () => readConfigFile(stateDir)));
|
|
1506
|
+
// GET /account — account identity for the console nav (name/org/plan from
|
|
1507
|
+
// config). The LIGHTWEIGHT twin of the account block inside /dashboard/data:
|
|
1508
|
+
// the /viz and /identity/ui pages need only this, not the metric payload;
|
|
1509
|
+
// also the natural whoami for the future OAuth session (#292). Bearer-only
|
|
1510
|
+
// (standard auth middleware, no shell exemption — it carries data); localhost
|
|
1511
|
+
// bypass applies. Handler lives in src/dashboard.ts next to its twin.
|
|
1512
|
+
app.get("/account", (0, dashboard_js_1.accountHandler)(() => readConfigFile(stateDir)));
|
|
1455
1513
|
// SSE endpoint — each connection gets its own McpServer + transport
|
|
1456
1514
|
app.get("/sse", async (req, res) => {
|
|
1457
1515
|
const transport = new sse_js_1.SSEServerTransport("/messages", res);
|
|
@@ -30,7 +30,7 @@ exports.MEMORY_SECTION_NAME = "memory";
|
|
|
30
30
|
* injected once per session into every agent on the fleet. */
|
|
31
31
|
function renderMemoryInstructions() {
|
|
32
32
|
return [
|
|
33
|
-
"
|
|
33
|
+
"Hicortex is your persistent identity and long-term memory: what you learn, decide, and correct survives every session, compaction, and model switch — one memory shared by all your agents.",
|
|
34
34
|
"- A `## Memory recall (auto)` index may arrive with prompts: it is a MENU, not content. Fetch a full memory with `hicortex_get(id)` when the entry could change how you handle the current task.",
|
|
35
35
|
"- Recall before assuming: `hicortex_search` for prior decisions/facts/preferences, `hicortex_recent` to catch up on a project.",
|
|
36
36
|
"- Cite any memory you rely on by id + date, and mark it `FETCHED` (you read the full memory via `hicortex_get`) or `SNIPPET` (the one-line entry only). Don't present a SNIPPET citation as established. On conflicts, newer memories supersede older.",
|
package/dist/nightly.js
CHANGED
|
@@ -75,6 +75,7 @@ const capture_js_1 = require("./capture.js");
|
|
|
75
75
|
const dashboard_js_1 = require("./dashboard.js");
|
|
76
76
|
const telemetry_js_1 = require("./telemetry.js");
|
|
77
77
|
const init_js_1 = require("./init.js");
|
|
78
|
+
const backup_js_1 = require("./backup.js");
|
|
78
79
|
const HICORTEX_HOME = (0, paths_js_1.hicortexHome)();
|
|
79
80
|
function readNightlyConfig(stateDir) {
|
|
80
81
|
const configPath = (0, node_path_1.join)(stateDir, "config.json");
|
|
@@ -168,7 +169,16 @@ function makeRemotePost(serverUrl, authToken) {
|
|
|
168
169
|
async function normalizePostResult(resp) {
|
|
169
170
|
if (resp.status === 201) {
|
|
170
171
|
const data = (await resp.json().catch(() => ({})));
|
|
171
|
-
|
|
172
|
+
// #287: the daemon reports the segment's metered usage. Shape-validated
|
|
173
|
+
// so a partial payload can't NaN the run's totals; a pre-#287 daemon
|
|
174
|
+
// simply omits it → capture sums zero (snapshot stays consolidation-only).
|
|
175
|
+
const usage = parseUsage(data.usage);
|
|
176
|
+
return {
|
|
177
|
+
status: 201,
|
|
178
|
+
distilled: data.distilled ?? 0,
|
|
179
|
+
dropped: data.dropped ?? [],
|
|
180
|
+
...(usage ? { usage } : {}),
|
|
181
|
+
};
|
|
172
182
|
}
|
|
173
183
|
if (resp.status === 200) {
|
|
174
184
|
const data = (await resp.json().catch(() => ({})));
|
|
@@ -177,6 +187,18 @@ async function normalizePostResult(resp) {
|
|
|
177
187
|
const data = (await resp.json().catch(() => ({})));
|
|
178
188
|
return { status: resp.status, error: data.error ?? "unknown error" };
|
|
179
189
|
}
|
|
190
|
+
/** Strict {prompt, completion, total} parser for /distill's usage field (#287);
|
|
191
|
+
* undefined on anything malformed — the caller then treats it as unmetered. */
|
|
192
|
+
function parseUsage(u) {
|
|
193
|
+
if (typeof u !== "object" || u === null)
|
|
194
|
+
return undefined;
|
|
195
|
+
const { prompt, completion, total } = u;
|
|
196
|
+
const num = (n) => typeof n === "number" && Number.isFinite(n) && n >= 0 ? n : null;
|
|
197
|
+
const p = num(prompt), c = num(completion), t = num(total);
|
|
198
|
+
if (p === null || c === null || t === null)
|
|
199
|
+
return undefined;
|
|
200
|
+
return { prompt: p, completion: c, total: t };
|
|
201
|
+
}
|
|
180
202
|
function writeLastRun(stateDir = HICORTEX_HOME) {
|
|
181
203
|
(0, state_js_1.updateState)((s) => {
|
|
182
204
|
s.lastNightly = new Date().toISOString();
|
|
@@ -347,6 +369,13 @@ async function runNightly(options = {}) {
|
|
|
347
369
|
let batches = [];
|
|
348
370
|
let memoriesIngested = 0;
|
|
349
371
|
let hadTransientFailure = false;
|
|
372
|
+
// #287: distill tokens metered by the daemon across this run's segment
|
|
373
|
+
// POSTs (summed from the /distill responses by the capture loop).
|
|
374
|
+
// Forwarded to the dashboard snapshot so new_this_run.tokens is the run's
|
|
375
|
+
// TRUE total (distill + consolidation) and the distill share lands in
|
|
376
|
+
// tokens_by_stage. Zero when the daemon predates the usage field — the
|
|
377
|
+
// snapshot writer gates on total > 0, so old servers keep today's shape.
|
|
378
|
+
let distillUsage;
|
|
350
379
|
// consolidateOnly (hosted service): skip capture entirely. The hosted
|
|
351
380
|
// consolidation timer uses this so per-tenant nightly runs don't ingest the
|
|
352
381
|
// operator's local sessions into the tenant's DB.
|
|
@@ -423,6 +452,7 @@ async function runNightly(options = {}) {
|
|
|
423
452
|
sourceDomain: savedConfig?.sourceDomain,
|
|
424
453
|
});
|
|
425
454
|
memoriesIngested = result.memoriesIngested;
|
|
455
|
+
distillUsage = result.distillUsage;
|
|
426
456
|
// A 429/401 stop must hold the watermark too (fix 1): the loop abandoned
|
|
427
457
|
// the remaining sessions, and mtime discovery would never re-find them.
|
|
428
458
|
hadTransientFailure = result.hadTransientFailure || result.stopped !== undefined;
|
|
@@ -430,7 +460,12 @@ async function runNightly(options = {}) {
|
|
|
430
460
|
finally {
|
|
431
461
|
releaseLock();
|
|
432
462
|
}
|
|
433
|
-
console.log(`[hicortex] Capture complete: ${memoriesIngested} new memories`
|
|
463
|
+
console.log(`[hicortex] Capture complete: ${memoriesIngested} new memories` +
|
|
464
|
+
// #287: distill tokens the daemon metered for those segments (absent
|
|
465
|
+
// when the daemon predates the usage field or nothing distilled).
|
|
466
|
+
(distillUsage && distillUsage.total > 0
|
|
467
|
+
? ` · ${distillUsage.total.toLocaleString()} distill tokens`
|
|
468
|
+
: ""));
|
|
434
469
|
// Prune aged-out cursors (90d) — only on a clean run so a transient
|
|
435
470
|
// failure doesn't drop a still-needed cursor.
|
|
436
471
|
if (!dryRun && !hadTransientFailure) {
|
|
@@ -630,6 +665,53 @@ async function runNightly(options = {}) {
|
|
|
630
665
|
}
|
|
631
666
|
}
|
|
632
667
|
console.log(`[hicortex] Nightly pipeline complete.`);
|
|
668
|
+
// Backup stage (#6, Phase 0B) — a transactionally-consistent snapshot of
|
|
669
|
+
// the irreplaceable data (DB + identity + state), packaged as one tar.gz
|
|
670
|
+
// the operator ships offsite via the optional `backupCommand` hook. Runs
|
|
671
|
+
// ONLY on a full nightly (capture-only is frequent + stateless; dry-run
|
|
672
|
+
// writes nothing). Backup failure must NOT fail the nightly — capture +
|
|
673
|
+
// consolidation have already succeeded; the snapshot is on disk and the
|
|
674
|
+
// failure surfaces as `backupOk:false` in the dashboard snapshot + telemetry
|
|
675
|
+
// for alerting (the operator's hook owns active alerting; no in-product
|
|
676
|
+
// channel yet — Phase 3).
|
|
677
|
+
let backupPath;
|
|
678
|
+
let backupBytes;
|
|
679
|
+
let backupOk;
|
|
680
|
+
if (!dryRun && !captureOnly) {
|
|
681
|
+
try {
|
|
682
|
+
const backupDir = typeof savedConfig?.backupDir === "string" && savedConfig.backupDir.trim()
|
|
683
|
+
? savedConfig.backupDir
|
|
684
|
+
: undefined;
|
|
685
|
+
const bRes = await (0, backup_js_1.createBackup)({ db, home: stateDir, outDir: backupDir });
|
|
686
|
+
backupPath = bRes.path;
|
|
687
|
+
backupBytes = bRes.bytes;
|
|
688
|
+
backupOk = true;
|
|
689
|
+
console.log(`[hicortex] Backup: ${bRes.files} files, ${bRes.bytes.toLocaleString()} bytes -> ${bRes.path}`);
|
|
690
|
+
const cmd = typeof savedConfig?.backupCommand === "string" && savedConfig.backupCommand.trim()
|
|
691
|
+
? savedConfig.backupCommand
|
|
692
|
+
: undefined;
|
|
693
|
+
if (cmd && backupPath) {
|
|
694
|
+
const hook = await (0, backup_js_1.runBackupHook)(backupPath, cmd);
|
|
695
|
+
if (!hook.ok) {
|
|
696
|
+
// The artifact is on disk; only the offsite copy failed. Keep
|
|
697
|
+
// backupPath/backupBytes (the snapshot records what was produced)
|
|
698
|
+
// but flip backupOk so the aggregate can alert.
|
|
699
|
+
backupOk = false;
|
|
700
|
+
console.error(`[hicortex] Backup hook failed (exit ${hook.exitCode ?? "n/a"}). ` +
|
|
701
|
+
`Artifact is on disk; offsite copy did NOT complete.`);
|
|
702
|
+
}
|
|
703
|
+
else {
|
|
704
|
+
console.log(`[hicortex] Backup hook ok (exit 0).`);
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
}
|
|
708
|
+
catch (err) {
|
|
709
|
+
// The whole backup stage failed (snapshot, tar, or write). Do NOT
|
|
710
|
+
// propagate — capture/consolidation already succeeded. Surface + continue.
|
|
711
|
+
backupOk = false;
|
|
712
|
+
console.error(`[hicortex] Backup FAILED: ${err instanceof Error ? err.message : String(err)}`);
|
|
713
|
+
}
|
|
714
|
+
}
|
|
633
715
|
// Dashboard snapshot (#224) — full nightly only. The snapshot reflects
|
|
634
716
|
// corpus state regardless of whether consolidation/LLM ran, so it is
|
|
635
717
|
// ALWAYS written here (the use case is history; an LLM-less install still
|
|
@@ -673,6 +755,12 @@ async function runNightly(options = {}) {
|
|
|
673
755
|
// when consolidation didn't run or made no metered calls).
|
|
674
756
|
tokensThisRun,
|
|
675
757
|
tokensByStage,
|
|
758
|
+
// #287: the capture phase's distill tokens (from the /distill
|
|
759
|
+
// responses). The writer merges them into `tokens` +
|
|
760
|
+
// `tokens_by_stage.distill` so the customer-facing total is the
|
|
761
|
+
// run's TRUE spend; zero (old daemon / nothing distilled) is a no-op
|
|
762
|
+
// and the row keeps its consolidation-only shape.
|
|
763
|
+
distillUsage,
|
|
676
764
|
// #255 CR: always-on budget usage — undefined when consolidation
|
|
677
765
|
// didn't run (capture-only / no_llm / throttled). Forwarded whenever
|
|
678
766
|
// consolidation ran so the digest renders a continuous used/max bar.
|
|
@@ -682,6 +770,13 @@ async function runNightly(options = {}) {
|
|
|
682
770
|
// (capture-only / no_llm / throttled) or didn't exhaust.
|
|
683
771
|
budgetExhausted,
|
|
684
772
|
budgetDeferredByStage,
|
|
773
|
+
// #6 backup stage — hoisted from the block above. Present whenever
|
|
774
|
+
// the backup stage ran (full nightly); undefined on capture-only /
|
|
775
|
+
// dry-run. backupOk flips to false on snapshot OR hook failure so the
|
|
776
|
+
// dashboard digest can flag a night the offsite copy didn't complete.
|
|
777
|
+
backupPath,
|
|
778
|
+
backupBytes,
|
|
779
|
+
backupOk,
|
|
685
780
|
}, memorySoftCapResolved);
|
|
686
781
|
}
|
|
687
782
|
catch (snapErr) {
|
|
@@ -728,6 +823,13 @@ async function runNightly(options = {}) {
|
|
|
728
823
|
// exhausted (false is omitted to keep the ping minimal; the aggregate
|
|
729
824
|
// treats absent as "not exhausted / not measurable").
|
|
730
825
|
...(budgetExhausted ? { budget_exhausted: true } : {}),
|
|
826
|
+
// #6 backup stage outcome — forwarded only when the backup stage ran
|
|
827
|
+
// (full nightly). `ok` is false on snapshot OR hook failure; the fleet
|
|
828
|
+
// aggregate surfaces a sustained drop in backup_ok as a data-loss risk.
|
|
829
|
+
// Absent on capture-only / dry-run / client runs (no backup ran).
|
|
830
|
+
...(backupOk !== undefined
|
|
831
|
+
? { backup: { ok: backupOk === true, bytes: backupBytes ?? 0 } }
|
|
832
|
+
: {}),
|
|
731
833
|
sessions: batches.length,
|
|
732
834
|
ok: !hadTransientFailure,
|
|
733
835
|
shown: adoption.shown,
|
package/dist/prompts.js
CHANGED
|
@@ -120,6 +120,7 @@ EXTRACT into this markdown format:
|
|
|
120
120
|
|
|
121
121
|
### Decisions Made
|
|
122
122
|
- [D] [SUBJECT]: [decision] — [reasoning] (${date})
|
|
123
|
+
(ONLY decisions the user explicitly made or confirmed — never an AI proposal)
|
|
123
124
|
|
|
124
125
|
### Knowledge Learned
|
|
125
126
|
- [K] [SUBJECT]: [knowledge] — [context/source] (${date})
|
|
@@ -145,10 +146,19 @@ TYPE TAG (critical — prefix EVERY bullet with exactly one letter + space):
|
|
|
145
146
|
- [K] KNOWLEDGE — a durable truth that will hold across sessions: "the API is at
|
|
146
147
|
:8787", "uv is used for packages", "config lives in ~/.hicortex/". Not tied
|
|
147
148
|
to a single moment.
|
|
148
|
-
- [D] DECISIONS — a choice
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
149
|
+
- [D] DECISIONS — a choice the USER explicitly made or confirmed in the
|
|
150
|
+
transcript: they said "do X / agreed / ok / go ahead", approved the plan, or
|
|
151
|
+
the transcript shows the change actually being carried out. It must be a
|
|
152
|
+
choice future work builds on and that a later decision can SUPERSEDE:
|
|
153
|
+
"switched from gemma4 to qwen3.5", "adopted the graded-schema tag model".
|
|
154
|
+
Not knowledge (it can change) and not experience (it persists and constrains).
|
|
155
|
+
- An AI recommendation or proposal is NEVER a decision, however detailed or
|
|
156
|
+
well-reasoned — even if the user seemed receptive. If the user declined,
|
|
157
|
+
deferred ("hold", "later", "wait for X"), or did not answer: record it as
|
|
158
|
+
[E] EXPERIENCE with proposal framing ("AI proposed X → user declined/held
|
|
159
|
+
because Y"), or under Corrections & Rejections.
|
|
160
|
+
- SELF-CHECK: if an item's text says "AI recommended/proposed" or the user
|
|
161
|
+
"has not (yet) confirmed" it, that item is NOT [D] — re-tag it [E].
|
|
152
162
|
- NEVER use [L] (learnings). Learnings are extracted by a SEPARATE reflection stage,
|
|
153
163
|
not here. If the model emits [L], it is wrong — re-tag as experience/knowledge/decisions.
|
|
154
164
|
The type tag goes BEFORE the subject, never as a section/category bracket.
|
|
@@ -161,6 +171,8 @@ entity. The subject is what a future reader would search for.
|
|
|
161
171
|
- NOT: "[E] User rejected AI's bundling of unknown loads"
|
|
162
172
|
- Write: "[K] Nightly capture (Hermes): cron sessions are excluded — source='cron' is skipped before distillation"
|
|
163
173
|
- NOT: "[K] Discovered that cron sessions are filtered out"
|
|
174
|
+
- Write: "[E] Qwen3.8-27B swap: AI proposed switching → user held on Qwen3.6-35B-A3B until an MoE variant ships"
|
|
175
|
+
- NOT: "[D] Qwen3.8-27B: switch from Qwen3.6-27B — drop-in upgrade"
|
|
164
176
|
Reason: each item's first words (after the type tag) become the memory's one-line
|
|
165
177
|
index entry AND dominate its search embedding. An item that opens with a category
|
|
166
178
|
label, a sentiment ("Strong Negative"), or "User rejected…" is unfindable — it
|
|
@@ -169,6 +181,9 @@ the subject; put reaction, intensity and reasoning AFTER it.
|
|
|
169
181
|
|
|
170
182
|
RULES:
|
|
171
183
|
- Extract MAX 20 items total (quality over quantity)
|
|
184
|
+
- Use EXACT names/versions/paths/numbers as they appear in the transcript —
|
|
185
|
+
never substitute what seems current or more standard (writing "Qwen3.6-27B"
|
|
186
|
+
when the transcript says "Qwen3.6-35B-A3B" is a fabrication)
|
|
172
187
|
- Each must be useful if recalled in a future session
|
|
173
188
|
- Skip: routine code edits, standard tool usage, trivial fixes
|
|
174
189
|
- Include: architectural decisions, debugging breakthroughs, user preferences,
|
package/dist/telemetry.d.ts
CHANGED
|
@@ -116,6 +116,20 @@ export interface TelemetryPayload {
|
|
|
116
116
|
* the dashboard snapshot, not on the wire (mirrors `tokens_this_run`).
|
|
117
117
|
*/
|
|
118
118
|
budget_exhausted?: boolean;
|
|
119
|
+
/**
|
|
120
|
+
* Backup stage outcome (#6, Phase 0B). Present on every full nightly (absent
|
|
121
|
+
* on capture-only / dry-run / client runs — no backup stage runs there).
|
|
122
|
+
* `ok` is false when EITHER the snapshot write OR the operator's
|
|
123
|
+
* `backupCommand` hook failed — the aggregate data-loss-risk signal (a
|
|
124
|
+
* sustained drop in `backup.ok` means offsite copies are silently not
|
|
125
|
+
* landing). `bytes` is the compressed artifact size (0 when the snapshot
|
|
126
|
+
* itself failed before producing a file). The operator's hook owns active
|
|
127
|
+
* alerting (email/Discord); this field is the passive fleet-health signal.
|
|
128
|
+
*/
|
|
129
|
+
backup?: {
|
|
130
|
+
ok: boolean;
|
|
131
|
+
bytes: number;
|
|
132
|
+
};
|
|
119
133
|
}
|
|
120
134
|
/**
|
|
121
135
|
* Check if telemetry is enabled. Disabled by:
|