@gamaze/hicortex 0.17.0 → 0.17.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -218,8 +218,10 @@ Config at `~/.hicortex/config.json`. Created by `init`. Key options:
218
218
  | `captureCooldownHours` | Success-cooldown (hours) for the **capture watchdog** (0.17). The capture timer polls every ~20 min; the watchdog captures only if more than this has elapsed since the last *successful* capture (`state.lastNightly`). Default `6` (≈4 captures/day). A failed preflight retries on the next poll (~20 min) — so a transient fire-instant network miss costs minutes, not a day (#239) |
219
219
  | `consolidationHours` | Hours (0–23, local) for the **consolidation** timer — the full nightly (capture + distill + score + reflect + link). Installed for **server/co-located only** (clients have no local DB). Default `[10, 22]`: the 22:00 evening slot runs after the day's capture waves (same-day results); the 10:00 morning slot runs *after* the morning capture so wake-up pushes are caught. Omitted on clients |
220
220
  | `consolidateMaxLlmCalls` | Ceiling on total LLM calls across all classify-tier consolidation stages (content-domain, link discovery, supersession) per run. A runaway **backstop**, not a throughput throttle — on a free local model the binding constraint is the nightly unit's wall-clock timeout, not call count. Default `5000` (was a hard-coded 200 that starved link/supersession during a classification backlog) |
221
+ | `memorySoftCap` | Soft cap on the memory corpus (default 10000). When the corpus exceeds this, the nightly's capacity-eviction stage removes the lowest-`effectiveStrength` memories (ties broken by oldest access) until under the cap — the active forgetting mechanism that bounds DB size, vector-index RAM, and consolidation workload. `0` disables eviction (indefinite growth — the pre-#245 behaviour). The evicted tail is cold by construction (effectiveStrength is the same decay-weighted score the recall ranker uses, so these were not surfacing in the top-k anyway). At 10K memories the load + JS sort is <100 ms |
222
+ | `updateChannel` | Release channel pinned into the generated daemon/timer ExecStart for **npx-thin** installs (global-binary installs use the absolute binary and are unaffected). A dist-tag (`"rc"`, `"next"`) or an exact version (`"0.17.1"`). E.g. `"rc"` → the timer runs `npx -y @gamaze/hicortex@rc nightly`, so the host tracks the rc dist-tag (an internal fleet can ride rc through a pre-promotion soak). Validated as `[\w.\-]+` (rejects anything that'd break the unit/plist templates). Absent → auto-detect (bare on `latest`, else `@next`). (0.17.1) |
221
223
  | `nightlyHour` | **Deprecated (0.17) single-slot fallback.** Local hour (0–23) honoured only when `consolidationHours` is absent — yields one daily consolidation slot at that hour (preserves the pre-0.17 "one daily job" intent). New installs should use `consolidationHours` |
222
- | `preflightTimeoutMs` | **Client mode only.** Per-attempt timeout for the nightly's server-reachability check before it starts capturing (default: 20000 ms, bumped from 15000 in 0.17 to absorb a cold Tailscale handshake on a roaming laptop) |
224
+ | `preflightTimeoutMs` | **Client mode only.** Per-attempt timeout for the nightly's server-reachability check before it starts capturing (default: 20000 ms, bumped from 15000 in 0.17 to absorb a slow link re-establishing after the client wakes) |
223
225
  | `preflightAttempts` | **Client mode only.** Reachability-check retries before the nightly aborts (default: 3; floored at 1). `1` = single try, no retry |
224
226
  | `preflightRetryGapMs` | **Client mode only.** Delay between reachability retries (default: 60000 ms). Note: timers don't advance while the machine is asleep, so on a sleeping laptop this gap counts awake-time, not wall-clock |
225
227
  | `scoreSimilarityWeight` | Weight of semantic similarity in the ranking score (default: 0.50) |
@@ -147,6 +147,8 @@
147
147
  <div id="digest-lessons"></div>
148
148
  <h2 style="margin-top:16px;">Dedup merges</h2>
149
149
  <div id="digest-dedup"></div>
150
+ <h2 id="digest-tokens-section" style="margin-top:16px;display:none;">Token usage breakdown by stage</h2>
151
+ <div id="digest-tokens"></div>
150
152
  </div>
151
153
 
152
154
  <div class="sub muted" style="margin-top:24px;">
@@ -205,13 +207,71 @@ function renderHeadline(h) {
205
207
  const ups = h.uses_per_showing === null || h.uses_per_showing === undefined
206
208
  ? "—"
207
209
  : (h.uses_per_showing * 100).toFixed(1) + "%";
210
+ // Capacity gauge (#245): X / cap memories. Amber at ≥85% of cap, red at ≥100%.
211
+ // Hidden when the cap is 0 (disabled — indefinite growth, the pre-#245 default).
212
+ let capStat = "";
213
+ if (h.memory_soft_cap && h.memory_soft_cap > 0) {
214
+ const cap = h.memory_soft_cap;
215
+ const n = h.total_memories;
216
+ const pct = Math.min(100, (n / cap) * 100);
217
+ const over = n >= cap;
218
+ const near = pct >= 85 && !over;
219
+ const barColor = over ? "var(--bad, #e05555)" : near ? "var(--warn, #e0b055)" : "var(--ok, #61c98f)";
220
+ capStat = `
221
+ <div class="stat" style="flex-basis:240px;">
222
+ <div class="v" style="font-size:18px;">${n.toLocaleString()} <span class="muted" style="font-size:13px;">/ ${cap.toLocaleString()}</span></div>
223
+ <div class="l">Memories vs soft cap ${over ? "(over cap)" : ""}</div>
224
+ <div style="margin-top:6px;height:6px;background:rgba(128,128,128,0.2);border-radius:3px;overflow:hidden;">
225
+ <div style="width:${pct.toFixed(1)}%;height:100%;background:${barColor};"></div>
226
+ </div>
227
+ </div>`;
228
+ }
229
+ // LLM tokens this period (#246): hidden when `used` is 0 (no metered run yet
230
+ // — nothing to show). When capped (cap > 0), render a usage bar with the
231
+ // percentage; amber at ≥85%, red at ≥100%. When uncapped (self-hosted
232
+ // default), show just the count. Both shapes include "this month" so the
233
+ // billing-period framing is uniform.
234
+ let tokenStat = "";
235
+ const tok = h.tokens;
236
+ if (tok && tok.used > 0) {
237
+ const usedStr = formatTokens(tok.used);
238
+ if (tok.cap > 0) {
239
+ const pct = Math.min(100, (tok.used / tok.cap) * 100);
240
+ const over = tok.used >= tok.cap;
241
+ const near = pct >= 85 && !over;
242
+ const barColor = over ? "var(--bad, #e05555)" : near ? "var(--warn, #e0b055)" : "var(--ok, #61c98f)";
243
+ tokenStat = `
244
+ <div class="stat" style="flex-basis:240px;">
245
+ <div class="v" style="font-size:18px;">${escapeHtml(usedStr)} <span class="muted" style="font-size:13px;">/ ${escapeHtml(formatTokens(tok.cap))}</span></div>
246
+ <div class="l">LLM tokens this month ${over ? "(over cap)" : ""}</div>
247
+ <div style="margin-top:6px;height:6px;background:rgba(128,128,128,0.2);border-radius:3px;overflow:hidden;">
248
+ <div style="width:${pct.toFixed(1)}%;height:100%;background:${barColor};"></div>
249
+ </div>
250
+ </div>`;
251
+ } else {
252
+ tokenStat = `
253
+ <div class="stat">
254
+ <div class="v">${escapeHtml(usedStr)}</div>
255
+ <div class="l">LLM tokens this month</div>
256
+ </div>`;
257
+ }
258
+ }
208
259
  $("headline").innerHTML = `
209
260
  <div class="stat"><div class="v">${h.total_memories}</div><div class="l">Total memories</div></div>
210
261
  <div class="stat"><div class="v">${ups}</div><div class="l">Uses per showing (headline)</div></div>
211
262
  <div class="stat"><div class="v warn">${h.cold_count}</div><div class="l">Cold long-tail (never shown/used)</div></div>
263
+ ${capStat}
264
+ ${tokenStat}
212
265
  `;
213
266
  }
214
267
 
268
+ /** Compact human-readable token count: 1234 → "1.2K", 1_500_000 → "1.5M". */
269
+ function formatTokens(n) {
270
+ if (n >= 1e6) return (n / 1e6).toFixed(1) + "M";
271
+ if (n >= 1e3) return (n / 1e3).toFixed(1) + "K";
272
+ return String(n);
273
+ }
274
+
215
275
  // ---------------------------------------------------------------------------
216
276
  // Render: growth chart (hand-rolled inline SVG line chart)
217
277
  //
@@ -348,8 +408,27 @@ function renderDigest(digest) {
348
408
  s.lessonsGenerated !== undefined ? `<span class="stage-pill">lessons: ${s.lessonsGenerated}</span>` : "",
349
409
  `<span class="stage-pill">dedup: ${s.dedup}</span>`,
350
410
  `<span class="stage-pill">supersession: ${s.supersession}</span>`,
411
+ s.evicted !== undefined && s.evicted > 0 ? `<span class="stage-pill warn">evicted: ${s.evicted}</span>` : "",
412
+ s.tokens !== undefined ? `<span class="stage-pill">tokens: ${escapeHtml(formatTokens(s.tokens))}</span>` : "",
351
413
  ].join(" ");
352
414
  $("digest-stages").innerHTML = stages;
415
+ // Per-stage token breakdown (#246) — only when the day's run metered tokens
416
+ // and the breakdown was carried (always present together with `tokens`).
417
+ const tbs = s.tokens_by_stage;
418
+ if (tbs && Object.keys(tbs).length > 0) {
419
+ const rows = Object.entries(tbs)
420
+ .sort((a, b) => (b[1].total - a[1].total))
421
+ .map(([k, v]) =>
422
+ `<tr><td class="k">${escapeHtml(k)}</td>` +
423
+ `<td>${escapeHtml(formatTokens(v.total))}</td>` +
424
+ `<td class="muted">${escapeHtml(formatTokens(v.prompt))} in / ${escapeHtml(formatTokens(v.completion))} out</td></tr>`
425
+ ).join("");
426
+ $("digest-tokens").innerHTML = `<table class="digest"><tbody>${rows}</tbody></table>`;
427
+ $("digest-tokens-section").style.display = "";
428
+ } else {
429
+ $("digest-tokens").innerHTML = `<div class="empty">No token usage that day (no metered LLM run).</div>`;
430
+ $("digest-tokens-section").style.display = "none";
431
+ }
353
432
 
354
433
  if (digest.sample.length === 0) {
355
434
  $("digest-sample").innerHTML = `<div class="empty">No new memories that day.</div>`;
package/dist/cli.d.ts CHANGED
@@ -6,8 +6,9 @@
6
6
  * server Start the MCP HTTP/SSE server (persistent daemon)
7
7
  * init Detect existing setup and configure for CC/OC
8
8
  * nightly Run capture + consolidate (manual trigger)
9
- * nightly --capture-only Capture only, skip consolidation
10
- * nightly --status Show nightly pipeline health check
9
+ * nightly --capture-only Capture only, skip consolidation
10
+ * nightly --consolidate-only Consolidate only, skip capture (hosted service)
11
+ * nightly --status Show nightly pipeline health check
11
12
  * relink Resumable link-discovery pass over the entire corpus (issue #143)
12
13
  * dedup Cluster + merge near-duplicate memories (issue #100)
13
14
  * dedup --apply Execute the merge (default: dry run)
package/dist/cli.js CHANGED
@@ -7,8 +7,9 @@
7
7
  * server Start the MCP HTTP/SSE server (persistent daemon)
8
8
  * init Detect existing setup and configure for CC/OC
9
9
  * nightly Run capture + consolidate (manual trigger)
10
- * nightly --capture-only Capture only, skip consolidation
11
- * nightly --status Show nightly pipeline health check
10
+ * nightly --capture-only Capture only, skip consolidation
11
+ * nightly --consolidate-only Consolidate only, skip capture (hosted service)
12
+ * nightly --status Show nightly pipeline health check
12
13
  * relink Resumable link-discovery pass over the entire corpus (issue #143)
13
14
  * dedup Cluster + merge near-duplicate memories (issue #100)
14
15
  * dedup --apply Execute the merge (default: dry run)
@@ -69,7 +70,12 @@ switch (command) {
69
70
  else {
70
71
  const dryRun = args.includes("--dry-run");
71
72
  const captureOnly = args.includes("--capture-only");
73
+ const consolidateOnly = args.includes("--consolidate-only");
72
74
  const watchdog = args.includes("--watchdog");
75
+ if (captureOnly && consolidateOnly) {
76
+ console.error("[hicortex] nightly: --capture-only and --consolidate-only are mutually exclusive");
77
+ process.exit(1);
78
+ }
73
79
  // Timestamp every log line. The nightly writes to a file (launchd /
74
80
  // systemd StandardOutput append) with NO per-line timestamp, which made
75
81
  // diagnosing capture gaps impossible (the #239 investigation couldn't
@@ -92,7 +98,7 @@ switch (command) {
92
98
  }
93
99
  }
94
100
  import("./nightly.js").then(({ runNightly }) => {
95
- runNightly({ dryRun, captureOnly, watchdog, recaptureWindowDays }).catch((err) => {
101
+ runNightly({ dryRun, captureOnly, consolidateOnly, watchdog, recaptureWindowDays }).catch((err) => {
96
102
  console.error("[hicortex] Nightly pipeline failed:", err);
97
103
  process.exit(1);
98
104
  });
@@ -285,6 +291,7 @@ Options:
285
291
  server --host <h> Host (default: 127.0.0.1)
286
292
  nightly --dry-run Preview without changes
287
293
  nightly --capture-only Capture only, skip consolidation (safe to run multiple times/day)
294
+ nightly --consolidate-only Consolidate only, skip capture (hosted-service per-tenant runs)
288
295
  nightly --recapture-window <days> Re-discover sessions quiet since <days> ago (one-shot #189 recovery)
289
296
  nightly --status Show nightly pipeline health
290
297
  relink --dry-run Discovery + counts only, zero writes, cursor untouched
@@ -63,12 +63,63 @@ export declare class BudgetTracker {
63
63
  maxCalls: number;
64
64
  callsUsed: number;
65
65
  callsByStage: Record<string, number>;
66
+ /**
67
+ * Token usage per stage (#246). Keys are the same stage labels passed to
68
+ * `use()`. A stage that made no metered calls (no usage returned — never the
69
+ * path on a healthy openai/ollama endpoint) is absent, NOT zero, so the
70
+ * dashboard can distinguish "nothing spent" from "no signal".
71
+ */
72
+ tokensByStage: Record<string, {
73
+ prompt: number;
74
+ completion: number;
75
+ total: number;
76
+ }>;
77
+ /** Run-wide totals — the sum of every recordUsage() call this run. */
78
+ totalTokens: {
79
+ prompt: number;
80
+ completion: number;
81
+ total: number;
82
+ };
66
83
  constructor(maxCalls: number);
67
84
  get exhausted(): boolean;
68
85
  get remaining(): number;
69
86
  use(stage: string, count?: number): boolean;
87
+ /**
88
+ * Record token usage from one LLM call (#246). Called by the consolidation
89
+ * stages after each metered completion. `undefined` usage (claude-cli path,
90
+ * or a non-conforming endpoint that returned no usage object) is a no-op —
91
+ * never recorded as zero, which would silently undercount real spend.
92
+ */
93
+ recordUsage(stage: string, usage: {
94
+ prompt_tokens: number;
95
+ completion_tokens: number;
96
+ total_tokens: number;
97
+ } | undefined): void;
70
98
  summary(): NonNullable<ConsolidationReport["budget"]>;
71
99
  }
100
+ /**
101
+ * Decide whether consolidation should be throttled this run based on the
102
+ * `llmTokensPerMonth` fair-use cap. Pure (no I/O) so it can be unit-tested
103
+ * independently of the nightly wiring.
104
+ *
105
+ * Returns `{ throttle: true, used, cap }` when the projected post-run total
106
+ * would exceed the cap; `{ throttle: false }` otherwise. The estimate is the
107
+ * previous run's actual usage (`llmTokensLastRun`, 0/absent on the first
108
+ * metered run = never throttle the first run — no baseline yet).
109
+ *
110
+ * `cap = 0` (the self-hosted default) → never throttle (unlimited).
111
+ * `periodStart` in a previous calendar month → period resets to 0 first
112
+ * (mirrors the reset logic in nightly.ts; both sides agree because both read
113
+ * the same state + clock).
114
+ */
115
+ export declare function shouldThrottleTokens(cap: number, period: {
116
+ total: number;
117
+ periodStart: string;
118
+ } | undefined, lastRunTokens: number, now?: Date): {
119
+ throttle: boolean;
120
+ used?: number;
121
+ cap?: number;
122
+ };
72
123
  /**
73
124
  * Parse JSON from LLM output, tolerating markdown fences and indexed formats.
74
125
  */
@@ -203,6 +254,17 @@ export declare function stageDecayPrune(db: Database.Database, dryRun: boolean):
203
254
  pruned: number;
204
255
  failed: number;
205
256
  };
257
+ /**
258
+ * Default soft cap on the memory corpus (#245). Above this the lowest-value
259
+ * memories are evicted each nightly. 10000 balances headroom for a busy
260
+ * self-hosted install against the noise cost of a bloated vector index
261
+ * (recall top-k competes against the long tail). Override via `memorySoftCap`.
262
+ */
263
+ export declare const DEFAULT_MEMORY_SOFT_CAP = 10000;
264
+ export declare function stageMemoryCapEviction(db: Database.Database, dryRun: boolean, cap: number): {
265
+ cap: number;
266
+ evicted: number;
267
+ };
206
268
  /**
207
269
  * Run the full consolidation pipeline. Returns a structured report.
208
270
  */
@@ -232,7 +294,11 @@ export declare function runConsolidation(db: Database.Database, llm: LlmClient,
232
294
  /** Total LLM-call ceiling across classify-tier stages (#241). The caller
233
295
  * reads `consolidateMaxLlmCalls` from config and passes it; unset → the
234
296
  * exported `CONSOLIDATE_MAX_LLM_CALLS` default (5000). */
235
- budgetMaxCalls?: number): Promise<ConsolidationReport>;
297
+ budgetMaxCalls?: number,
298
+ /** Soft cap on the corpus (#245). Nightly.ts reads `memorySoftCap` from
299
+ * config and passes it; unset → `DEFAULT_MEMORY_SOFT_CAP` (10000). `0`
300
+ * disables eviction (indefinite growth). */
301
+ memorySoftCap?: number): Promise<ConsolidationReport>;
236
302
  /**
237
303
  * Calculate milliseconds until the next occurrence of a given hour (local time).
238
304
  */
@@ -38,8 +38,9 @@ var __importStar = (this && this.__importStar) || (function () {
38
38
  };
39
39
  })();
40
40
  Object.defineProperty(exports, "__esModule", { value: true });
41
- exports.DEFAULT_SUPERSESSION_MAX_CALLS = exports.DEFAULT_SUPERSESSION_MIN_SIMILARITY = exports.BudgetTracker = exports.REFLECTION_CONTRADICTION_MIN_COSINE = exports.l2ToCosine = exports.CROSS_PROJECT_LINK_THRESHOLD = exports.CONSOLIDATE_LINK_TOP_K = exports.CONSOLIDATE_LINK_THRESHOLD = exports.CONSOLIDATE_MAX_LLM_CALLS = void 0;
41
+ exports.DEFAULT_MEMORY_SOFT_CAP = exports.DEFAULT_SUPERSESSION_MAX_CALLS = exports.DEFAULT_SUPERSESSION_MIN_SIMILARITY = exports.BudgetTracker = exports.REFLECTION_CONTRADICTION_MIN_COSINE = exports.l2ToCosine = exports.CROSS_PROJECT_LINK_THRESHOLD = exports.CONSOLIDATE_LINK_TOP_K = exports.CONSOLIDATE_LINK_THRESHOLD = exports.CONSOLIDATE_MAX_LLM_CALLS = void 0;
42
42
  exports.isContradictionCandidate = isContradictionCandidate;
43
+ exports.shouldThrottleTokens = shouldThrottleTokens;
43
44
  exports.parseJsonLenient = parseJsonLenient;
44
45
  exports.rebuildContentModuleIndex = rebuildContentModuleIndex;
45
46
  exports.discoverLinkCandidates = discoverLinkCandidates;
@@ -49,6 +50,7 @@ exports.buildSupersessionPrompt = buildSupersessionPrompt;
49
50
  exports.parseSupersessionReply = parseSupersessionReply;
50
51
  exports.stageSupersession = stageSupersession;
51
52
  exports.stageDecayPrune = stageDecayPrune;
53
+ exports.stageMemoryCapEviction = stageMemoryCapEviction;
52
54
  exports.runConsolidation = runConsolidation;
53
55
  exports.msUntilHour = msUntilHour;
54
56
  exports.scheduleConsolidation = scheduleConsolidation;
@@ -125,6 +127,19 @@ class BudgetTracker {
125
127
  maxCalls;
126
128
  callsUsed = 0;
127
129
  callsByStage = {};
130
+ /**
131
+ * Token usage per stage (#246). Keys are the same stage labels passed to
132
+ * `use()`. A stage that made no metered calls (no usage returned — never the
133
+ * path on a healthy openai/ollama endpoint) is absent, NOT zero, so the
134
+ * dashboard can distinguish "nothing spent" from "no signal".
135
+ */
136
+ tokensByStage = {};
137
+ /** Run-wide totals — the sum of every recordUsage() call this run. */
138
+ totalTokens = {
139
+ prompt: 0,
140
+ completion: 0,
141
+ total: 0,
142
+ };
128
143
  constructor(maxCalls) {
129
144
  this.maxCalls = maxCalls;
130
145
  }
@@ -144,17 +159,73 @@ class BudgetTracker {
144
159
  this.callsByStage[stage] = (this.callsByStage[stage] ?? 0) + count;
145
160
  return true;
146
161
  }
162
+ /**
163
+ * Record token usage from one LLM call (#246). Called by the consolidation
164
+ * stages after each metered completion. `undefined` usage (claude-cli path,
165
+ * or a non-conforming endpoint that returned no usage object) is a no-op —
166
+ * never recorded as zero, which would silently undercount real spend.
167
+ */
168
+ recordUsage(stage, usage) {
169
+ if (!usage)
170
+ return;
171
+ const cur = this.tokensByStage[stage] ?? { prompt: 0, completion: 0, total: 0 };
172
+ cur.prompt += usage.prompt_tokens;
173
+ cur.completion += usage.completion_tokens;
174
+ cur.total += usage.total_tokens;
175
+ this.tokensByStage[stage] = cur;
176
+ this.totalTokens.prompt += usage.prompt_tokens;
177
+ this.totalTokens.completion += usage.completion_tokens;
178
+ this.totalTokens.total += usage.total_tokens;
179
+ }
147
180
  summary() {
148
181
  return {
149
182
  max_calls: this.maxCalls,
150
183
  calls_used: this.callsUsed,
151
184
  calls_remaining: this.remaining,
152
185
  calls_by_stage: { ...this.callsByStage },
186
+ tokens_by_stage: Object.fromEntries(Object.entries(this.tokensByStage).map(([k, v]) => [k, { ...v }])),
187
+ tokens_total: { ...this.totalTokens },
153
188
  };
154
189
  }
155
190
  }
156
191
  exports.BudgetTracker = BudgetTracker;
157
192
  // ---------------------------------------------------------------------------
193
+ // Token fair-use throttle decision (#246)
194
+ // ---------------------------------------------------------------------------
195
+ /**
196
+ * Decide whether consolidation should be throttled this run based on the
197
+ * `llmTokensPerMonth` fair-use cap. Pure (no I/O) so it can be unit-tested
198
+ * independently of the nightly wiring.
199
+ *
200
+ * Returns `{ throttle: true, used, cap }` when the projected post-run total
201
+ * would exceed the cap; `{ throttle: false }` otherwise. The estimate is the
202
+ * previous run's actual usage (`llmTokensLastRun`, 0/absent on the first
203
+ * metered run = never throttle the first run — no baseline yet).
204
+ *
205
+ * `cap = 0` (the self-hosted default) → never throttle (unlimited).
206
+ * `periodStart` in a previous calendar month → period resets to 0 first
207
+ * (mirrors the reset logic in nightly.ts; both sides agree because both read
208
+ * the same state + clock).
209
+ */
210
+ function shouldThrottleTokens(cap, period, lastRunTokens, now = new Date()) {
211
+ if (cap <= 0)
212
+ return { throttle: false };
213
+ let periodTotal = period?.total ?? 0;
214
+ const periodStart = period?.periodStart;
215
+ if (periodStart) {
216
+ const start = new Date(periodStart);
217
+ if (start.getUTCFullYear() !== now.getUTCFullYear() ||
218
+ start.getUTCMonth() !== now.getUTCMonth()) {
219
+ // Stale period → reset accrual to 0 before the check.
220
+ periodTotal = 0;
221
+ }
222
+ }
223
+ if (periodTotal + lastRunTokens > cap) {
224
+ return { throttle: true, used: periodTotal, cap };
225
+ }
226
+ return { throttle: false };
227
+ }
228
+ // ---------------------------------------------------------------------------
158
229
  // JSON parsing helper
159
230
  // ---------------------------------------------------------------------------
160
231
  /**
@@ -239,8 +310,9 @@ async function stageImportance(db, memories, llm, budget, dryRun) {
239
310
  break;
240
311
  }
241
312
  try {
242
- const raw = await llm.completeFast(prompt, 256);
243
- let scores = parseJsonLenient(raw, null);
313
+ const r = await llm.completeFast(prompt, 256);
314
+ budget.recordUsage("importance", r.usage);
315
+ let scores = parseJsonLenient(r.text, null);
244
316
  if (!Array.isArray(scores)) {
245
317
  scores = new Array(batch.length).fill(0.5);
246
318
  }
@@ -300,8 +372,9 @@ async function stageReflection(db, memories, llm, budget, embedFn, dryRun) {
300
372
  return { lessons_generated: 0, skipped: true, reason: "budget_exhausted" };
301
373
  }
302
374
  try {
303
- const raw = await llm.completeReflect(prompt, 2048);
304
- const lessons = parseJsonLenient(raw, []);
375
+ const r = await llm.completeReflect(prompt, 2048);
376
+ budget.recordUsage("reflection", r.usage);
377
+ const lessons = parseJsonLenient(r.text, []);
305
378
  if (!Array.isArray(lessons)) {
306
379
  return { lessons_generated: 0, failed: true };
307
380
  }
@@ -349,9 +422,14 @@ async function stageReflection(db, memories, llm, budget, embedFn, dryRun) {
349
422
  const existingText = similarLessons[0].content.slice(0, 300);
350
423
  const newText = content.slice(0, 300);
351
424
  try {
352
- const verdict = await llm.completeFast(`Two lessons from an AI memory system. Do they CONTRADICT each other (opposite advice on the same topic)?\n\n` +
425
+ const verdictR = await llm.completeFast(`Two lessons from an AI memory system. Do they CONTRADICT each other (opposite advice on the same topic)?\n\n` +
353
426
  `EXISTING: ${existingText}\n\nNEW: ${newText}\n\n` +
354
427
  `Answer ONLY "yes" or "no". If the new lesson updates/refines the existing one (not contradicts), answer "no".`, 16);
428
+ // Stage label "contradiction_check" matches the budget.use() call
429
+ // above (separate counter from the reflection call proper). Token
430
+ // accounting follows the same stage partition as the call counter.
431
+ budget.recordUsage("contradiction_check", verdictR.usage);
432
+ const verdict = verdictR.text;
355
433
  if (verdict.toLowerCase().trim().startsWith("yes")) {
356
434
  contradicted = true;
357
435
  console.log(`[hicortex] Lesson suppressed (contradicts existing): "${lessonText.slice(0, 80)}"`);
@@ -462,7 +540,10 @@ async function stageContentDomains(db, domains, llm, budget, embedFn, dryRun, st
462
540
  // classifyMemoryTags returns null ONLY on infra error (throws after retry) —
463
541
  // skip that memory, leaving domain/tags/strength untouched so a later
464
542
  // run retries it (issue #150: never file or decay on infra errors).
465
- const result = await (0, domain_classify_js_1.classifyMemoryTags)(row.content, row.project, domains, llm);
543
+ // The onUsage callback (#246) wires the metered call's token accounting
544
+ // into this stage's BudgetTracker slot — same stage label the budget.use
545
+ // call above uses, so call count + tokens stay aligned.
546
+ const result = await (0, domain_classify_js_1.classifyMemoryTags)(row.content, row.project, domains, llm, (u) => budget.recordUsage("content_domain", u));
466
547
  if (result === null) {
467
548
  console.warn(`[hicortex] content-domain: infra error classifying ${row.id} — skipped (will retry)`);
468
549
  continue;
@@ -599,8 +680,9 @@ async function stageDomainCuration(db, llm, budget, dryRun, stateDir) {
599
680
  .map((r) => `${r.project}: ${r.cnt} / ${lessonsByProject.get(r.project) ?? 0}`)
600
681
  .join("\n");
601
682
  try {
602
- const raw = await llm.completeFast((0, prompts_js_1.domainCuration)(projectLines), 1024);
603
- const parsed = parseJsonLenient(raw, []);
683
+ const r = await llm.completeFast((0, prompts_js_1.domainCuration)(projectLines), 1024);
684
+ budget.recordUsage("domain_curation", r.usage);
685
+ const parsed = parseJsonLenient(r.text, []);
604
686
  if (!Array.isArray(parsed) || parsed.length === 0) {
605
687
  console.warn("[hicortex] Domain curation: LLM returned empty/invalid response, using fallback");
606
688
  domains = projectRows.map((r) => ({
@@ -934,16 +1016,19 @@ function parseSupersessionReply(reply) {
934
1016
  }
935
1017
  /**
936
1018
  * ONE classify-tier LLM call judging whether `newContent` supersedes
937
- * `oldContent`. Returns null on any infra error or unparseable reply — the
938
- * caller treats null as "skip this pair" (never mis-links on ambiguity).
1019
+ * `oldContent`. Returns `{verdict, usage}` — verdict is null on any infra error
1020
+ * or unparseable reply (the caller treats null as "skip this pair", never
1021
+ * mis-links on ambiguity). `usage` is the call's token accounting (#246),
1022
+ * surfaced even on a null verdict so the BudgetTracker still meters a
1023
+ * network-round-tripped attempt (the cost is real even if the parse failed).
939
1024
  */
940
1025
  async function classifySupersession(llm, oldContent, newContent) {
941
1026
  try {
942
- const raw = await llm.completeClassify(buildSupersessionPrompt(oldContent, newContent), 32);
943
- return parseSupersessionReply(raw);
1027
+ const r = await llm.completeClassify(buildSupersessionPrompt(oldContent, newContent), 32);
1028
+ return { verdict: parseSupersessionReply(r.text), usage: r.usage };
944
1029
  }
945
1030
  catch {
946
- return null;
1031
+ return { verdict: null, usage: undefined };
947
1032
  }
948
1033
  }
949
1034
  /**
@@ -1033,7 +1118,10 @@ async function stageSupersession(db, llm, budget, embedFn, dryRun, stateDir, opt
1033
1118
  if (callsUsed >= maxCalls || !budget.use("supersession"))
1034
1119
  break;
1035
1120
  callsUsed++;
1036
- const verdict = await classifySupersession(llm, neighbor.content, candidate.content);
1121
+ const { verdict, usage } = await classifySupersession(llm, neighbor.content, candidate.content);
1122
+ // Meter every round-tripped attempt (#246) — even a null verdict spent
1123
+ // real tokens. The stage label matches the budget.use() above.
1124
+ budget.recordUsage("supersession", usage);
1037
1125
  evaluated++;
1038
1126
  if (verdict === null) {
1039
1127
  skippedInfra++;
@@ -1111,11 +1199,100 @@ function stageDecayPrune(db, dryRun) {
1111
1199
  }
1112
1200
  return { candidates, pruned, failed };
1113
1201
  }
1202
+ // ---------------------------------------------------------------------------
1203
+ // Stage 4.5: Memory cap eviction (#245)
1204
+ // ---------------------------------------------------------------------------
1205
+ //
1206
+ // The active forgetting mechanism. The pre-#245 prune (stageDecayPrune above)
1207
+ // is inert by design — the strength floor (~0.3162) + the `< 0.01` threshold +
1208
+ // the 365-day decay half-life means a never-accessed memory takes ~3 years to
1209
+ // become eligible, so the corpus grew without bound. This stage bounds it:
1210
+ // when the count exceeds `memorySoftCap`, the lowest-effectiveStrength
1211
+ // memories are evicted until under the cap.
1212
+ //
1213
+ // Eviction reuses the SAME effectiveStrength() the recall ranker uses — no
1214
+ // formula duplication, so the eviction criterion cannot drift from what
1215
+ // surfaces in the top-k. The evicted tail is, by construction, the tail that
1216
+ // was not surfacing anyway (cold, decayed). Ties are broken by oldest
1217
+ // COALESCE(last_accessed, created_at) — i.e. the memories that have gone
1218
+ // longest without anyone looking at them.
1219
+ //
1220
+ // `cap = 0` disables the stage (indefinite growth — the pre-#245 default is
1221
+ // preserved opt-out). The JS-side sort is O(n log n); at 10K memories the
1222
+ // load + compute is <100 ms, a rounding error against the LLM-bound phases.
1223
+ /**
1224
+ * Default soft cap on the memory corpus (#245). Above this the lowest-value
1225
+ * memories are evicted each nightly. 10000 balances headroom for a busy
1226
+ * self-hosted install against the noise cost of a bloated vector index
1227
+ * (recall top-k competes against the long tail). Override via `memorySoftCap`.
1228
+ */
1229
+ exports.DEFAULT_MEMORY_SOFT_CAP = 10000;
1230
+ function stageMemoryCapEviction(db, dryRun, cap) {
1231
+ // `0` = explicitly disabled (current/legacy behaviour). The guard is on `<=`
1232
+ // not `===` to also absorb a stray negative (readNonNegativeConfig already
1233
+ // rejects negatives at the boundary, but this stage is callable directly).
1234
+ if (cap <= 0)
1235
+ return { cap, evicted: 0 };
1236
+ const count = storage.countMemories(db);
1237
+ if (count <= cap)
1238
+ return { cap, evicted: 0 };
1239
+ const surplus = count - cap;
1240
+ // Load the fields effectiveStrength needs + the tiebreak. base_strength is
1241
+ // NOT NULL after scoring; the `?? 0.5` mirrors stageDecayPrune's defensive
1242
+ // default for unscored rows (inserts at 0.5). last_accessed is NULL until
1243
+ // first /recall-index exposure — COALESCE to created_at for the tiebreak so
1244
+ // never-shown memories sort by when they entered the corpus.
1245
+ const rows = db
1246
+ .prepare(`SELECT id, base_strength, last_accessed, access_count, created_at
1247
+ FROM memories`)
1248
+ .all();
1249
+ const linkCounts = storage.getAllLinkCounts(db);
1250
+ const now = new Date();
1251
+ // Decorate + sort: lowest effectiveStrength first; ties broken by oldest
1252
+ // COALESCE(last_accessed, created_at). The victims are the first `surplus`.
1253
+ const decorated = rows.map((r) => {
1254
+ const eff = (0, retrieval_js_1.effectiveStrength)(r.base_strength ?? 0.5, r.last_accessed, now, {
1255
+ accessCount: r.access_count ?? 0,
1256
+ linkCount: linkCounts.get(r.id) ?? 0,
1257
+ });
1258
+ return {
1259
+ id: r.id,
1260
+ eff,
1261
+ lastTouch: r.last_accessed ?? r.created_at,
1262
+ };
1263
+ });
1264
+ decorated.sort((a, b) =>
1265
+ // ASC by effectiveStrength, then ASC by lastTouch (oldest first = evict).
1266
+ a.eff !== b.eff ? a.eff - b.eff
1267
+ : a.lastTouch < b.lastTouch ? -1 : a.lastTouch > b.lastTouch ? 1 : 0);
1268
+ const victims = decorated.slice(0, surplus);
1269
+ if (dryRun) {
1270
+ console.log(`[hicortex] Memory cap eviction (dry-run): would remove ${victims.length} ` +
1271
+ `lowest-value memories (corpus ${count}, cap ${cap}).`);
1272
+ return { cap, evicted: victims.length };
1273
+ }
1274
+ // deleteMemory cascades: memory_links (both directions), memory_tags,
1275
+ // memory_vectors, and the FTS index (via the AFTER DELETE trigger on
1276
+ // memories, db.ts — no manual FTS cleanup needed). Wrap the batch in a
1277
+ // transaction so a failure leaves the corpus consistent (all-or-nothing).
1278
+ const tx = db.transaction(() => {
1279
+ for (const v of victims)
1280
+ storage.deleteMemory(db, v.id);
1281
+ });
1282
+ tx();
1283
+ console.log(`[hicortex] Memory cap eviction: removed ${victims.length} lowest-value ` +
1284
+ `memories (corpus was ${count}, cap ${cap}).`);
1285
+ return { cap, evicted: victims.length };
1286
+ }
1114
1287
  async function runConsolidation(db, llm, embedFn, dryRun = false, skipReflection = false, stateDir, domainOptions, supersessionOptions,
1115
1288
  /** Total LLM-call ceiling across classify-tier stages (#241). The caller
1116
1289
  * reads `consolidateMaxLlmCalls` from config and passes it; unset → the
1117
1290
  * exported `CONSOLIDATE_MAX_LLM_CALLS` default (5000). */
1118
- budgetMaxCalls) {
1291
+ budgetMaxCalls,
1292
+ /** Soft cap on the corpus (#245). Nightly.ts reads `memorySoftCap` from
1293
+ * config and passes it; unset → `DEFAULT_MEMORY_SOFT_CAP` (10000). `0`
1294
+ * disables eviction (indefinite growth). */
1295
+ memorySoftCap) {
1119
1296
  const start = new Date();
1120
1297
  const report = {
1121
1298
  started_at: start.toISOString(),
@@ -1141,6 +1318,10 @@ budgetMaxCalls) {
1141
1318
  new_memory_count: precheck.newMemories.length,
1142
1319
  unscored_count: scoreMemories.length - precheck.newMemories.length,
1143
1320
  };
1321
+ // Memory cap eviction (#245) — runs BEFORE the precheck skip so the corpus
1322
+ // is bounded even on quiet nights (no new memories → precheck would skip,
1323
+ // but the cap stage is pure DB: cheap, idempotent when under cap).
1324
+ report.stages.memory_cap = stageMemoryCapEviction(db, dryRun, memorySoftCap ?? exports.DEFAULT_MEMORY_SOFT_CAP);
1144
1325
  if (skip) {
1145
1326
  report.status = "skipped";
1146
1327
  report.completed_at = new Date().toISOString();
@@ -1192,6 +1373,7 @@ budgetMaxCalls) {
1192
1373
  report.stages.supersession = await stageSupersession(db, llm, budget, embedFn, dryRun, stateDir, supersessionOptions);
1193
1374
  // Stage 4: Decay & Prune
1194
1375
  report.stages.decay_prune = stageDecayPrune(db, dryRun);
1376
+ // (Memory cap eviction moved before the precheck skip — see above.)
1195
1377
  }
1196
1378
  catch (err) {
1197
1379
  report.status = "failed";