@gamaze/hicortex 0.20.7 → 0.20.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/README.md +18 -41
  2. package/assets/dashboard.html +3989 -836
  3. package/dist/calibration.d.ts +293 -0
  4. package/dist/calibration.js +379 -0
  5. package/dist/capture-health.d.ts +87 -0
  6. package/dist/capture-health.js +106 -0
  7. package/dist/capture-pause.d.ts +86 -0
  8. package/dist/capture-pause.js +127 -0
  9. package/dist/capture.d.ts +24 -3
  10. package/dist/capture.js +11 -1
  11. package/dist/classify-domains.d.ts +6 -0
  12. package/dist/classify-domains.js +7 -1
  13. package/dist/cli.js +38 -3
  14. package/dist/config-read.d.ts +1 -1
  15. package/dist/config-read.js +96 -9
  16. package/dist/consolidate.d.ts +114 -68
  17. package/dist/consolidate.js +302 -182
  18. package/dist/dashboard.d.ts +326 -6
  19. package/dist/dashboard.js +592 -7
  20. package/dist/db.js +105 -0
  21. package/dist/dedup.d.ts +34 -26
  22. package/dist/dedup.js +91 -57
  23. package/dist/distiller.js +1 -1
  24. package/dist/domain-classify.d.ts +7 -6
  25. package/dist/domain-classify.js +12 -10
  26. package/dist/eval/decay-eval.d.ts +3 -3
  27. package/dist/eval/decay-eval.js +4 -4
  28. package/dist/eval/importance-eval.d.ts +85 -0
  29. package/dist/eval/importance-eval.js +286 -0
  30. package/dist/eval/planted-eval.d.ts +26 -0
  31. package/dist/eval/planted-eval.js +97 -0
  32. package/dist/eval/planted-fixtures.d.ts +107 -0
  33. package/dist/eval/planted-fixtures.js +283 -0
  34. package/dist/eval/planted-harness.d.ts +176 -0
  35. package/dist/eval/planted-harness.js +649 -0
  36. package/dist/eval/ranking-battery.d.ts +78 -0
  37. package/dist/eval/ranking-battery.js +181 -0
  38. package/dist/eval/ranking-eval.d.ts +41 -0
  39. package/dist/eval/ranking-eval.js +391 -0
  40. package/dist/eval/ranking-fixtures.d.ts +77 -0
  41. package/dist/eval/ranking-fixtures.js +226 -0
  42. package/dist/identity-store.d.ts +21 -0
  43. package/dist/identity-store.js +49 -0
  44. package/dist/index.js +4 -3
  45. package/dist/init.d.ts +23 -3
  46. package/dist/init.js +84 -9
  47. package/dist/llm.d.ts +43 -58
  48. package/dist/llm.js +87 -101
  49. package/dist/mcp-server.d.ts +12 -0
  50. package/dist/mcp-server.js +213 -32
  51. package/dist/nightly.d.ts +9 -1
  52. package/dist/nightly.js +164 -110
  53. package/dist/nofit.d.ts +4 -11
  54. package/dist/nofit.js +6 -23
  55. package/dist/prompts.d.ts +10 -0
  56. package/dist/prompts.js +28 -5
  57. package/dist/recall-index.d.ts +30 -28
  58. package/dist/recall-index.js +21 -18
  59. package/dist/recall-registry.d.ts +2 -1
  60. package/dist/recall-registry.js +35 -1
  61. package/dist/reconsolidation.d.ts +168 -87
  62. package/dist/reconsolidation.js +818 -377
  63. package/dist/relink.js +3 -4
  64. package/dist/rescore-importance.d.ts +80 -0
  65. package/dist/rescore-importance.js +236 -0
  66. package/dist/retrieval.d.ts +80 -35
  67. package/dist/retrieval.js +322 -105
  68. package/dist/run-deadline.d.ts +62 -0
  69. package/dist/run-deadline.js +73 -0
  70. package/dist/schema-prototypes.d.ts +3 -3
  71. package/dist/schema-prototypes.js +3 -3
  72. package/dist/stages.d.ts +37 -0
  73. package/dist/stages.js +51 -0
  74. package/dist/state.d.ts +34 -9
  75. package/dist/storage.d.ts +50 -18
  76. package/dist/storage.js +125 -30
  77. package/dist/telemetry.d.ts +8 -7
  78. package/dist/token-budget.js +3 -4
  79. package/dist/type-classify.js +4 -4
  80. package/dist/types.d.ts +143 -155
  81. package/domains.example.json +4 -5
  82. package/hermes-plugin/hicortex/README.md +2 -2
  83. package/openclaw.plugin.json +1 -1
  84. package/package.json +4 -1
  85. package/pi-extension/hicortex/README.md +1 -1
  86. package/server.json +3 -3
@@ -0,0 +1,73 @@
1
+ "use strict";
2
+ /**
3
+ * The ONE cooperative wall-clock deadline for a nightly run (#405).
4
+ *
5
+ * Generalizes the #401/#402/#404 reconsolidation pattern (a deadline checked
6
+ * at safe boundaries + resumable cursors) from one stage to the whole
7
+ * pipeline: capture segments, every consolidation stage boundary, the
8
+ * item loops, and the deterministic merge zone all check the SAME deadline,
9
+ * created once at nightly start. On expiry each check site stops cleanly at
10
+ * its last safe boundary — cursors (capture per-session, supersession,
11
+ * reconsolidation) hold below unconfirmed work, so the next run resumes
12
+ * without redoing confirmed work or losing deferred work.
13
+ *
14
+ * Deferral is a REPORTED outcome, not an error: a run whose deadline fired
15
+ * reports consolidation status "deferred" and does NOT advance
16
+ * `lastConsolidated` (the same gate as endpoint_down — otherwise the
17
+ * pending-set queries would silently lose the deferred work).
18
+ *
19
+ * One structured log line per deferred stage (`event=deadline_deferred`,
20
+ * the same key=value style as `event=budget_exhausted` /
21
+ * `event=circuit_open`) so `grep event=deadline_deferred` is one of the two
22
+ * nightly health greps.
23
+ */
24
+ Object.defineProperty(exports, "__esModule", { value: true });
25
+ exports.DEFAULT_NIGHTLY_TIME_BUDGET_MINUTES = void 0;
26
+ exports.createRunDeadline = createRunDeadline;
27
+ exports.resolveNightlyTimeBudgetMinutes = resolveNightlyTimeBudgetMinutes;
28
+ const config_read_js_1 = require("./config-read.js");
29
+ /** Default wall-clock budget for a full nightly run, in minutes (#405).
30
+ * 240 sits between the old stage-local reconsolidation clock (120) and the
31
+ * old systemd backstop (360) — the deadline is now the operating bound and
32
+ * the unit backstop (budget + 60 min slack) only catches a true hang. */
33
+ exports.DEFAULT_NIGHTLY_TIME_BUDGET_MINUTES = 240;
34
+ /**
35
+ * Create a run deadline of `minutes` (already-resolved value; see
36
+ * resolveNightlyTimeBudgetMinutes for the config boundary).
37
+ */
38
+ function createRunDeadline(minutes, now = Date.now) {
39
+ const start = now();
40
+ const deadlineAt = start + Math.max(0, Math.round(minutes * 60_000));
41
+ const logged = new Set();
42
+ const deferred = [];
43
+ return {
44
+ deadlineAt,
45
+ remainingMs: () => Math.max(0, deadlineAt - now()),
46
+ expired: () => now() >= deadlineAt,
47
+ hit(stage) {
48
+ if (now() < deadlineAt)
49
+ return false;
50
+ if (!logged.has(stage)) {
51
+ logged.add(stage);
52
+ deferred.push(stage);
53
+ // Structured (grep-able) + human-readable — same contract as
54
+ // event=budget_exhausted (consolidate.ts) and event=circuit_open
55
+ // (llm.ts). remaining_ms=0 states the reason plainly.
56
+ console.warn(`[hicortex] event=deadline_deferred stage=${stage} ` +
57
+ `deadline_minutes=${(deadlineAt - start) / 60_000} remaining_ms=0`);
58
+ }
59
+ return true;
60
+ },
61
+ deferredStages: () => deferred,
62
+ };
63
+ }
64
+ /**
65
+ * Resolve `nightlyTimeBudgetMinutes` from the saved config (#405): positive
66
+ * finite number wins; absent/0/invalid → the 240 default. Unlike the old
67
+ * `reconsolidationMaxMinutes`, 0 does NOT disable — there is ALWAYS a
68
+ * deadline (readPositiveConfig rejects 0 with a warn, which is the loud
69
+ * migration signal for an operator who used 0 as "off").
70
+ */
71
+ function resolveNightlyTimeBudgetMinutes(config) {
72
+ return (0, config_read_js_1.readPositiveConfig)(config ?? {}, "nightlyTimeBudgetMinutes", exports.DEFAULT_NIGHTLY_TIME_BUDGET_MINUTES);
73
+ }
@@ -121,9 +121,9 @@ export declare function computeTagWeights(db: Database.Database, memoryId: strin
121
121
  *
122
122
  * Used by the no-fit path (nofit.ts, owner amendment 07.07): when the LLM
123
123
  * says no domain fits, the memory can still earn a WEAK primary from pure
124
- * embedding association — provided the best cosine clears the configured
125
- * weakPrimaryFloor (the caller checks the floor; this function just reports
126
- * the argmax).
124
+ * embedding association — provided the best cosine clears the weak-primary
125
+ * floor (release-managed since #408; the caller checks the floor, this
126
+ * function just reports the argmax).
127
127
  *
128
128
  * Returns null when the memory has no stored vector or no configured domain
129
129
  * has a prototype (nothing to associate against). Ties resolve to the FIRST
@@ -239,9 +239,9 @@ function computeTagWeights(db, memoryId, tags, prototypes) {
239
239
  *
240
240
  * Used by the no-fit path (nofit.ts, owner amendment 07.07): when the LLM
241
241
  * says no domain fits, the memory can still earn a WEAK primary from pure
242
- * embedding association — provided the best cosine clears the configured
243
- * weakPrimaryFloor (the caller checks the floor; this function just reports
244
- * the argmax).
242
+ * embedding association — provided the best cosine clears the weak-primary
243
+ * floor (release-managed since #408; the caller checks the floor, this
244
+ * function just reports the argmax).
245
245
  *
246
246
  * Returns null when the memory has no stored vector or no configured domain
247
247
  * has a prototype (nothing to associate against). Ties resolve to the FIRST
@@ -0,0 +1,37 @@
1
+ /**
2
+ * Memory maturity stages — the console's derived presentation layer
3
+ * (#409/#421 Phase 1).
4
+ *
5
+ * The E-reframe (standing ruling in the #409 thread): stages are DERIVED
6
+ * presentation of the one numeric strength signal, never stored state. This
7
+ * module holds the pure derivation only — thresholds live in calibration.ts
8
+ * (the #408 single home, release-managed with the evolution contract); no
9
+ * db, no I/O, no config reads.
10
+ *
11
+ * Wire keys are forming/belief/truth/fading (lifecycle order). The console
12
+ * renders human labels over these keys; nothing else consumes them.
13
+ */
14
+ /** A memory's derived maturity stage (lifecycle order). */
15
+ export type Stage = "forming" | "belief" | "truth" | "fading";
16
+ /** The four stages in lifecycle order: forming → belief → truth → fading. */
17
+ export declare const STAGE_KEYS: readonly Stage[];
18
+ /**
19
+ * Derive a memory's stage from its effective strength and recency.
20
+ *
21
+ * The recency gate runs FIRST: a memory untouched for
22
+ * STAGE_FADING_DAYS days is Fading no matter how strong it is — decay is a
23
+ * function of access in this system (effectiveStrength already folds
24
+ * last_accessed in, but the gate makes long silence legible on its own).
25
+ * Then the strength bands (calibrated against the measured production
26
+ * distribution — see calibration.ts): < FADING → fading (the weak cluster),
27
+ * ≥ TRUTH → truth, ≥ BELIEF → belief, else forming.
28
+ *
29
+ * `daysSinceAccess` is whole days (UTC day diff is fine — capture is
30
+ * night-resolution) since the memory was last touched. The DASHBOARD handler
31
+ * passes days since `last_accessed`, falling back to days since `created_at`
32
+ * when last_accessed is NULL (never accessed since ingest — creation is the
33
+ * last real signal). `null` means "recency unknown" and skips the gate
34
+ * (strength-only derivation) — the handler never sends it; it exists so the
35
+ * function is total over caller data quality.
36
+ */
37
+ export declare function deriveStage(effectiveStrength: number, daysSinceAccess: number | null): Stage;
package/dist/stages.js ADDED
@@ -0,0 +1,51 @@
1
+ "use strict";
2
+ /**
3
+ * Memory maturity stages — the console's derived presentation layer
4
+ * (#409/#421 Phase 1).
5
+ *
6
+ * The E-reframe (standing ruling in the #409 thread): stages are DERIVED
7
+ * presentation of the one numeric strength signal, never stored state. This
8
+ * module holds the pure derivation only — thresholds live in calibration.ts
9
+ * (the #408 single home, release-managed with the evolution contract); no
10
+ * db, no I/O, no config reads.
11
+ *
12
+ * Wire keys are forming/belief/truth/fading (lifecycle order). The console
13
+ * renders human labels over these keys; nothing else consumes them.
14
+ */
15
+ Object.defineProperty(exports, "__esModule", { value: true });
16
+ exports.STAGE_KEYS = void 0;
17
+ exports.deriveStage = deriveStage;
18
+ const calibration_js_1 = require("./calibration.js");
19
+ /** The four stages in lifecycle order: forming → belief → truth → fading. */
20
+ exports.STAGE_KEYS = ["forming", "belief", "truth", "fading"];
21
+ /**
22
+ * Derive a memory's stage from its effective strength and recency.
23
+ *
24
+ * The recency gate runs FIRST: a memory untouched for
25
+ * STAGE_FADING_DAYS days is Fading no matter how strong it is — decay is a
26
+ * function of access in this system (effectiveStrength already folds
27
+ * last_accessed in, but the gate makes long silence legible on its own).
28
+ * Then the strength bands (calibrated against the measured production
29
+ * distribution — see calibration.ts): < FADING → fading (the weak cluster),
30
+ * ≥ TRUTH → truth, ≥ BELIEF → belief, else forming.
31
+ *
32
+ * `daysSinceAccess` is whole days (UTC day diff is fine — capture is
33
+ * night-resolution) since the memory was last touched. The DASHBOARD handler
34
+ * passes days since `last_accessed`, falling back to days since `created_at`
35
+ * when last_accessed is NULL (never accessed since ingest — creation is the
36
+ * last real signal). `null` means "recency unknown" and skips the gate
37
+ * (strength-only derivation) — the handler never sends it; it exists so the
38
+ * function is total over caller data quality.
39
+ */
40
+ function deriveStage(effectiveStrength, daysSinceAccess) {
41
+ if (daysSinceAccess !== null && daysSinceAccess >= calibration_js_1.STAGE_FADING_DAYS) {
42
+ return "fading";
43
+ }
44
+ if (effectiveStrength < calibration_js_1.STAGE_FADING_STRENGTH)
45
+ return "fading";
46
+ if (effectiveStrength >= calibration_js_1.STAGE_TRUTH_STRENGTH)
47
+ return "truth";
48
+ if (effectiveStrength >= calibration_js_1.STAGE_BELIEF_STRENGTH)
49
+ return "belief";
50
+ return "forming";
51
+ }
package/dist/state.d.ts CHANGED
@@ -55,22 +55,39 @@ export interface HicortexState {
55
55
  * B) — highest memories.rowid whose decision/correction candidates have
56
56
  * been evaluated (or infra-skipped) this run. Absent/0 = never run. Unlike
57
57
  * relinkCursor/domainCursor (separate resumable CLI commands), this cursor
58
- * advances a SMALL amount per night (config `supersessionMaxCalls`, default
59
- * 30) as part of the regular nightly — the corpus is back-processed
60
- * gradually over many nights.
58
+ * advances within the shared nightly LLM call budget as part of the regular
59
+ * nightly — the corpus is back-processed gradually over many nights.
61
60
  */
62
61
  supersessionCursor?: number;
63
62
  /**
64
63
  * Resume cursor for the nightly's reconsolidation stage (#384) — highest
65
64
  * memories.rowid whose candidates have been evaluated (or infra-skipped)
66
- * this run. Absent/0 = never run. Same advance-past-considered-candidates
67
- * discipline as supersessionCursor, with one addition: when a rewrite group
68
- * could not be applied (budget exhausted / rewrite-call infra error), the
69
- * cursor holds BELOW the earliest candidate contributing to an un-applied
70
- * group so those pairs are re-detected next run — a confirmed correction is
71
- * never silently dropped by the cursor passing it.
65
+ * with all their CONFIRMED work applied. Absent/0 = never run. Same
66
+ * advance-past-considered-candidates discipline as supersessionCursor,
67
+ * with one addition (#439): confirmed merges and rewrite groups apply at
68
+ * the candidate boundary — the END of the iteration that confirmed them —
69
+ * and the cursor advances past a candidate only when that apply landed.
70
+ * A deferral (budget refusal at the rewrite call, deadline at the
71
+ * boundary, backup failure, lock-busy merge surviving the final drain)
72
+ * holds the cursor BELOW the current candidate, so the hold is bounded to
73
+ * ONE candidate's pairs: next run re-detects and re-judges exactly those
74
+ * (dup-over-loss — a confirmed resolution is never silently dropped by the
75
+ * cursor passing it). The ONE cross-candidate window is the lock-busy
76
+ * retry list (plus deadline/backup-dropped tails re-queued at their
77
+ * boundary): while any retry merge is pending, every persisted checkpoint
78
+ * clamps below its earliest contributor, so a killed or stopped run can
79
+ * never strand a confirmed merge behind the cursor (fix round, #440).
72
80
  */
73
81
  reconsolidationCursor?: number;
82
+ /**
83
+ * #439 scan high-water for the reconsolidation stage — the highest
84
+ * memories.rowid any run has ENTERED, never held back by un-applied work
85
+ * (unlike reconsolidationCursor, which holds below deferred applies).
86
+ * Seeds the re-judged/new verdict split: a verdict call on a candidate
87
+ * at/below this mark is a re-judgment of previously judged work. Absent =
88
+ * never run; persisted alongside the cursor at every checkpoint.
89
+ */
90
+ reconsolidationScannedRowid?: number;
74
91
  /**
75
92
  * Resume cursor for `hicortex classify-types` (#216) — highest memories.rowid
76
93
  * whose batch has been fully committed. Absent/0 = never run (or reset).
@@ -78,6 +95,14 @@ export interface HicortexState {
78
95
  * interruption never loses more than the in-flight batch.
79
96
  */
80
97
  typeCursor?: number;
98
+ /**
99
+ * Resume cursor for `hicortex rescore-importance` (#425) — highest live
100
+ * memories.rowid re-judged under the current rubric. Absent/0 = never run
101
+ * (or reset). Advances per committed batch (LLM calls are 10 rows each
102
+ * inside a `--batch` slice); an infra error holds it at the last fully
103
+ * committed slice so a re-run resumes cleanly.
104
+ */
105
+ rescoreImportanceCursor?: number;
81
106
  /**
82
107
  * LLM token usage accrued this billing period (#246). Period reset is
83
108
  * monthly: when `periodStart` is in a previous calendar month, the totals
package/dist/storage.d.ts CHANGED
@@ -8,6 +8,13 @@ import type { Memory, MemoryLink, InsertMemoryOptions } from "./types.js";
8
8
  * Serialize a Float32Array embedding to a Buffer for sqlite-vec.
9
9
  */
10
10
  export declare function embedToBlob(embedding: Float32Array): Buffer;
11
+ /**
12
+ * #421 machine × harness identity: optional capture-machine provenance on
13
+ * /ingest + /distill. A string, trimmed, capped at 128 chars; anything else
14
+ * (absent, wrong type, blank) becomes NULL — a wrong machine name is worse
15
+ * than none. Never filtered; presentation grouping only.
16
+ */
17
+ export declare function sanitizeSourceMachine(v: unknown): string | null;
11
18
  /**
12
19
  * Insert a memory and its vector embedding. Returns the memory's UUID.
13
20
  *
@@ -35,6 +42,28 @@ export declare function updateMemory(db: Database.Database, memoryId: string, fi
35
42
  * Atomically increment access_count and reset last_accessed.
36
43
  */
37
44
  export declare function strengthenMemory(db: Database.Database, memoryId: string, nowIsoStr: string): void;
45
+ /**
46
+ * Record an owner corroboration (#423 phase 3): one UPDATE that bumps
47
+ * corroboration_count, nudges base_strength up by the calibration delta
48
+ * (MIN-capped at the importance ceiling, #425; COALESCE keeps NULL bases
49
+ * honest — the unscored default), stamps the importance watermark, and
50
+ * refreshes last_accessed (the console's "last confirmed" cell reads it: an
51
+ * enrich IS a confirmation, and it keeps the stage fading gate honest).
52
+ * EVIDENCE ABOUT IMPORTANCE ONLY — never access_count (reserved for real
53
+ * recall use) nor shown_count (index exposure): faking either corrupts the
54
+ * uses-per-showing adoption metric.
55
+ *
56
+ * Nightly interaction: the importance pool keys on importance_scored_at IS
57
+ * NULL (#425), so the watermark stamp here keeps the pre-#425 contract — an
58
+ * enriched memory leaves the pool and the owner's mark stands in for (and is
59
+ * never stomped by) the first LLM score. Dedup merges take
60
+ * max(base_strength), so a merge never loses the mark. Returns the fresh row
61
+ * values (re-read after the update); null when the id matches nothing.
62
+ */
63
+ export declare function enrichMemory(db: Database.Database, memoryId: string, nowIsoStr: string): {
64
+ corroborationCount: number;
65
+ baseStrength: number;
66
+ } | null;
38
67
  /**
39
68
  * Record that memories appeared in a pushed recall index (#192): bump
40
69
  * shown_count and refresh last_accessed (a mild strengthen — the decay clock
@@ -132,15 +161,15 @@ export declare function vectorSearch(db: Database.Database, queryEmbedding: Floa
132
161
  distance: number;
133
162
  }>;
134
163
  /**
135
- * BM25F field weights (config-driven via {@link configureBm25Fts}, called from
136
- * retrieval.configureScoring at boot). The order mirrors the FTS5 column
137
- * declaration in db.ts (content, project, domain) — `bm25(memories_fts, …)`
138
- * takes weights POSITIONALLY, so a new FTS column MUST be added here in the
139
- * same position or the weighting silently shifts. Defaults favor scope fields
140
- * (project/domain) over body so cross-scope noise that wins on raw token
141
- * frequency (the marine "battery" memory on a hardware query) is demoted
142
- * without excluding it — the same "graded, never binary" discipline as
143
- * computeScore's affinity terms.
164
+ * BM25F field weights (release-managed since #408 — the defaults resolve from
165
+ * calibration.ts; {@link configureBm25Fts} is the eval/test seam). The order
166
+ * mirrors the FTS5 column declaration in db.ts (content, project, domain) —
167
+ * `bm25(memories_fts, …)` takes weights POSITIONALLY, so a new FTS column
168
+ * MUST be added here in the same position or the weighting silently shifts.
169
+ * Defaults favor scope fields (project/domain) over body so cross-scope noise
170
+ * that wins on raw token frequency (the marine "battery" memory on a hardware
171
+ * query) is demoted without excluding it — the same "graded, never binary"
172
+ * discipline as computeScore's affinity terms.
144
173
  */
145
174
  export interface Bm25Weights {
146
175
  body: number;
@@ -148,14 +177,14 @@ export interface Bm25Weights {
148
177
  domain: number;
149
178
  }
150
179
  /**
151
- * Configure BM25F weights from config. Called by retrieval.configureScoring
152
- * (which itself is called at server + nightly boot) so storage and retrieval
153
- * rank identically. Invalid/out-of-range values keep the shipped default per
154
- * key. Range [0, ∞) — a 0 weight effectively drops that field from the score;
155
- * negative values are rejected (BM25F sign semantics break otherwise). Returns
156
- * the resolved set for logging/tests.
180
+ * Configure BM25F weights from RESOLVED overrides (the eval/test seam — #408;
181
+ * production calls it with no argument so storage and retrieval rank with the
182
+ * identical calibration defaults). Invalid/out-of-range values keep the
183
+ * shipped default per key. Range [0, ∞) — a 0 weight effectively drops that
184
+ * field from the score; negative values are rejected (BM25F sign semantics
185
+ * break otherwise). Returns the resolved set for logging/tests.
157
186
  */
158
- export declare function configureBm25Fts(config?: Record<string, unknown> | null): Bm25Weights;
187
+ export declare function configureBm25Fts(overrides?: Partial<Bm25Weights> | null): Bm25Weights;
159
188
  /** Current resolved weights (tests + status output). */
160
189
  export declare function getBm25Weights(): Bm25Weights;
161
190
  /**
@@ -267,8 +296,11 @@ export declare function getPruneCandidates(db: Database.Database, cutoffIso: str
267
296
  */
268
297
  export declare function getAllLinkCounts(db: Database.Database): Map<string, number>;
269
298
  /**
270
- * Get all memories with default base_strength (never scored).
271
- * Absorbed memories are excluded (#384): they are invisible to recall, so
299
+ * Get all never-scored memories — the nightly importance pool, keyed on the
300
+ * importance_scored_at watermark (#425, migration v19). The pre-v19 sentinel
301
+ * (`base_strength = 0.5`) re-rolled every row the model genuinely scored
302
+ * 0.5, every night; NULL watermark = never scored under ANY rubric. Absorbed
303
+ * memories are excluded (#384): they are invisible to recall, so
272
304
  * importance-scoring one would spend an LLM call on dead evidence.
273
305
  */
274
306
  export declare function getUnscoredMemories(db: Database.Database): Memory[];
package/dist/storage.js CHANGED
@@ -3,14 +3,49 @@
3
3
  * Storage layer — CRUD operations for the SQLite + sqlite-vec database.
4
4
  * Ported from hicortex/storage.py. All functions are synchronous (better-sqlite3).
5
5
  */
6
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
7
+ if (k2 === undefined) k2 = k;
8
+ var desc = Object.getOwnPropertyDescriptor(m, k);
9
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
10
+ desc = { enumerable: true, get: function() { return m[k]; } };
11
+ }
12
+ Object.defineProperty(o, k2, desc);
13
+ }) : (function(o, m, k, k2) {
14
+ if (k2 === undefined) k2 = k;
15
+ o[k2] = m[k];
16
+ }));
17
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
18
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
19
+ }) : function(o, v) {
20
+ o["default"] = v;
21
+ });
22
+ var __importStar = (this && this.__importStar) || (function () {
23
+ var ownKeys = function(o) {
24
+ ownKeys = Object.getOwnPropertyNames || function (o) {
25
+ var ar = [];
26
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
27
+ return ar;
28
+ };
29
+ return ownKeys(o);
30
+ };
31
+ return function (mod) {
32
+ if (mod && mod.__esModule) return mod;
33
+ var result = {};
34
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
35
+ __setModuleDefault(result, mod);
36
+ return result;
37
+ };
38
+ })();
6
39
  Object.defineProperty(exports, "__esModule", { value: true });
7
40
  exports.FTS_MATCH_MAX_TOKENS = void 0;
8
41
  exports.embedToBlob = embedToBlob;
42
+ exports.sanitizeSourceMachine = sanitizeSourceMachine;
9
43
  exports.insertMemory = insertMemory;
10
44
  exports.resolveMemoryId = resolveMemoryId;
11
45
  exports.getMemory = getMemory;
12
46
  exports.updateMemory = updateMemory;
13
47
  exports.strengthenMemory = strengthenMemory;
48
+ exports.enrichMemory = enrichMemory;
14
49
  exports.touchMemoriesShown = touchMemoriesShown;
15
50
  exports.deleteMemory = deleteMemory;
16
51
  exports.memoryRowid = memoryRowid;
@@ -38,6 +73,7 @@ exports.getAllLinkCounts = getAllLinkCounts;
38
73
  exports.getUnscoredMemories = getUnscoredMemories;
39
74
  const node_crypto_1 = require("node:crypto");
40
75
  const schema_prototypes_js_1 = require("./schema-prototypes.js");
76
+ const CALIBRATION = __importStar(require("./calibration.js"));
41
77
  // ---------------------------------------------------------------------------
42
78
  // Helpers
43
79
  // ---------------------------------------------------------------------------
@@ -56,6 +92,18 @@ function rowToMemory(row) {
56
92
  // ---------------------------------------------------------------------------
57
93
  // Single memory CRUD
58
94
  // ---------------------------------------------------------------------------
95
+ /**
96
+ * #421 machine × harness identity: optional capture-machine provenance on
97
+ * /ingest + /distill. A string, trimmed, capped at 128 chars; anything else
98
+ * (absent, wrong type, blank) becomes NULL — a wrong machine name is worse
99
+ * than none. Never filtered; presentation grouping only.
100
+ */
101
+ function sanitizeSourceMachine(v) {
102
+ if (typeof v !== "string")
103
+ return null;
104
+ const t = v.trim();
105
+ return t.length > 0 ? t.slice(0, 128) : null;
106
+ }
59
107
  /**
60
108
  * Insert a memory and its vector embedding. Returns the memory's UUID.
61
109
  *
@@ -74,9 +122,9 @@ function insertMemory(db, content, embedding, opts = {}) {
74
122
  .prepare(`INSERT OR IGNORE INTO memories
75
123
  (id, content, base_strength, last_accessed, access_count,
76
124
  created_at, ingested_at, source_agent, source_agent_id, source_session,
77
- source_domain, project, privacy, memory_type)
78
- VALUES (?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
79
- .run(id, content, opts.baseStrength ?? 0.5, ts, ts, ingestedTs, opts.sourceAgent ?? "default", opts.sourceAgentId ?? null, sourceSession, opts.sourceDomain ?? null, opts.project ?? null, opts.privacy ?? null, opts.memoryType ?? "experience");
125
+ source_domain, project, privacy, memory_type, source_machine)
126
+ VALUES (?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
127
+ .run(id, content, opts.baseStrength ?? 0.5, ts, ts, ingestedTs, opts.sourceAgent ?? "default", opts.sourceAgentId ?? null, sourceSession, opts.sourceDomain ?? null, opts.project ?? null, opts.privacy ?? null, opts.memoryType ?? "experience", opts.sourceMachine ?? null);
80
128
  if (result.changes > 0) {
81
129
  // New row — store its vector.
82
130
  db.prepare("INSERT INTO memory_vectors (id, embedding) VALUES (?, ?)").run(id, embedToBlob(embedding));
@@ -138,6 +186,10 @@ const ALLOWED_UPDATE_FIELDS = new Set([
138
186
  // reconsolidation stage, explicit ingest marks, and history rollback.
139
187
  // Code-defined vocabulary — see Memory.status.
140
188
  "status",
189
+ // Importance scored-at watermark (#425, migration v19): written by
190
+ // stageImportance, the enrich path, and the rescore-importance backfill —
191
+ // the moment a row's base_strength is settled under some rubric.
192
+ "importance_scored_at",
141
193
  ]);
142
194
  /**
143
195
  * Update specific fields on a memory.
@@ -164,6 +216,42 @@ function strengthenMemory(db, memoryId, nowIsoStr) {
164
216
  SET access_count = access_count + 1, last_accessed = ?
165
217
  WHERE id = ?`).run(nowIsoStr, memoryId);
166
218
  }
219
+ /**
220
+ * Record an owner corroboration (#423 phase 3): one UPDATE that bumps
221
+ * corroboration_count, nudges base_strength up by the calibration delta
222
+ * (MIN-capped at the importance ceiling, #425; COALESCE keeps NULL bases
223
+ * honest — the unscored default), stamps the importance watermark, and
224
+ * refreshes last_accessed (the console's "last confirmed" cell reads it: an
225
+ * enrich IS a confirmation, and it keeps the stage fading gate honest).
226
+ * EVIDENCE ABOUT IMPORTANCE ONLY — never access_count (reserved for real
227
+ * recall use) nor shown_count (index exposure): faking either corrupts the
228
+ * uses-per-showing adoption metric.
229
+ *
230
+ * Nightly interaction: the importance pool keys on importance_scored_at IS
231
+ * NULL (#425), so the watermark stamp here keeps the pre-#425 contract — an
232
+ * enriched memory leaves the pool and the owner's mark stands in for (and is
233
+ * never stomped by) the first LLM score. Dedup merges take
234
+ * max(base_strength), so a merge never loses the mark. Returns the fresh row
235
+ * values (re-read after the update); null when the id matches nothing.
236
+ */
237
+ function enrichMemory(db, memoryId, nowIsoStr) {
238
+ const result = db
239
+ .prepare(`UPDATE memories
240
+ SET corroboration_count = corroboration_count + 1,
241
+ base_strength = MIN(?, COALESCE(base_strength, 0.5) + ?),
242
+ importance_scored_at = COALESCE(importance_scored_at, ?),
243
+ last_accessed = ?
244
+ WHERE id = ?`)
245
+ .run(CALIBRATION.IMPORTANCE_CEILING, CALIBRATION.ENRICH_STRENGTH_DELTA, nowIsoStr, nowIsoStr, memoryId);
246
+ if (result.changes === 0)
247
+ return null;
248
+ const row = db
249
+ .prepare("SELECT corroboration_count, base_strength FROM memories WHERE id = ?")
250
+ .get(memoryId);
251
+ return row
252
+ ? { corroborationCount: row.corroboration_count, baseStrength: row.base_strength }
253
+ : null;
254
+ }
167
255
  /**
168
256
  * Record that memories appeared in a pushed recall index (#192): bump
169
257
  * shown_count and refresh last_accessed (a mild strengthen — the decay clock
@@ -354,28 +442,28 @@ function vectorSearch(db, queryEmbedding, limit = 10, excludeIds = []) {
354
442
  return results;
355
443
  }
356
444
  const BM25_DEFAULTS = {
357
- body: 1.0,
358
- project: 2.0,
359
- domain: 2.0,
445
+ body: CALIBRATION.BM25_WEIGHT_BODY,
446
+ project: CALIBRATION.BM25_WEIGHT_PROJECT,
447
+ domain: CALIBRATION.BM25_WEIGHT_DOMAIN,
360
448
  };
361
449
  let bm25Weights = { ...BM25_DEFAULTS };
362
450
  /**
363
- * Configure BM25F weights from config. Called by retrieval.configureScoring
364
- * (which itself is called at server + nightly boot) so storage and retrieval
365
- * rank identically. Invalid/out-of-range values keep the shipped default per
366
- * key. Range [0, ∞) — a 0 weight effectively drops that field from the score;
367
- * negative values are rejected (BM25F sign semantics break otherwise). Returns
368
- * the resolved set for logging/tests.
369
- */
370
- function configureBm25Fts(config) {
371
- const num = (key, dflt) => {
372
- const v = Number(config?.[key]);
373
- return Number.isFinite(v) && v >= 0 ? v : dflt;
451
+ * Configure BM25F weights from RESOLVED overrides (the eval/test seam — #408;
452
+ * production calls it with no argument so storage and retrieval rank with the
453
+ * identical calibration defaults). Invalid/out-of-range values keep the
454
+ * shipped default per key. Range [0, ∞) — a 0 weight effectively drops that
455
+ * field from the score; negative values are rejected (BM25F sign semantics
456
+ * break otherwise). Returns the resolved set for logging/tests.
457
+ */
458
+ function configureBm25Fts(overrides) {
459
+ const num = (v, dflt) => {
460
+ const n = Number(v);
461
+ return Number.isFinite(n) && n >= 0 ? n : dflt;
374
462
  };
375
463
  bm25Weights = {
376
- body: num("bm25WeightBody", BM25_DEFAULTS.body),
377
- project: num("bm25WeightProject", BM25_DEFAULTS.project),
378
- domain: num("bm25WeightDomain", BM25_DEFAULTS.domain),
464
+ body: num(overrides?.body, BM25_DEFAULTS.body),
465
+ project: num(overrides?.project, BM25_DEFAULTS.project),
466
+ domain: num(overrides?.domain, BM25_DEFAULTS.domain),
379
467
  };
380
468
  return { ...bm25Weights };
381
469
  }
@@ -487,14 +575,18 @@ function searchFts(db, query, limit = 10, sourceAgent) {
487
575
  * Create a link between two memories.
488
576
  */
489
577
  function addLink(db, sourceId, targetId, relationship, strength = 0.5) {
490
- // Guard: superseded_by and corrected_by are the ranking-demotion /
491
- // correction-resolution signals, so never let a different relationship
492
- // clobber an existing one for the same pair — INSERT OR REPLACE would
493
- // otherwise silently remove the resolution (corrected_by is protected
494
- // exactly like superseded_by, #384 AC9).
495
- if (relationship !== "superseded_by" && relationship !== "corrected_by") {
578
+ // Guard: superseded_by, corrected_by, and conflicts are the ranking-demotion /
579
+ // correction-resolution / conflict-preservation signals, so never let a
580
+ // different relationship clobber an existing one for the same pair — INSERT
581
+ // OR REPLACE would otherwise silently remove the resolution (corrected_by is
582
+ // protected exactly like superseded_by, #384 AC9; conflicts joins the set in
583
+ // #393 guard-C — a conflicts edge marks the pair as never-blendable, so it
584
+ // must survive arbitrary later link writes exactly the same way).
585
+ if (relationship !== "superseded_by" &&
586
+ relationship !== "corrected_by" &&
587
+ relationship !== "conflicts") {
496
588
  const protectedLink = db
497
- .prepare("SELECT 1 FROM memory_links WHERE source_id = ? AND target_id = ? AND relationship IN ('superseded_by', 'corrected_by') LIMIT 1")
589
+ .prepare("SELECT 1 FROM memory_links WHERE source_id = ? AND target_id = ? AND relationship IN ('superseded_by', 'corrected_by', 'conflicts') LIMIT 1")
498
590
  .get(sourceId, targetId);
499
591
  if (protectedLink)
500
592
  return;
@@ -641,14 +733,17 @@ function getAllLinkCounts(db) {
641
733
  return counts;
642
734
  }
643
735
  /**
644
- * Get all memories with default base_strength (never scored).
645
- * Absorbed memories are excluded (#384): they are invisible to recall, so
736
+ * Get all never-scored memories — the nightly importance pool, keyed on the
737
+ * importance_scored_at watermark (#425, migration v19). The pre-v19 sentinel
738
+ * (`base_strength = 0.5`) re-rolled every row the model genuinely scored
739
+ * 0.5, every night; NULL watermark = never scored under ANY rubric. Absorbed
740
+ * memories are excluded (#384): they are invisible to recall, so
646
741
  * importance-scoring one would spend an LLM call on dead evidence.
647
742
  */
648
743
  function getUnscoredMemories(db) {
649
744
  const rows = db
650
745
  .prepare(`SELECT * FROM memories
651
- WHERE base_strength = 0.5 AND COALESCE(status, '') != 'absorbed'
746
+ WHERE importance_scored_at IS NULL AND COALESCE(status, '') != 'absorbed'
652
747
  ORDER BY ingested_at ASC`)
653
748
  .all();
654
749
  return rows.map(rowToMemory);
@@ -91,17 +91,17 @@ export interface TelemetryPayload {
91
91
  * `runConsolidation`'s status: "completed" | "skipped" | "failed", plus
92
92
  * "no_llm" when consolidation was skipped because no LLM was configured,
93
93
  * "throttled" (#246) when the run was skipped because the
94
- * `llmTokensPerMonth` fair-use cap was projected to be exceeded, and
95
- * "endpoint_down" (#337) when the pre-consolidation readiness probe failed
96
- * or the LLM circuit breaker was open after the run — a TRANSIENT state
97
- * (retried next run), never reported as "completed" even though the stages
98
- * fail soft.
94
+ * `llmTokensPerMonth` fair-use cap was projected to be exceeded,
95
+ * "endpoint_down" (#337) when the LLM circuit breaker was open after the
96
+ * run, and "deferred" (#405) when the run-wide wall-clock deadline fired —
97
+ * the latter two are TRANSIENT states (retried next run), never reported
98
+ * as "completed" even though the stages fail soft.
99
99
  * "skipped" = the built-in nothing-to-do short-circuit (no new + no unscored
100
100
  * memories → zero LLM calls), NOT a failure. Lets the fleet aggregate tell a
101
101
  * real consolidation run from a no-op without repurposing `ok` (which is the
102
102
  * capture-health signal). 0.17+.
103
103
  */
104
- consolidation?: "completed" | "skipped" | "failed" | "no_llm" | "throttled" | "endpoint_down";
104
+ consolidation?: "completed" | "skipped" | "failed" | "no_llm" | "throttled" | "endpoint_down" | "deferred";
105
105
  /**
106
106
  * Total LLM tokens consumed by THIS nightly's consolidation (#246) — the
107
107
  * BudgetTracker total. Server-mode only (capture-only + client runs make no
@@ -111,7 +111,8 @@ export interface TelemetryPayload {
111
111
  */
112
112
  tokens_this_run?: number;
113
113
  /**
114
- * True when the per-tenant consolidation budget (`consolidateMaxLlmCalls`)
114
+ * True when the per-tenant consolidation budget (`nightlyLlmCallBudget`,
115
+ * #405 — formerly consolidateMaxLlmCalls)
115
116
  * was exhausted this run (#255) — LLM-bound stages deferred remaining work.
116
117
  * A quality-degradation signal concentrated on heavy users; absent on a
117
118
  * pre-#255 ping, a capture-only/no-LLM/throttled/skipped run, or when the
@@ -105,10 +105,9 @@ function recordDistillUsage(stateDir, usage) {
105
105
  let periodStart = "";
106
106
  (0, state_js_1.updateState)((s) => {
107
107
  const prev = s.llmTokensThisPeriod;
108
- // Monthly reset (year+month) — matches shouldThrottleTokens's staleness check.
109
- const stale = !prev?.periodStart ||
110
- new Date(prev.periodStart).getUTCFullYear() !== new Date().getUTCFullYear() ||
111
- new Date(prev.periodStart).getUTCMonth() !== new Date().getUTCMonth();
108
+ // Monthly reset — the ONE staleness helper (#405; was a hand-rolled copy
109
+ // of shouldThrottleTokens's check).
110
+ const stale = (0, consolidate_js_1.isStaleTokenPeriod)(prev?.periodStart);
112
111
  if (stale) {
113
112
  s.llmTokensThisPeriod = {
114
113
  prompt: usage.prompt,