@gamaze/hicortex 0.20.7 → 0.20.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -41
- package/dist/calibration.d.ts +174 -0
- package/dist/calibration.js +231 -0
- package/dist/capture.d.ts +15 -3
- package/dist/capture.js +10 -1
- package/dist/classify-domains.d.ts +6 -0
- package/dist/classify-domains.js +7 -1
- package/dist/cli.js +2 -3
- package/dist/config-read.d.ts +1 -1
- package/dist/config-read.js +96 -9
- package/dist/consolidate.d.ts +79 -68
- package/dist/consolidate.js +218 -174
- package/dist/dashboard.d.ts +4 -3
- package/dist/dedup.d.ts +34 -26
- package/dist/dedup.js +91 -57
- package/dist/distiller.js +1 -1
- package/dist/domain-classify.d.ts +7 -6
- package/dist/domain-classify.js +12 -10
- package/dist/eval/decay-eval.d.ts +3 -3
- package/dist/eval/decay-eval.js +4 -4
- package/dist/eval/planted-eval.d.ts +26 -0
- package/dist/eval/planted-eval.js +97 -0
- package/dist/eval/planted-fixtures.d.ts +107 -0
- package/dist/eval/planted-fixtures.js +283 -0
- package/dist/eval/planted-harness.d.ts +176 -0
- package/dist/eval/planted-harness.js +649 -0
- package/dist/index.js +4 -3
- package/dist/init.d.ts +9 -3
- package/dist/init.js +52 -9
- package/dist/llm.d.ts +43 -58
- package/dist/llm.js +87 -101
- package/dist/mcp-server.js +29 -29
- package/dist/nightly.js +105 -103
- package/dist/nofit.d.ts +4 -11
- package/dist/nofit.js +6 -23
- package/dist/recall-index.d.ts +30 -28
- package/dist/recall-index.js +21 -18
- package/dist/recall-registry.d.ts +2 -1
- package/dist/recall-registry.js +35 -1
- package/dist/reconsolidation.d.ts +124 -72
- package/dist/reconsolidation.js +359 -148
- package/dist/relink.js +3 -4
- package/dist/retrieval.d.ts +68 -35
- package/dist/retrieval.js +292 -104
- package/dist/run-deadline.d.ts +62 -0
- package/dist/run-deadline.js +73 -0
- package/dist/schema-prototypes.d.ts +3 -3
- package/dist/schema-prototypes.js +3 -3
- package/dist/state.d.ts +2 -3
- package/dist/storage.d.ts +16 -16
- package/dist/storage.js +62 -24
- package/dist/telemetry.d.ts +8 -7
- package/dist/token-budget.js +3 -4
- package/dist/type-classify.js +4 -4
- package/dist/types.d.ts +95 -155
- package/domains.example.json +4 -5
- package/hermes-plugin/hicortex/README.md +2 -2
- package/openclaw.plugin.json +1 -1
- package/package.json +2 -1
- package/pi-extension/hicortex/README.md +1 -1
- package/server.json +3 -3
package/dist/retrieval.js
CHANGED
|
@@ -3,14 +3,15 @@
|
|
|
3
3
|
* Retrieval layer with composite scoring, RRF fusion, and graph traversal.
|
|
4
4
|
* Ported from hicortex/retrieval.py — same scoring model and weights.
|
|
5
5
|
*
|
|
6
|
-
* Scoring model (weights are
|
|
6
|
+
* Scoring model (weights are RELEASE-MANAGED since #408 — calibration.ts;
|
|
7
|
+
* configureScoring is the eval/test seam only):
|
|
7
8
|
* score = similarity * 0.50 + effective_strength * 0.20
|
|
8
9
|
* + connection_score * 0.15 + recency * 0.15
|
|
9
10
|
* + fresh-memory bonus (≤ 0.15, linear over the first 7 days)
|
|
10
11
|
* then × 0.50 if the memory was superseded by a later decision
|
|
11
12
|
*
|
|
12
13
|
* Decay model (B+E+D):
|
|
13
|
-
* base_decay = derived from
|
|
14
|
+
* base_decay = derived from the calibration half-life (365 → ~1-year
|
|
14
15
|
* half-life at importance 0.5, importance-scaled either way)
|
|
15
16
|
* decay_rate = 1 - base_decay * (1 - importance)
|
|
16
17
|
* decay_rate = 1 - (1 - decay_rate) * 0.7^access_count
|
|
@@ -52,7 +53,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
52
53
|
};
|
|
53
54
|
})();
|
|
54
55
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
55
|
-
exports.SESSION_INTENT_ALPHA = exports.DEFAULT_DECAY_HALF_LIFE_DAYS = void 0;
|
|
56
|
+
exports.BELIEF_WALK_MAX_HOPS = exports.SESSION_INTENT_ALPHA = exports.DEFAULT_DECAY_HALF_LIFE_DAYS = void 0;
|
|
56
57
|
exports.decayConstantForHalfLife = decayConstantForHalfLife;
|
|
57
58
|
exports.configureDecay = configureDecay;
|
|
58
59
|
exports.configureRecall = configureRecall;
|
|
@@ -64,7 +65,9 @@ exports.blendQueryVector = blendQueryVector;
|
|
|
64
65
|
exports.recallQueryVector = recallQueryVector;
|
|
65
66
|
exports.findSupersededIds = findSupersededIds;
|
|
66
67
|
exports.findDemotedIds = findDemotedIds;
|
|
68
|
+
exports.beliefWalkTerminal = beliefWalkTerminal;
|
|
67
69
|
exports.l2ToCosine = l2ToCosine;
|
|
70
|
+
exports.cosineBetweenVectors = cosineBetweenVectors;
|
|
68
71
|
exports.effectiveStrength = effectiveStrength;
|
|
69
72
|
exports.computeScore = computeScore;
|
|
70
73
|
exports.retrieve = retrieve;
|
|
@@ -72,11 +75,10 @@ exports.searchRecent = searchRecent;
|
|
|
72
75
|
const storage = __importStar(require("./storage.js"));
|
|
73
76
|
const schema_prototypes_js_1 = require("./schema-prototypes.js");
|
|
74
77
|
const type_labels_js_1 = require("./type-labels.js");
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
*
|
|
78
|
-
|
|
79
|
-
exports.DEFAULT_DECAY_HALF_LIFE_DAYS = 365;
|
|
78
|
+
const CALIBRATION = __importStar(require("./calibration.js"));
|
|
79
|
+
/** Default decay half-life (days) at importance 0.5 — release-managed
|
|
80
|
+
* (#408): the constant lives in calibration.ts with its provenance. */
|
|
81
|
+
exports.DEFAULT_DECAY_HALF_LIFE_DAYS = CALIBRATION.DECAY_HALF_LIFE_DAYS;
|
|
80
82
|
/**
|
|
81
83
|
* Derive the per-hour base decay constant from a half-life target: for the
|
|
82
84
|
* decayable portion, retention^hours = 0.5 at `days`, evaluated at the
|
|
@@ -89,110 +91,98 @@ function decayConstantForHalfLife(days) {
|
|
|
89
91
|
}
|
|
90
92
|
let BASE_DECAY = decayConstantForHalfLife(exports.DEFAULT_DECAY_HALF_LIFE_DAYS);
|
|
91
93
|
/**
|
|
92
|
-
* Configure the decay speed
|
|
93
|
-
*
|
|
94
|
-
*
|
|
94
|
+
* Configure the decay speed for THIS process (the eval/test seam — #408).
|
|
95
|
+
* Production NEVER passes an argument: every process scores with the
|
|
96
|
+
* calibration half-life (calibration.ts DECAY_HALF_LIFE_DAYS). An
|
|
97
|
+
* invalid/absent value keeps the default. Exported value for tests.
|
|
95
98
|
*/
|
|
96
|
-
function configureDecay(
|
|
97
|
-
const days = Number(
|
|
99
|
+
function configureDecay(halfLifeDays) {
|
|
100
|
+
const days = Number(halfLifeDays);
|
|
98
101
|
BASE_DECAY = decayConstantForHalfLife(Number.isFinite(days) && days > 0 ? days : exports.DEFAULT_DECAY_HALF_LIFE_DAYS);
|
|
99
102
|
return BASE_DECAY;
|
|
100
103
|
}
|
|
101
104
|
const RECALL_DEFAULTS = {
|
|
102
|
-
searchLimit:
|
|
103
|
-
recentLimit:
|
|
104
|
-
recentWindowDays:
|
|
105
|
-
coldExposureSlots:
|
|
105
|
+
searchLimit: CALIBRATION.SEARCH_LIMIT,
|
|
106
|
+
recentLimit: CALIBRATION.RECENT_LIMIT,
|
|
107
|
+
recentWindowDays: CALIBRATION.RECENT_WINDOW_DAYS,
|
|
108
|
+
coldExposureSlots: CALIBRATION.COLD_EXPOSURE_SLOTS,
|
|
106
109
|
};
|
|
107
110
|
let recallDefaults = { ...RECALL_DEFAULTS };
|
|
108
111
|
/**
|
|
109
|
-
* Configure recall breadth from
|
|
110
|
-
*
|
|
111
|
-
*
|
|
112
|
+
* Configure recall breadth from RESOLVED overrides (the eval/test seam —
|
|
113
|
+
* #408). Production calls this with no argument: the calibration defaults
|
|
114
|
+
* (calibration.ts) apply. Invalid/absent values keep the shipped default per
|
|
115
|
+
* key. Returns the resolved values (for logging + tests).
|
|
112
116
|
*/
|
|
113
|
-
function configureRecall(
|
|
114
|
-
const pick = (
|
|
115
|
-
const
|
|
116
|
-
return Number.isFinite(
|
|
117
|
+
function configureRecall(overrides) {
|
|
118
|
+
const pick = (v, dflt) => {
|
|
119
|
+
const n = Number(v);
|
|
120
|
+
return Number.isFinite(n) && n >= 0 ? Math.floor(n) : dflt;
|
|
117
121
|
};
|
|
118
122
|
recallDefaults = {
|
|
119
|
-
searchLimit: Math.max(1, pick(
|
|
120
|
-
recentLimit: Math.max(1, pick(
|
|
121
|
-
recentWindowDays: Math.max(1, pick(
|
|
122
|
-
coldExposureSlots: pick(
|
|
123
|
+
searchLimit: Math.max(1, pick(overrides?.searchLimit, RECALL_DEFAULTS.searchLimit)),
|
|
124
|
+
recentLimit: Math.max(1, pick(overrides?.recentLimit, RECALL_DEFAULTS.recentLimit)),
|
|
125
|
+
recentWindowDays: Math.max(1, pick(overrides?.recentWindowDays, RECALL_DEFAULTS.recentWindowDays)),
|
|
126
|
+
coldExposureSlots: pick(overrides?.coldExposureSlots, RECALL_DEFAULTS.coldExposureSlots),
|
|
123
127
|
};
|
|
124
128
|
return { ...recallDefaults };
|
|
125
129
|
}
|
|
126
130
|
const SCORING_DEFAULTS = {
|
|
127
|
-
similarity:
|
|
128
|
-
strength:
|
|
129
|
-
connections:
|
|
130
|
-
recency:
|
|
131
|
-
freshnessBoostDays:
|
|
132
|
-
freshnessBoostWeight:
|
|
133
|
-
supersededDemotion:
|
|
134
|
-
projectAffinity:
|
|
135
|
-
domainAffinity:
|
|
136
|
-
// #205
|
|
137
|
-
// values (60 and 0.8)
|
|
138
|
-
//
|
|
139
|
-
//
|
|
140
|
-
//
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
// focused-family "login/CORS/webhook" turns) keep recall@5 = 1.0.
|
|
146
|
-
rrfK: 60,
|
|
147
|
-
rrfCompositeWeight: 0.8,
|
|
148
|
-
rrfFtsWeight: 0.5,
|
|
149
|
-
rrfVectorWeight: 1.0,
|
|
131
|
+
similarity: CALIBRATION.SCORE_SIMILARITY_WEIGHT,
|
|
132
|
+
strength: CALIBRATION.SCORE_STRENGTH_WEIGHT,
|
|
133
|
+
connections: CALIBRATION.SCORE_CONNECTIONS_WEIGHT,
|
|
134
|
+
recency: CALIBRATION.SCORE_RECENCY_WEIGHT,
|
|
135
|
+
freshnessBoostDays: CALIBRATION.FRESHNESS_BOOST_DAYS,
|
|
136
|
+
freshnessBoostWeight: CALIBRATION.FRESHNESS_BOOST_WEIGHT,
|
|
137
|
+
supersededDemotion: CALIBRATION.SUPERSEDED_DEMOTION,
|
|
138
|
+
projectAffinity: CALIBRATION.PROJECT_AFFINITY_WEIGHT,
|
|
139
|
+
domainAffinity: CALIBRATION.DOMAIN_AFFINITY_WEIGHT,
|
|
140
|
+
// #205 (calibration.ts): rrfK + rrfCompositeWeight match the pre-#205
|
|
141
|
+
// hardcoded values (60 and 0.8); the FTS per-list weight (1.0 → 0.5) is the
|
|
142
|
+
// one deliberate nudge toward vector — the bisection point where BM25F +
|
|
143
|
+
// composite-affinity flip the token-exact marine body match below the
|
|
144
|
+
// same-scope hardware field while pure-keyword queries keep recall@5 = 1.0.
|
|
145
|
+
rrfK: CALIBRATION.RRF_K,
|
|
146
|
+
rrfCompositeWeight: CALIBRATION.RRF_COMPOSITE_WEIGHT,
|
|
147
|
+
rrfFtsWeight: CALIBRATION.RRF_FTS_WEIGHT,
|
|
148
|
+
rrfVectorWeight: CALIBRATION.RRF_VECTOR_WEIGHT,
|
|
150
149
|
};
|
|
151
150
|
let scoringWeights = { ...SCORING_DEFAULTS };
|
|
152
151
|
/**
|
|
153
|
-
* Configure scoring weights + ranking knobs from
|
|
154
|
-
*
|
|
155
|
-
*
|
|
156
|
-
*
|
|
157
|
-
*
|
|
158
|
-
*
|
|
159
|
-
*
|
|
160
|
-
* SAME config object for one-place tuning.
|
|
152
|
+
* Configure scoring weights + ranking knobs from RESOLVED overrides (the
|
|
153
|
+
* eval/test seam — #408). Production calls this with no argument: the
|
|
154
|
+
* calibration defaults (calibration.ts) apply, identically in the daemon and
|
|
155
|
+
* the nightly. Invalid/absent values keep the shipped default per key.
|
|
156
|
+
* Returns the resolved set for logging/tests. (The #205 BM25F field weights
|
|
157
|
+
* are NOT touched here — they live in storage.ts and resolve from the same
|
|
158
|
+
* calibration module via storage.configureBm25Fts.)
|
|
161
159
|
*/
|
|
162
|
-
function configureScoring(
|
|
163
|
-
const num = (
|
|
164
|
-
const
|
|
165
|
-
return Number.isFinite(
|
|
160
|
+
function configureScoring(overrides) {
|
|
161
|
+
const num = (v, dflt, min, max) => {
|
|
162
|
+
const n = Number(v);
|
|
163
|
+
return Number.isFinite(n) && n >= min && n <= max ? n : dflt;
|
|
166
164
|
};
|
|
167
|
-
// #205 BM25F weights use a [0, ∞) range (no upper bound
|
|
168
|
-
//
|
|
169
|
-
const numW = (
|
|
170
|
-
const
|
|
171
|
-
return Number.isFinite(
|
|
165
|
+
// #205 BM25F-style weights use a [0, ∞) range (no upper bound; 0 drops the
|
|
166
|
+
// field/list entirely). Invalid ⇒ default.
|
|
167
|
+
const numW = (v, dflt) => {
|
|
168
|
+
const n = Number(v);
|
|
169
|
+
return Number.isFinite(n) && n >= 0 ? n : dflt;
|
|
172
170
|
};
|
|
173
171
|
scoringWeights = {
|
|
174
|
-
similarity: num(
|
|
175
|
-
strength: num(
|
|
176
|
-
connections: num(
|
|
177
|
-
recency: num(
|
|
178
|
-
freshnessBoostDays: num(
|
|
179
|
-
freshnessBoostWeight: num(
|
|
180
|
-
supersededDemotion: num(
|
|
181
|
-
projectAffinity: num(
|
|
182
|
-
domainAffinity: num(
|
|
183
|
-
rrfK: numW(
|
|
184
|
-
rrfCompositeWeight: num(
|
|
185
|
-
rrfFtsWeight: numW(
|
|
186
|
-
rrfVectorWeight: numW(
|
|
172
|
+
similarity: num(overrides?.similarity, SCORING_DEFAULTS.similarity, 0, 1),
|
|
173
|
+
strength: num(overrides?.strength, SCORING_DEFAULTS.strength, 0, 1),
|
|
174
|
+
connections: num(overrides?.connections, SCORING_DEFAULTS.connections, 0, 1),
|
|
175
|
+
recency: num(overrides?.recency, SCORING_DEFAULTS.recency, 0, 1),
|
|
176
|
+
freshnessBoostDays: num(overrides?.freshnessBoostDays, SCORING_DEFAULTS.freshnessBoostDays, 0, 365),
|
|
177
|
+
freshnessBoostWeight: num(overrides?.freshnessBoostWeight, SCORING_DEFAULTS.freshnessBoostWeight, 0, 1),
|
|
178
|
+
supersededDemotion: num(overrides?.supersededDemotion, SCORING_DEFAULTS.supersededDemotion, 0, 1),
|
|
179
|
+
projectAffinity: num(overrides?.projectAffinity, SCORING_DEFAULTS.projectAffinity, 0, 1),
|
|
180
|
+
domainAffinity: num(overrides?.domainAffinity, SCORING_DEFAULTS.domainAffinity, 0, 1),
|
|
181
|
+
rrfK: numW(overrides?.rrfK, SCORING_DEFAULTS.rrfK),
|
|
182
|
+
rrfCompositeWeight: num(overrides?.rrfCompositeWeight, SCORING_DEFAULTS.rrfCompositeWeight, 0, 1),
|
|
183
|
+
rrfFtsWeight: numW(overrides?.rrfFtsWeight, SCORING_DEFAULTS.rrfFtsWeight),
|
|
184
|
+
rrfVectorWeight: numW(overrides?.rrfVectorWeight, SCORING_DEFAULTS.rrfVectorWeight),
|
|
187
185
|
};
|
|
188
|
-
// #205: push BM25F field weights into storage so searchFts uses them. Same
|
|
189
|
-
// config object, one tuning surface; storage owns the module-level mirror
|
|
190
|
-
// next to the FTS column declaration (the positional order matters there).
|
|
191
|
-
storage.configureBm25Fts({
|
|
192
|
-
bm25WeightBody: numW("bm25WeightBody", 1.0),
|
|
193
|
-
bm25WeightProject: numW("bm25WeightProject", 2.0),
|
|
194
|
-
bm25WeightDomain: numW("bm25WeightDomain", 2.0),
|
|
195
|
-
});
|
|
196
186
|
return { ...scoringWeights };
|
|
197
187
|
}
|
|
198
188
|
/** Current resolved weights (tests + status output). */
|
|
@@ -200,11 +190,11 @@ function getScoringWeights() {
|
|
|
200
190
|
return { ...scoringWeights };
|
|
201
191
|
}
|
|
202
192
|
// ---------------------------------------------------------------------------
|
|
203
|
-
// Session-intent keying (#192, 0.15.3). ONE
|
|
204
|
-
//
|
|
205
|
-
//
|
|
206
|
-
//
|
|
207
|
-
//
|
|
193
|
+
// Session-intent keying (#192, 0.15.3). ONE calibration constant (#408):
|
|
194
|
+
// SESSION_INTENT_WEIGHT 0.33 blend weight of the rolling centroid in the
|
|
195
|
+
// search vector: query = (1-w)·prompt + w·centroid.
|
|
196
|
+
// configureSessionIntent(0) is the eval-only
|
|
197
|
+
// kill-switch (pure prompt). Range [0, 1].
|
|
208
198
|
//
|
|
209
199
|
// The EMA rate α is a shipped constant (SESSION_INTENT_ALPHA, 0.4), not a
|
|
210
200
|
// second knob — owner directive 0.15.3: one knob is enough to tune/disable;
|
|
@@ -218,18 +208,18 @@ function getScoringWeights() {
|
|
|
218
208
|
// ---------------------------------------------------------------------------
|
|
219
209
|
/** EMA rate for the session-intent centroid: centroid_new = (1-α)·old + α·prompt. */
|
|
220
210
|
exports.SESSION_INTENT_ALPHA = 0.4;
|
|
221
|
-
const SESSION_INTENT_DEFAULT_WEIGHT =
|
|
211
|
+
const SESSION_INTENT_DEFAULT_WEIGHT = CALIBRATION.SESSION_INTENT_WEIGHT;
|
|
222
212
|
let sessionIntentWeight = SESSION_INTENT_DEFAULT_WEIGHT;
|
|
223
213
|
/**
|
|
224
|
-
* Configure session-intent keying
|
|
225
|
-
*
|
|
226
|
-
*
|
|
227
|
-
*
|
|
228
|
-
*
|
|
229
|
-
* one call.
|
|
214
|
+
* Configure session-intent keying for THIS process (the eval/test seam —
|
|
215
|
+
* #408). Production calls this with no argument: the calibration weight
|
|
216
|
+
* (calibration.ts SESSION_INTENT_WEIGHT) applies. `weight` is [0,1] (0 =
|
|
217
|
+
* disabled — the eval kill-switch); invalid/out-of-range values keep the
|
|
218
|
+
* shipped default. Returns `{ weight, alpha }` — alpha is the fixed constant,
|
|
219
|
+
* surfaced so the recall closure passes it to the registry in one call.
|
|
230
220
|
*/
|
|
231
|
-
function configureSessionIntent(
|
|
232
|
-
const v = Number(
|
|
221
|
+
function configureSessionIntent(weight) {
|
|
222
|
+
const v = Number(weight);
|
|
233
223
|
sessionIntentWeight =
|
|
234
224
|
Number.isFinite(v) && v >= 0 && v <= 1 ? v : SESSION_INTENT_DEFAULT_WEIGHT;
|
|
235
225
|
return { weight: sessionIntentWeight, alpha: exports.SESSION_INTENT_ALPHA };
|
|
@@ -305,6 +295,123 @@ function findDemotedIds(db, candidateIds) {
|
|
|
305
295
|
.all(...candidateIds, ...candidateIds);
|
|
306
296
|
return new Set(rows.map((r) => r.id));
|
|
307
297
|
}
|
|
298
|
+
// ---------------------------------------------------------------------------
|
|
299
|
+
// Belief walk (#393 increment D)
|
|
300
|
+
// ---------------------------------------------------------------------------
|
|
301
|
+
/**
|
|
302
|
+
* Hop cap for the belief walk (#393 D). Supersession edges advance
|
|
303
|
+
* created_at monotonically (the stage only links old → new), so chains are
|
|
304
|
+
* acyclic by construction and 10 hops is far beyond any real revision depth;
|
|
305
|
+
* the cap is cheap insurance (see beliefWalkTerminal for why it is needed
|
|
306
|
+
* anyway).
|
|
307
|
+
*/
|
|
308
|
+
exports.BELIEF_WALK_MAX_HOPS = 10;
|
|
309
|
+
/**
|
|
310
|
+
* The one outgoing supersession edge to follow from `id` (#393 D): when a
|
|
311
|
+
* memory carries several `superseded_by` edges the NEWEST target by
|
|
312
|
+
* created_at wins (deterministic target_id tie-break), null when there is
|
|
313
|
+
* none. Indexed by idx_links_source; one row read.
|
|
314
|
+
*/
|
|
315
|
+
function nextSupersedingId(db, id) {
|
|
316
|
+
const row = db
|
|
317
|
+
.prepare(`SELECT ml.target_id AS target_id
|
|
318
|
+
FROM memory_links ml
|
|
319
|
+
JOIN memories m ON m.id = ml.target_id
|
|
320
|
+
WHERE ml.source_id = ? AND ml.relationship = 'superseded_by'
|
|
321
|
+
ORDER BY m.created_at DESC, ml.target_id ASC
|
|
322
|
+
LIMIT 1`)
|
|
323
|
+
.get(id);
|
|
324
|
+
return row ? row.target_id : null;
|
|
325
|
+
}
|
|
326
|
+
/**
|
|
327
|
+
* Terminal of the supersession chain starting at `id` (#393 D): follow
|
|
328
|
+
* superseded_by edges transitively until a memory with no outgoing edge and
|
|
329
|
+
* return it — `id` itself when there is nothing to walk, or whichever node
|
|
330
|
+
* the walk stopped on when it aborts. Shared by retrieval (the belief-walk
|
|
331
|
+
* splice in retrieve()/searchRecent()) and the eval harness (the
|
|
332
|
+
* planted-pairs version_chain class probe), so both agree on what "the
|
|
333
|
+
* chain's current truth" is.
|
|
334
|
+
*
|
|
335
|
+
* Edges advance created_at monotonically (acyclic by construction), BUT
|
|
336
|
+
* applyExplicitMark does no age check and created_at is backdatable from
|
|
337
|
+
* session_date — so a cycle or an absurdly long chain is not impossible.
|
|
338
|
+
* The visited set (seeded with `id`) and the BELIEF_WALK_MAX_HOPS cap are
|
|
339
|
+
* cheap insurance against exactly that; the supersededDemotion multiplier
|
|
340
|
+
* in computeScore remains the safety net for rows the walk does not fully
|
|
341
|
+
* resolve (no edge, cycle, cap abort, absorbed terminal).
|
|
342
|
+
*/
|
|
343
|
+
function beliefWalkTerminal(db, id, maxHops = exports.BELIEF_WALK_MAX_HOPS) {
|
|
344
|
+
const visited = new Set([id]);
|
|
345
|
+
let current = id;
|
|
346
|
+
for (let hop = 0; hop < maxHops; hop++) {
|
|
347
|
+
const next = nextSupersedingId(db, current);
|
|
348
|
+
if (next === null || visited.has(next))
|
|
349
|
+
return current;
|
|
350
|
+
visited.add(next);
|
|
351
|
+
current = next;
|
|
352
|
+
}
|
|
353
|
+
return current;
|
|
354
|
+
}
|
|
355
|
+
/**
|
|
356
|
+
* The belief-walk splice (#393 D) — the retrieval-side half of "newest wins
|
|
357
|
+
* by construction". Applied to the FINAL top-k of retrieve()/searchRecent(),
|
|
358
|
+
* after the sort and after the cold-exposure splice: a superseded candidate
|
|
359
|
+
* is replaced IN ITS SLOT by its chain's terminal (beliefWalkTerminal), so a
|
|
360
|
+
* strong stale record can never outrank — or appear alongside — the truth
|
|
361
|
+
* that replaced it. The ancestor stays fetchable by id (evidence) but never
|
|
362
|
+
* surfaces as a competing truth in recall.
|
|
363
|
+
*
|
|
364
|
+
* Per entry of `top`, in order:
|
|
365
|
+
* - not superseded → kept untouched (fast path, zero queries)
|
|
366
|
+
* - terminal === entry → kept (no outgoing edge — the entry is its
|
|
367
|
+
* own terminal; a self-loop aborts here too.
|
|
368
|
+
* A hop-cap or non-self-cycle abort stops the
|
|
369
|
+
* walk on a DIFFERENT node, so the entry
|
|
370
|
+
* falls through to the bullets below — e.g.
|
|
371
|
+
* a 2-cycle with both members in top drops
|
|
372
|
+
* both, conservative but nothing stale
|
|
373
|
+
* surfaces (corrupt-data corner only; edges
|
|
374
|
+
* advance created_at, acyclic by
|
|
375
|
+
* construction). supersededDemotion in
|
|
376
|
+
* computeScore stays the safety net for rows
|
|
377
|
+
* the walk does not resolve)
|
|
378
|
+
* - terminal already surfaced → the entry is DROPPED (its truth is present)
|
|
379
|
+
* - otherwise → buildReplacement(terminal); null (terminal
|
|
380
|
+
* unfetchable or absorbed — absorbed rows
|
|
381
|
+
* are invisible to recall by contract) keeps
|
|
382
|
+
* the ancestor in-slot, fail-soft
|
|
383
|
+
*
|
|
384
|
+
* The replacement takes the ancestor's SLOT (position), not its score — no
|
|
385
|
+
* re-sort after the splice; ranking remains the sort's verdict. Idempotent
|
|
386
|
+
* by construction: walk-stable and unsuperseded entries are returned as-is.
|
|
387
|
+
*/
|
|
388
|
+
function applyBeliefWalk(db, top, supersededIds, buildReplacement) {
|
|
389
|
+
const present = new Set(top.map((t) => t.mem.id));
|
|
390
|
+
const out = [];
|
|
391
|
+
for (const entry of top) {
|
|
392
|
+
if (!supersededIds.has(entry.mem.id)) {
|
|
393
|
+
out.push(entry);
|
|
394
|
+
continue;
|
|
395
|
+
}
|
|
396
|
+
const terminal = beliefWalkTerminal(db, entry.mem.id);
|
|
397
|
+
if (terminal === entry.mem.id) {
|
|
398
|
+
out.push(entry);
|
|
399
|
+
continue;
|
|
400
|
+
}
|
|
401
|
+
if (present.has(terminal)) {
|
|
402
|
+
// The chain's truth is already surfaced — drop the ancestor.
|
|
403
|
+
continue;
|
|
404
|
+
}
|
|
405
|
+
const replacement = buildReplacement(terminal);
|
|
406
|
+
if (replacement === null) {
|
|
407
|
+
out.push(entry);
|
|
408
|
+
continue;
|
|
409
|
+
}
|
|
410
|
+
out.push(replacement);
|
|
411
|
+
present.add(terminal);
|
|
412
|
+
}
|
|
413
|
+
return out;
|
|
414
|
+
}
|
|
308
415
|
/**
|
|
309
416
|
* Placeholder L2 distance for candidates that have no measured vector
|
|
310
417
|
* distance (FTS-only hits and graph-discovered neighbors). Chosen so that
|
|
@@ -332,6 +439,28 @@ const DEFAULT_RRF_K = 60;
|
|
|
332
439
|
function l2ToCosine(distance) {
|
|
333
440
|
return 1 - (distance * distance) / 2;
|
|
334
441
|
}
|
|
442
|
+
/**
|
|
443
|
+
* Cosine similarity between two stored embeddings (#393 increment B). The
|
|
444
|
+
* similarity source measures cosines transitively via vec0 L2 distances; the
|
|
445
|
+
* scout source finds its candidates through FTS (no vec0 query), so it
|
|
446
|
+
* measures the pair cosine directly from the stored vectors instead —
|
|
447
|
+
* valid because every embedding we store is L2-normalized (embedder.ts).
|
|
448
|
+
* Used as link strength / a ranker, never as a gate (the scout has no
|
|
449
|
+
* similarity floor — that is the point of the increment).
|
|
450
|
+
*/
|
|
451
|
+
function cosineBetweenVectors(a, b) {
|
|
452
|
+
let dot = 0;
|
|
453
|
+
let na = 0;
|
|
454
|
+
let nb = 0;
|
|
455
|
+
for (let i = 0; i < a.length; i++) {
|
|
456
|
+
dot += a[i] * b[i];
|
|
457
|
+
na += a[i] * a[i];
|
|
458
|
+
nb += b[i] * b[i];
|
|
459
|
+
}
|
|
460
|
+
if (na === 0 || nb === 0)
|
|
461
|
+
return 0;
|
|
462
|
+
return dot / Math.sqrt(na * nb);
|
|
463
|
+
}
|
|
335
464
|
// ---------------------------------------------------------------------------
|
|
336
465
|
// Timestamp parsing
|
|
337
466
|
// ---------------------------------------------------------------------------
|
|
@@ -715,6 +844,38 @@ async function retrieve(db, embedFn, query, options) {
|
|
|
715
844
|
}
|
|
716
845
|
}
|
|
717
846
|
}
|
|
847
|
+
// 6b. Belief walk (#393 D) — a superseded top-k member is replaced in-slot
|
|
848
|
+
// by its chain's terminal. A terminal already in `scored` reuses its honest
|
|
849
|
+
// entry (similarity/RRF/score); one outside the candidate set is computed
|
|
850
|
+
// fresh with source "graph" and RRF share 0 (it entered by LINK, not by any
|
|
851
|
+
// retrieval channel — finalScore = composite × rrfCompositeWeight only).
|
|
852
|
+
// The strengthen() below then fires on the SURFACED set: the terminal
|
|
853
|
+
// accrues the use signal, the dropped ancestor does not. supersededDemotion
|
|
854
|
+
// in computeScore stays as the safety net for rows the walk does not reach.
|
|
855
|
+
const scoredById = new Map(scored.map((s) => [s.mem.id, s]));
|
|
856
|
+
top = applyBeliefWalk(db, top, supersededIds, (terminalId) => {
|
|
857
|
+
const existing = scoredById.get(terminalId);
|
|
858
|
+
if (existing)
|
|
859
|
+
return existing;
|
|
860
|
+
const mem = storage.getMemory(db, terminalId);
|
|
861
|
+
if (!mem || mem.status === "absorbed")
|
|
862
|
+
return null; // fail-soft
|
|
863
|
+
// Hard-filter invariant (#393 D): sourceAgent is the one hard filter left
|
|
864
|
+
// in retrieve(), and the walk must not bypass it — a terminal authored by
|
|
865
|
+
// a different agent never slips into recall through its link entry
|
|
866
|
+
// (mirrors the graph pull-in guard above; fail-soft: the ancestor keeps
|
|
867
|
+
// its slot, demoted).
|
|
868
|
+
if (sourceAgent && mem.source_agent !== sourceAgent)
|
|
869
|
+
return null;
|
|
870
|
+
const connCount = storage.getLinks(db, terminalId, "both").length;
|
|
871
|
+
const composite = computeScore(mem, DEFAULT_GRAPH_DISTANCE, connCount, maxConnections, now, { superseded: supersededIds.has(terminalId) });
|
|
872
|
+
const finalScore = composite * scoringWeights.rrfCompositeWeight;
|
|
873
|
+
const effStr = effectiveStrength(mem.base_strength ?? 0.5, mem.last_accessed, now, {
|
|
874
|
+
accessCount: mem.access_count ?? 0,
|
|
875
|
+
linkCount: connCount,
|
|
876
|
+
});
|
|
877
|
+
return { mem, finalScore, effStr, connCount, similarity: null, source: "graph" };
|
|
878
|
+
});
|
|
718
879
|
const results = top.map((t) => formatResult(t.mem, t.finalScore, t.effStr, t.connCount, {
|
|
719
880
|
similarity: t.similarity,
|
|
720
881
|
source: t.source,
|
|
@@ -765,7 +926,34 @@ function searchRecent(db, options) {
|
|
|
765
926
|
scored.push({ mem, score, effStr, connCount });
|
|
766
927
|
}
|
|
767
928
|
scored.sort((a, b) => b.score - a.score);
|
|
768
|
-
|
|
929
|
+
let top = scored.slice(0, limit);
|
|
930
|
+
// Belief walk (#393 D) — same contract as retrieve(): a superseded recent
|
|
931
|
+
// memory is replaced in-slot by its chain terminal (this path has no
|
|
932
|
+
// similarity/RRF channels, so a fresh replacement carries the composite
|
|
933
|
+
// score only). The trailing strengthen() runs over the WALKED list — the
|
|
934
|
+
// terminal accrues the use signal, the dropped ancestor does not.
|
|
935
|
+
// supersededDemotion in computeScore stays as the safety net for rows the
|
|
936
|
+
// walk does not reach (no edge, absorbed terminal, cycles/caps).
|
|
937
|
+
top = applyBeliefWalk(db, top, supersededRecent, (terminalId) => {
|
|
938
|
+
const mem = storage.getMemory(db, terminalId);
|
|
939
|
+
if (!mem || mem.status === "absorbed")
|
|
940
|
+
return null; // fail-soft
|
|
941
|
+
// Hard-filter invariant (#393 D, review fix): project is the one hard
|
|
942
|
+
// filter in searchRecent(), and the walk must not bypass it — supersession
|
|
943
|
+
// edges are discovered corpus-wide (no project predicate), so a terminal
|
|
944
|
+
// in a different project never slips into project-scoped recall through
|
|
945
|
+
// its link entry (mirrors retrieve()'s sourceAgent guard; fail-soft: the
|
|
946
|
+
// ancestor keeps its slot, demoted).
|
|
947
|
+
if (project && mem.project !== project)
|
|
948
|
+
return null;
|
|
949
|
+
const connCount = storage.getLinks(db, terminalId, "both").length;
|
|
950
|
+
const score = computeScore(mem, DEFAULT_GRAPH_DISTANCE, connCount, maxConnections, now, { superseded: supersededRecent.has(terminalId) });
|
|
951
|
+
const effStr = effectiveStrength(mem.base_strength ?? 0.5, mem.last_accessed, now, {
|
|
952
|
+
accessCount: mem.access_count ?? 0,
|
|
953
|
+
linkCount: connCount,
|
|
954
|
+
});
|
|
955
|
+
return { mem, score, effStr, connCount };
|
|
956
|
+
});
|
|
769
957
|
const results = top.map((t) => formatResult(t.mem, t.score, t.effStr, t.connCount));
|
|
770
958
|
strengthen(db, top.map((t) => t.mem), now);
|
|
771
959
|
return results;
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The ONE cooperative wall-clock deadline for a nightly run (#405).
|
|
3
|
+
*
|
|
4
|
+
* Generalizes the #401/#402/#404 reconsolidation pattern (a deadline checked
|
|
5
|
+
* at safe boundaries + resumable cursors) from one stage to the whole
|
|
6
|
+
* pipeline: capture segments, every consolidation stage boundary, the
|
|
7
|
+
* item loops, and the deterministic merge zone all check the SAME deadline,
|
|
8
|
+
* created once at nightly start. On expiry each check site stops cleanly at
|
|
9
|
+
* its last safe boundary — cursors (capture per-session, supersession,
|
|
10
|
+
* reconsolidation) hold below unconfirmed work, so the next run resumes
|
|
11
|
+
* without redoing confirmed work or losing deferred work.
|
|
12
|
+
*
|
|
13
|
+
* Deferral is a REPORTED outcome, not an error: a run whose deadline fired
|
|
14
|
+
* reports consolidation status "deferred" and does NOT advance
|
|
15
|
+
* `lastConsolidated` (the same gate as endpoint_down — otherwise the
|
|
16
|
+
* pending-set queries would silently lose the deferred work).
|
|
17
|
+
*
|
|
18
|
+
* One structured log line per deferred stage (`event=deadline_deferred`,
|
|
19
|
+
* the same key=value style as `event=budget_exhausted` /
|
|
20
|
+
* `event=circuit_open`) so `grep event=deadline_deferred` is one of the two
|
|
21
|
+
* nightly health greps.
|
|
22
|
+
*/
|
|
23
|
+
/** Default wall-clock budget for a full nightly run, in minutes (#405).
|
|
24
|
+
* 240 sits between the old stage-local reconsolidation clock (120) and the
|
|
25
|
+
* old systemd backstop (360) — the deadline is now the operating bound and
|
|
26
|
+
* the unit backstop (budget + 60 min slack) only catches a true hang. */
|
|
27
|
+
export declare const DEFAULT_NIGHTLY_TIME_BUDGET_MINUTES = 240;
|
|
28
|
+
/**
|
|
29
|
+
* Cooperative deadline handle. Injectable clock (`now`) so tests can pin or
|
|
30
|
+
* advance time deterministically (the reconsolidation.test.ts 0.000001-minute
|
|
31
|
+
* pattern). All methods are safe to call after expiry — they keep answering.
|
|
32
|
+
*/
|
|
33
|
+
export interface RunDeadline {
|
|
34
|
+
/** Absolute epoch-ms timestamp the run must finish by. */
|
|
35
|
+
readonly deadlineAt: number;
|
|
36
|
+
/** Milliseconds left until the deadline (never negative). */
|
|
37
|
+
remainingMs(): number;
|
|
38
|
+
/** True once the clock has passed the deadline. */
|
|
39
|
+
expired(): boolean;
|
|
40
|
+
/**
|
|
41
|
+
* Check + record in one call: returns true when the deadline has fired,
|
|
42
|
+
* logging `event=deadline_deferred stage=<name>` ONCE per stage name (a
|
|
43
|
+
* boundary check and in-loop checks for the same stage log once). Use the
|
|
44
|
+
* same stage name at a stage's boundary and inside its loops.
|
|
45
|
+
*/
|
|
46
|
+
hit(stage: string): boolean;
|
|
47
|
+
/** Stage names that deferred so far this run (insertion order). */
|
|
48
|
+
deferredStages(): readonly string[];
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Create a run deadline of `minutes` (already-resolved value; see
|
|
52
|
+
* resolveNightlyTimeBudgetMinutes for the config boundary).
|
|
53
|
+
*/
|
|
54
|
+
export declare function createRunDeadline(minutes: number, now?: () => number): RunDeadline;
|
|
55
|
+
/**
|
|
56
|
+
* Resolve `nightlyTimeBudgetMinutes` from the saved config (#405): positive
|
|
57
|
+
* finite number wins; absent/0/invalid → the 240 default. Unlike the old
|
|
58
|
+
* `reconsolidationMaxMinutes`, 0 does NOT disable — there is ALWAYS a
|
|
59
|
+
* deadline (readPositiveConfig rejects 0 with a warn, which is the loud
|
|
60
|
+
* migration signal for an operator who used 0 as "off").
|
|
61
|
+
*/
|
|
62
|
+
export declare function resolveNightlyTimeBudgetMinutes(config: Record<string, unknown> | null | undefined): number;
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* The ONE cooperative wall-clock deadline for a nightly run (#405).
|
|
4
|
+
*
|
|
5
|
+
* Generalizes the #401/#402/#404 reconsolidation pattern (a deadline checked
|
|
6
|
+
* at safe boundaries + resumable cursors) from one stage to the whole
|
|
7
|
+
* pipeline: capture segments, every consolidation stage boundary, the
|
|
8
|
+
* item loops, and the deterministic merge zone all check the SAME deadline,
|
|
9
|
+
* created once at nightly start. On expiry each check site stops cleanly at
|
|
10
|
+
* its last safe boundary — cursors (capture per-session, supersession,
|
|
11
|
+
* reconsolidation) hold below unconfirmed work, so the next run resumes
|
|
12
|
+
* without redoing confirmed work or losing deferred work.
|
|
13
|
+
*
|
|
14
|
+
* Deferral is a REPORTED outcome, not an error: a run whose deadline fired
|
|
15
|
+
* reports consolidation status "deferred" and does NOT advance
|
|
16
|
+
* `lastConsolidated` (the same gate as endpoint_down — otherwise the
|
|
17
|
+
* pending-set queries would silently lose the deferred work).
|
|
18
|
+
*
|
|
19
|
+
* One structured log line per deferred stage (`event=deadline_deferred`,
|
|
20
|
+
* the same key=value style as `event=budget_exhausted` /
|
|
21
|
+
* `event=circuit_open`) so `grep event=deadline_deferred` is one of the two
|
|
22
|
+
* nightly health greps.
|
|
23
|
+
*/
|
|
24
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
25
|
+
exports.DEFAULT_NIGHTLY_TIME_BUDGET_MINUTES = void 0;
|
|
26
|
+
exports.createRunDeadline = createRunDeadline;
|
|
27
|
+
exports.resolveNightlyTimeBudgetMinutes = resolveNightlyTimeBudgetMinutes;
|
|
28
|
+
const config_read_js_1 = require("./config-read.js");
|
|
29
|
+
/** Default wall-clock budget for a full nightly run, in minutes (#405).
|
|
30
|
+
* 240 sits between the old stage-local reconsolidation clock (120) and the
|
|
31
|
+
* old systemd backstop (360) — the deadline is now the operating bound and
|
|
32
|
+
* the unit backstop (budget + 60 min slack) only catches a true hang. */
|
|
33
|
+
exports.DEFAULT_NIGHTLY_TIME_BUDGET_MINUTES = 240;
|
|
34
|
+
/**
|
|
35
|
+
* Create a run deadline of `minutes` (already-resolved value; see
|
|
36
|
+
* resolveNightlyTimeBudgetMinutes for the config boundary).
|
|
37
|
+
*/
|
|
38
|
+
function createRunDeadline(minutes, now = Date.now) {
|
|
39
|
+
const start = now();
|
|
40
|
+
const deadlineAt = start + Math.max(0, Math.round(minutes * 60_000));
|
|
41
|
+
const logged = new Set();
|
|
42
|
+
const deferred = [];
|
|
43
|
+
return {
|
|
44
|
+
deadlineAt,
|
|
45
|
+
remainingMs: () => Math.max(0, deadlineAt - now()),
|
|
46
|
+
expired: () => now() >= deadlineAt,
|
|
47
|
+
hit(stage) {
|
|
48
|
+
if (now() < deadlineAt)
|
|
49
|
+
return false;
|
|
50
|
+
if (!logged.has(stage)) {
|
|
51
|
+
logged.add(stage);
|
|
52
|
+
deferred.push(stage);
|
|
53
|
+
// Structured (grep-able) + human-readable — same contract as
|
|
54
|
+
// event=budget_exhausted (consolidate.ts) and event=circuit_open
|
|
55
|
+
// (llm.ts). remaining_ms=0 states the reason plainly.
|
|
56
|
+
console.warn(`[hicortex] event=deadline_deferred stage=${stage} ` +
|
|
57
|
+
`deadline_minutes=${(deadlineAt - start) / 60_000} remaining_ms=0`);
|
|
58
|
+
}
|
|
59
|
+
return true;
|
|
60
|
+
},
|
|
61
|
+
deferredStages: () => deferred,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Resolve `nightlyTimeBudgetMinutes` from the saved config (#405): positive
|
|
66
|
+
* finite number wins; absent/0/invalid → the 240 default. Unlike the old
|
|
67
|
+
* `reconsolidationMaxMinutes`, 0 does NOT disable — there is ALWAYS a
|
|
68
|
+
* deadline (readPositiveConfig rejects 0 with a warn, which is the loud
|
|
69
|
+
* migration signal for an operator who used 0 as "off").
|
|
70
|
+
*/
|
|
71
|
+
function resolveNightlyTimeBudgetMinutes(config) {
|
|
72
|
+
return (0, config_read_js_1.readPositiveConfig)(config ?? {}, "nightlyTimeBudgetMinutes", exports.DEFAULT_NIGHTLY_TIME_BUDGET_MINUTES);
|
|
73
|
+
}
|