@gamaze/hicortex 0.15.3 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -60,12 +60,28 @@ interface ScoringWeights {
60
60
  freshnessBoostDays: number;
61
61
  freshnessBoostWeight: number;
62
62
  supersededDemotion: number;
63
+ /** #203 soft affinity boost on exact project match. */
64
+ projectAffinity: number;
65
+ /** #203 soft affinity boost multiplier on max overlapping domain-tag weight. */
66
+ domainAffinity: number;
67
+ /** #205 RRF k parameter (1/(k+rank+1)). Larger ⇒ shallower rank curve. */
68
+ rrfK: number;
69
+ /** #205 composite-score share of the final blend (RRF gets the remainder). */
70
+ rrfCompositeWeight: number;
71
+ /** #205 per-list RRF weight for the FTS list (BM25-driven candidates). */
72
+ rrfFtsWeight: number;
73
+ /** #205 per-list RRF weight for the vector list (KNN-driven candidates). */
74
+ rrfVectorWeight: number;
63
75
  }
64
76
  /**
65
77
  * Configure scoring weights + ranking knobs from config. Called at boot by the
66
78
  * server and the nightly (alongside configureDecay/configureRecall) so
67
79
  * retrieval and consolidation rank identically. Invalid/absent values keep the
68
- * shipped default per key. Returns the resolved set for logging/tests.
80
+ * shipped default per key. Returns the resolved set for logging/tests. Also
81
+ * pushes the #205 BM25F field weights into storage (storage.configureBm25Fts)
82
+ * so searchFts ranks with the same config — BM25F weights live in storage.ts
83
+ * (next to the FTS column declaration they mirror) but are read here from the
84
+ * SAME config object for one-place tuning.
69
85
  */
70
86
  export declare function configureScoring(config?: Record<string, unknown> | null): ScoringWeights;
71
87
  /** Current resolved weights (tests + status output). */
@@ -127,9 +143,31 @@ export declare function effectiveStrength(baseStrength: number, lastAccessed: st
127
143
  /**
128
144
  * Return a composite relevance score in [0, 1] for a candidate memory.
129
145
  * Exported for exact-value tests of the similarity component (#145).
146
+ *
147
+ * #203 soft affinity (options.scope + options.tagWeights): two additive,
148
+ * graded, zero-boost-neutral terms — project affinity (exact project match)
149
+ * and domain affinity (max overlapping memory_tags.weight × scope). Both are
150
+ * 0 when the scope is absent (byte-identical to pre-#203) and NEVER negative
151
+ * (a foreign memory adds 0, never a penalty — penalties re-introduce
152
+ * soft-exclusion). See `AffinityScope`.
130
153
  */
154
+ export interface AffinityScope {
155
+ /** Exact-match project from the client (CC/OC cwd-derived; /search project). */
156
+ project?: string | null;
157
+ /** Hermes mission domains declared in plugin config. Drawn from the same
158
+ * vocabulary as memory_tags (config `domains`). */
159
+ missionDomains?: string[];
160
+ }
131
161
  export declare function computeScore(memory: Memory, distance: number, connectionCount: number, maxConnections: number, now: Date, options?: {
132
162
  superseded?: boolean;
163
+ /** #203: when present, project/domain affinity boosts are applied. */
164
+ scope?: AffinityScope;
165
+ /** Candidate's graded domain tags (memory_tags rows). Loaded batched for
166
+ * the whole candidate set in retrieve(); used for domain affinity. */
167
+ tagWeights?: Array<{
168
+ tag: string;
169
+ weight: number | null;
170
+ }>;
133
171
  }): number;
134
172
  export interface EmbedFn {
135
173
  (text: string): Promise<Float32Array>;
@@ -137,12 +175,25 @@ export interface EmbedFn {
137
175
  /**
138
176
  * Main retrieval: BM25 + vector search with RRF fusion, graph traversal,
139
177
  * and composite scoring. Strengthens accessed memories.
178
+ *
179
+ * #203 retrieval scoping: `project` and `missionDomains` are SOFT affinity
180
+ * terms in computeScore (zero-boost neutral, never a penalty), NOT filters.
181
+ * `privacy` remains a hard filter (security boundary). `sourceAgent` remains a
182
+ * hard filter (kept for completeness; no production caller currently passes
183
+ * it). When neither project nor missionDomains is sent, scoring is byte-
184
+ * identical to pre-#203 — the kill-switch / no-op guarantee.
140
185
  */
141
186
  export declare function retrieve(db: Database.Database, embedFn: EmbedFn, query: string, options?: {
142
187
  limit?: number;
188
+ /** #203: soft project affinity (exact match boost in computeScore).
189
+ * Formerly a hard WHERE filter (#192); softening removes cross-scope
190
+ * starvation without excluding anything. */
143
191
  project?: string | null;
144
192
  privacy?: string[];
145
193
  sourceAgent?: string;
194
+ /** #203: Hermes mission domains (declared in plugin config). Soft domain
195
+ * affinity in computeScore via max overlapping memory_tags.weight. */
196
+ missionDomains?: string[];
146
197
  /** #192: skip access strengthening — for pushed recall (/recall-index),
147
198
  * where appearing in results must not count as use. */
148
199
  noStrengthen?: boolean;
package/dist/retrieval.js CHANGED
@@ -128,19 +128,45 @@ const SCORING_DEFAULTS = {
128
128
  freshnessBoostDays: 7,
129
129
  freshnessBoostWeight: 0.15,
130
130
  supersededDemotion: 0.5,
131
+ projectAffinity: 0.15,
132
+ domainAffinity: 0.15,
133
+ // #205 defaults: rrfK + rrfCompositeWeight match the pre-#205 hardcoded
134
+ // values (60 and 0.8) so the no-config path is byte-identical to 0.15.3
135
+ // except for the FTS per-list weight (1.0 → 0.5) — the one deliberate
136
+ // nudge toward vector that the recall-sweep eval gates. The eval showed
137
+ // 0.7 was too timid (Q4 marine contamination persisted) and 0.5 is the
138
+ // bisection point where BM25F + composite-affinity finally flip the
139
+ // token-exact marine body match below the same-scope hardware field
140
+ // (Q4 ON contamination 0.20 → 0.00). 0.5 is still "conservative" — FTS
141
+ // contributes half its RRF share, enough that pure-keyword queries (the
142
+ // focused-family "login/CORS/webhook" turns) keep recall@5 = 1.0.
143
+ rrfK: 60,
144
+ rrfCompositeWeight: 0.8,
145
+ rrfFtsWeight: 0.5,
146
+ rrfVectorWeight: 1.0,
131
147
  };
132
148
  let scoringWeights = { ...SCORING_DEFAULTS };
133
149
  /**
134
150
  * Configure scoring weights + ranking knobs from config. Called at boot by the
135
151
  * server and the nightly (alongside configureDecay/configureRecall) so
136
152
  * retrieval and consolidation rank identically. Invalid/absent values keep the
137
- * shipped default per key. Returns the resolved set for logging/tests.
153
+ * shipped default per key. Returns the resolved set for logging/tests. Also
154
+ * pushes the #205 BM25F field weights into storage (storage.configureBm25Fts)
155
+ * so searchFts ranks with the same config — BM25F weights live in storage.ts
156
+ * (next to the FTS column declaration they mirror) but are read here from the
157
+ * SAME config object for one-place tuning.
138
158
  */
139
159
  function configureScoring(config) {
140
160
  const num = (key, dflt, min, max) => {
141
161
  const v = Number(config?.[key]);
142
162
  return Number.isFinite(v) && v >= min && v <= max ? v : dflt;
143
163
  };
164
+ // #205 BM25F weights use a [0, ∞) range (no upper bound — a field can dominate
165
+ // if the operator wills it; 0 drops the field entirely). Invalid ⇒ default.
166
+ const numW = (key, dflt) => {
167
+ const v = Number(config?.[key]);
168
+ return Number.isFinite(v) && v >= 0 ? v : dflt;
169
+ };
144
170
  scoringWeights = {
145
171
  similarity: num("scoreSimilarityWeight", SCORING_DEFAULTS.similarity, 0, 1),
146
172
  strength: num("scoreStrengthWeight", SCORING_DEFAULTS.strength, 0, 1),
@@ -149,7 +175,21 @@ function configureScoring(config) {
149
175
  freshnessBoostDays: num("freshnessBoostDays", SCORING_DEFAULTS.freshnessBoostDays, 0, 365),
150
176
  freshnessBoostWeight: num("freshnessBoostWeight", SCORING_DEFAULTS.freshnessBoostWeight, 0, 1),
151
177
  supersededDemotion: num("supersededDemotion", SCORING_DEFAULTS.supersededDemotion, 0, 1),
178
+ projectAffinity: num("projectAffinityWeight", SCORING_DEFAULTS.projectAffinity, 0, 1),
179
+ domainAffinity: num("domainAffinityWeight", SCORING_DEFAULTS.domainAffinity, 0, 1),
180
+ rrfK: numW("rrfK", SCORING_DEFAULTS.rrfK),
181
+ rrfCompositeWeight: num("rrfCompositeWeight", SCORING_DEFAULTS.rrfCompositeWeight, 0, 1),
182
+ rrfFtsWeight: numW("rrfFtsWeight", SCORING_DEFAULTS.rrfFtsWeight),
183
+ rrfVectorWeight: numW("rrfVectorWeight", SCORING_DEFAULTS.rrfVectorWeight),
152
184
  };
185
+ // #205: push BM25F field weights into storage so searchFts uses them. Same
186
+ // config object, one tuning surface; storage owns the module-level mirror
187
+ // next to the FTS column declaration (the positional order matters there).
188
+ storage.configureBm25Fts({
189
+ bm25WeightBody: numW("bm25WeightBody", 1.0),
190
+ bm25WeightProject: numW("bm25WeightProject", 2.0),
191
+ bm25WeightDomain: numW("bm25WeightDomain", 2.0),
192
+ });
153
193
  return { ...scoringWeights };
154
194
  }
155
195
  /** Current resolved weights (tests + status output). */
@@ -233,7 +273,12 @@ function findSupersededIds(db, candidateIds) {
233
273
  * these candidates to cosine 0.875, outranking most true vector matches.
234
274
  */
235
275
  const DEFAULT_GRAPH_DISTANCE = 1.0;
236
- const RRF_K = 60;
276
+ // RRF_K is no longer a module constant (#205): it lives in scoringWeights.rrfK
277
+ // (default 60, the pre-#205 hardcoded value) and is read at every retrieve()
278
+ // call so config changes apply without a restart. The DEFAULT_RRF_K here is a
279
+ // fallback for reciprocalRankFusion's optional k argument (tests + the rare
280
+ // non-retrieve caller), NOT the production path.
281
+ const DEFAULT_RRF_K = 60;
237
282
  /**
238
283
  * Convert an L2 distance (as returned by sqlite-vec's vec0 `distance`) to
239
284
  * cosine similarity. Valid because our embeddings are L2-normalized
@@ -285,10 +330,6 @@ function effectiveStrength(baseStrength, lastAccessed, now, options) {
285
330
  const floor = baseStrength * importance * 0.1;
286
331
  return floor + (baseStrength - floor) * Math.pow(decayRate, hours);
287
332
  }
288
- /**
289
- * Return a composite relevance score in [0, 1] for a candidate memory.
290
- * Exported for exact-value tests of the similarity component (#145).
291
- */
292
333
  function computeScore(memory, distance, connectionCount, maxConnections, now, options) {
293
334
  // TRUE cosine similarity (#145). The old `1 − distance` compressed real
294
335
  // cosines (cos 0.8 scored 0.37) and the 0-clamp at that scale flattened
@@ -327,6 +368,35 @@ function computeScore(memory, distance, connectionCount, maxConnections, now, op
327
368
  const freshness = 1 - ageDays / scoringWeights.freshnessBoostDays;
328
369
  score += freshness * scoringWeights.freshnessBoostWeight;
329
370
  }
371
+ // #203 soft affinity (retrieval scoping). Two graded, additive terms — both
372
+ // ZERO when the scope is absent (byte-identical ranking) and ZERO for a
373
+ // non-matching memory (never a penalty). Project affinity is a flat boost on
374
+ // exact project match; domain affinity is max(overlapping tag weight) × the
375
+ // domain weight. NULL tag weights (not yet computed by the nightly
376
+ // reconsolidation) count as 0 — we never invent a boost from missing
377
+ // association strength. Affinity rides the 0.8 composite side only (the RRF
378
+ // 0.2 side is #205 territory and untouched here).
379
+ const scope = options?.scope;
380
+ if (scope) {
381
+ if (scope.project && memory.project === scope.project) {
382
+ score += scoringWeights.projectAffinity;
383
+ }
384
+ const domains = scope.missionDomains;
385
+ if (domains && domains.length > 0 && options.tagWeights && options.tagWeights.length > 0) {
386
+ const domainSet = domains.length === 1 ? null : new Set(domains);
387
+ let maxWeight = 0;
388
+ for (const tw of options.tagWeights) {
389
+ const overlaps = domainSet ? domainSet.has(tw.tag) : tw.tag === domains[0];
390
+ if (overlaps) {
391
+ const w = tw.weight ?? 0;
392
+ if (w > maxWeight)
393
+ maxWeight = w;
394
+ }
395
+ }
396
+ if (maxWeight > 0)
397
+ score += maxWeight * scoringWeights.domainAffinity;
398
+ }
399
+ }
330
400
  // Superseded demotion (#191 Phase B): a memory whose decision was reversed by
331
401
  // a later one keeps its content and strength but must not outrank the
332
402
  // decision that replaced it. Applied as an explicit multiplier here rather
@@ -382,6 +452,7 @@ function formatResult(memory, score, effStr, connections, provenance) {
382
452
  access_count: memory.access_count ?? 0,
383
453
  memory_type: memory.memory_type ?? "episode",
384
454
  project: memory.project ?? null,
455
+ source_agent: memory.source_agent ?? null,
385
456
  created_at: memory.created_at ?? "",
386
457
  connections,
387
458
  similarity: provenance ? provenance.similarity : undefined,
@@ -407,12 +478,28 @@ function strengthen(db, memories, now) {
407
478
  // ---------------------------------------------------------------------------
408
479
  // Reciprocal Rank Fusion
409
480
  // ---------------------------------------------------------------------------
410
- function reciprocalRankFusion(rankedLists, k = RRF_K) {
481
+ /**
482
+ * Reciprocal Rank Fusion (#205 per-list weights).
483
+ *
484
+ * Each list contributes `weight / (k + rank + 1)` per item. The pre-#205 form
485
+ * (symmetric 1.0 weight on every list) is recovered by omitting `weight`:
486
+ * `{ ids }` defaults to weight 1.0 — so callers that don't care about per-list
487
+ * rebalancing (tests, alternative uses) keep working unchanged.
488
+ *
489
+ * Per-list weights are the #205 lever for "nudging toward vector": FTS was
490
+ * winning cross-scope collisions on raw token overlap (marine "battery" beat
491
+ * hardware "battery" because the marine row had a tighter token match), so the
492
+ * shipped default drops FTS to 0.5 while vector stays at 1.0 (0.7 was too
493
+ * timid — Q4 marine contamination persisted; see SCORING_DEFAULTS). The composite
494
+ * score (which carries the #203 affinity boost) then breaks the tie in scope.
495
+ */
496
+ function reciprocalRankFusion(rankedLists, k = DEFAULT_RRF_K) {
411
497
  const scores = new Map();
412
- for (const ranked of rankedLists) {
413
- for (let rank = 0; rank < ranked.length; rank++) {
414
- const mid = ranked[rank];
415
- scores.set(mid, (scores.get(mid) ?? 0) + 1.0 / (k + rank + 1));
498
+ for (const list of rankedLists) {
499
+ const w = list.weight ?? 1.0;
500
+ for (let rank = 0; rank < list.ids.length; rank++) {
501
+ const mid = list.ids[rank];
502
+ scores.set(mid, (scores.get(mid) ?? 0) + w / (k + rank + 1));
416
503
  }
417
504
  }
418
505
  return scores;
@@ -420,26 +507,42 @@ function reciprocalRankFusion(rankedLists, k = RRF_K) {
420
507
  /**
421
508
  * Main retrieval: BM25 + vector search with RRF fusion, graph traversal,
422
509
  * and composite scoring. Strengthens accessed memories.
510
+ *
511
+ * #203 retrieval scoping: `project` and `missionDomains` are SOFT affinity
512
+ * terms in computeScore (zero-boost neutral, never a penalty), NOT filters.
513
+ * `privacy` remains a hard filter (security boundary). `sourceAgent` remains a
514
+ * hard filter (kept for completeness; no production caller currently passes
515
+ * it). When neither project nor missionDomains is sent, scoring is byte-
516
+ * identical to pre-#203 — the kill-switch / no-op guarantee.
423
517
  */
424
518
  async function retrieve(db, embedFn, query, options) {
425
519
  const limit = options?.limit ?? recallDefaults.searchLimit;
426
520
  const project = options?.project;
427
521
  const privacy = options?.privacy;
428
522
  const sourceAgent = options?.sourceAgent;
523
+ const missionDomains = options?.missionDomains;
429
524
  const now = new Date();
525
+ // #203 affinity scope — passed to computeScore for every candidate. Built
526
+ // once; absent fields yield no boost (zero-boost neutral).
527
+ const scope = project || (missionDomains && missionDomains.length > 0)
528
+ ? { project: project ?? undefined, missionDomains }
529
+ : undefined;
430
530
  // 1. Embed — or reuse the caller-provided vector (session-intent blend).
431
531
  const queryEmbedding = options?.queryEmbedding ?? (await embedFn(query));
432
532
  // 2. Dual retrieval — vector + BM25.
433
533
  // #192: sqlite-vec can't push filters into the KNN, so filtered queries must
434
534
  // over-fetch — the old flat limit*3 intersected a global top-15 with (for the
435
535
  // median project) ~1% of the corpus, starving every filtered query.
436
- const filtered = Boolean(project || privacy || sourceAgent);
536
+ // #203: project is NO LONGER a filter (soft affinity now), so it does not
537
+ // trigger over-fetch; privacy/sourceAgent still do (they remain hard filters).
538
+ const filtered = Boolean(privacy || sourceAgent);
437
539
  const fetchLimit = filtered ? Math.min(limit * 20, 200) : limit * 3;
438
540
  let vecCandidates = storage.vectorSearch(db, queryEmbedding, fetchLimit, []);
439
541
  let ftsCandidates = [];
440
542
  try {
441
- // privacy/sourceAgent/project are all pushed into the FTS SQL.
442
- ftsCandidates = storage.searchFts(db, query, fetchLimit, privacy, sourceAgent, project ?? undefined);
543
+ // privacy/sourceAgent are pushed into the FTS SQL (hard filters). project
544
+ // is NOT (it is a soft affinity boost in computeScore as of #203).
545
+ ftsCandidates = storage.searchFts(db, query, fetchLimit, privacy, sourceAgent);
443
546
  }
444
547
  catch {
445
548
  // FTS5 search can fail on special characters; fall back to vector-only
@@ -447,20 +550,25 @@ async function retrieve(db, embedFn, query, options) {
447
550
  if (vecCandidates.length === 0 && ftsCandidates.length === 0) {
448
551
  return [];
449
552
  }
450
- // Post-filter vector candidates (sqlite-vec can't filter)
451
- if (project) {
452
- vecCandidates = vecCandidates.filter((c) => c.project === project);
453
- }
553
+ // Post-filter vector candidates (sqlite-vec can't filter). privacy stays a
554
+ // hard filter (security boundary); project was removed here (#203 — it is now
555
+ // scored, not filtered). sourceAgent stays (see options doc).
454
556
  if (privacy) {
455
557
  vecCandidates = vecCandidates.filter((c) => privacy.includes(c.privacy));
456
558
  }
457
559
  if (sourceAgent) {
458
560
  vecCandidates = vecCandidates.filter((c) => c.source_agent === sourceAgent);
459
561
  }
460
- // 3. RRF fusion
562
+ // 3. RRF fusion (#205: per-list weights + config-driven k). The vector list
563
+ // carries the composite-side affinity in the next step, so we let it dominate
564
+ // RRF too — the FTS list is down-weighted to break token-collision ties that
565
+ // the affinity alone cannot reach (marine "battery" vs hardware "battery").
461
566
  const vecRanked = vecCandidates.map((c) => c.id);
462
567
  const ftsRanked = ftsCandidates.map((c) => c.id);
463
- const rrfScores = reciprocalRankFusion([vecRanked, ftsRanked]);
568
+ const rrfScores = reciprocalRankFusion([
569
+ { ids: vecRanked, weight: scoringWeights.rrfVectorWeight },
570
+ { ids: ftsRanked, weight: scoringWeights.rrfFtsWeight },
571
+ ], scoringWeights.rrfK);
464
572
  // Build unified candidate map (with retrieval-channel provenance, #192)
465
573
  const candidateMap = new Map();
466
574
  for (const c of vecCandidates) {
@@ -484,8 +592,8 @@ async function retrieve(db, embedFn, query, options) {
484
592
  const mem = storage.getMemory(db, gid);
485
593
  if (!mem)
486
594
  continue;
487
- if (project && mem.project !== project)
488
- continue;
595
+ // #203: project check removed — project is a soft affinity in computeScore,
596
+ // not a filter. privacy (security) and sourceAgent stay as hard filters.
489
597
  if (privacy && !privacy.includes(mem.privacy))
490
598
  continue;
491
599
  if (sourceAgent && mem.source_agent !== sourceAgent)
@@ -501,10 +609,19 @@ async function retrieve(db, embedFn, query, options) {
501
609
  // One query for the whole candidate set (#191 Phase B): superseded memories
502
610
  // are demoted in computeScore rather than strength-penalized.
503
611
  const supersededIds = findSupersededIds(db, [...candidateMap.keys()]);
612
+ // #203: ONE batched load of every candidate's graded domain tags — fed to
613
+ // computeScore for domain affinity. Only needed when the scope carries
614
+ // missionDomains; absent otherwise (skips the query entirely on /search and
615
+ // other unscoped callers — byte-identical to pre-#203).
616
+ const tagWeightsByMemory = scope?.missionDomains && scope.missionDomains.length > 0
617
+ ? storage.getMemoryTagsWeightedBatched(db, [...candidateMap.keys()])
618
+ : undefined;
504
619
  for (const [mid, { mem, distance, source }] of candidateMap) {
505
620
  const connCount = connectionCounts.get(mid) ?? 0;
506
621
  const composite = computeScore(mem, distance, connCount, maxConnections, now, {
507
622
  superseded: supersededIds.has(mid),
623
+ scope,
624
+ tagWeights: tagWeightsByMemory?.get(mid),
508
625
  });
509
626
  const effStr = effectiveStrength(mem.base_strength ?? 0.5, mem.last_accessed, now, {
510
627
  accessCount: mem.access_count ?? 0,
@@ -512,7 +629,11 @@ async function retrieve(db, embedFn, query, options) {
512
629
  });
513
630
  const rrf = rrfScores.get(mid) ?? 0;
514
631
  const normalizedRrf = maxRrf > 0 ? rrf / maxRrf : 0;
515
- const finalScore = composite * 0.8 + normalizedRrf * 0.2;
632
+ // #205: composite/RRF blend is now config-driven (rrfCompositeWeight, 0.8
633
+ // default = pre-#205 behavior). The RRF share is the complement so the two
634
+ // always sum to 1.0 — the knob tunes the BALANCE, not the total.
635
+ const compositeWeight = scoringWeights.rrfCompositeWeight;
636
+ const finalScore = composite * compositeWeight + normalizedRrf * (1 - compositeWeight);
516
637
  // Measured cosine only for vector-matched candidates; FTS/graph hits carry
517
638
  // the neutral placeholder distance, which is not a real similarity.
518
639
  const similarity = source === "vector" || source === "both"
package/dist/storage.d.ts CHANGED
@@ -89,6 +89,17 @@ export declare function getMemoryTagsWeighted(db: Database.Database, memoryId: s
89
89
  tag: string;
90
90
  weight: number | null;
91
91
  }>;
92
+ /**
93
+ * Batched weighted-tag load for a candidate set (#203 domain affinity). ONE
94
+ * query for the whole set — never call getMemoryTagsWeighted per-candidate in
95
+ * a ranking loop. Returns a Map keyed by memory_id; memories with no tags are
96
+ * simply absent from the map (caller treats missing as "no domain boost").
97
+ * Ordering within each memory mirrors getMemoryTagsWeighted (weight DESC).
98
+ */
99
+ export declare function getMemoryTagsWeightedBatched(db: Database.Database, memoryIds: string[]): Map<string, Array<{
100
+ tag: string;
101
+ weight: number | null;
102
+ }>>;
92
103
  /**
93
104
  * Read the stored embedding for a memory from memory_vectors.
94
105
  * Returns null when the row is missing (caller falls back to re-embedding).
@@ -106,10 +117,52 @@ export declare function vectorSearch(db: Database.Database, queryEmbedding: Floa
106
117
  distance: number;
107
118
  }>;
108
119
  /**
109
- * Full-text search using FTS5 BM25 ranking.
110
- * Returns memories with a rank field (lower is better).
120
+ * BM25F field weights (config-driven via {@link configureBm25Fts}, called from
121
+ * retrieval.configureScoring at boot). The order mirrors the FTS5 column
122
+ * declaration in db.ts (content, project, domain) — `bm25(memories_fts, …)`
123
+ * takes weights POSITIONALLY, so a new FTS column MUST be added here in the
124
+ * same position or the weighting silently shifts. Defaults favor scope fields
125
+ * (project/domain) over body so cross-scope noise that wins on raw token
126
+ * frequency (the marine "battery" memory on a hardware query) is demoted
127
+ * without excluding it — the same "graded, never binary" discipline as
128
+ * computeScore's affinity terms.
129
+ */
130
+ export interface Bm25Weights {
131
+ body: number;
132
+ project: number;
133
+ domain: number;
134
+ }
135
+ /**
136
+ * Configure BM25F weights from config. Called by retrieval.configureScoring
137
+ * (which itself is called at server + nightly boot) so storage and retrieval
138
+ * rank identically. Invalid/out-of-range values keep the shipped default per
139
+ * key. Range [0, ∞) — a 0 weight effectively drops that field from the score;
140
+ * negative values are rejected (BM25F sign semantics break otherwise). Returns
141
+ * the resolved set for logging/tests.
111
142
  */
112
- export declare function searchFts(db: Database.Database, query: string, limit?: number, privacy?: string[], sourceAgent?: string, project?: string): Array<Memory & {
143
+ export declare function configureBm25Fts(config?: Record<string, unknown> | null): Bm25Weights;
144
+ /** Current resolved weights (tests + status output). */
145
+ export declare function getBm25Weights(): Bm25Weights;
146
+ /**
147
+ * Full-text search using FTS5 fielded BM25 (BM25F) ranking.
148
+ * Returns memories with a rank field (lower is better — see sign note below).
149
+ *
150
+ * `project` is NOT a filter here (#203): the hard project WHERE from #192 was
151
+ * removed — project is now a soft affinity boost in retrieval.computeScore AND
152
+ * a weighted field in BM25F (#205). `privacy` stays a hard filter (security
153
+ * boundary, not a relevance signal). `sourceAgent` stays a hard filter (kept
154
+ * for completeness; no production caller of retrieve() currently passes it).
155
+ *
156
+ * #205 sign handling: FTS5's `bm25(table, w0, w1, …)` returns a NEGATIVE score
157
+ * where MORE-negative = better match (it is 1 − the normalized BM25 score,
158
+ * which is itself positive — the negation is the FTS5 convention so that
159
+ * `ORDER BY bm25(…)` ASC gives best-first, matching the legacy `ORDER BY
160
+ * fts.rank` direction). Higher field weight ⇒ that column contributes MORE to
161
+ * the per-row score ⇒ matches in that field float up. We bind weights
162
+ * positionally as parameters (NOT string-interpolated) so query-planner
163
+ * caching is unaffected and the config path is the only editor.
164
+ */
165
+ export declare function searchFts(db: Database.Database, query: string, limit?: number, privacy?: string[], sourceAgent?: string): Array<Memory & {
113
166
  rank: number;
114
167
  }>;
115
168
  /**
package/dist/storage.js CHANGED
@@ -15,8 +15,11 @@ exports.deleteMemory = deleteMemory;
15
15
  exports.setMemoryTags = setMemoryTags;
16
16
  exports.getMemoryTags = getMemoryTags;
17
17
  exports.getMemoryTagsWeighted = getMemoryTagsWeighted;
18
+ exports.getMemoryTagsWeightedBatched = getMemoryTagsWeightedBatched;
18
19
  exports.getStoredEmbedding = getStoredEmbedding;
19
20
  exports.vectorSearch = vectorSearch;
21
+ exports.configureBm25Fts = configureBm25Fts;
22
+ exports.getBm25Weights = getBm25Weights;
20
23
  exports.searchFts = searchFts;
21
24
  exports.addLink = addLink;
22
25
  exports.getLinks = getLinks;
@@ -244,6 +247,33 @@ function getMemoryTagsWeighted(db, memoryId) {
244
247
  .all(memoryId);
245
248
  return rows;
246
249
  }
250
+ /**
251
+ * Batched weighted-tag load for a candidate set (#203 domain affinity). ONE
252
+ * query for the whole set — never call getMemoryTagsWeighted per-candidate in
253
+ * a ranking loop. Returns a Map keyed by memory_id; memories with no tags are
254
+ * simply absent from the map (caller treats missing as "no domain boost").
255
+ * Ordering within each memory mirrors getMemoryTagsWeighted (weight DESC).
256
+ */
257
+ function getMemoryTagsWeightedBatched(db, memoryIds) {
258
+ const out = new Map();
259
+ if (memoryIds.length === 0)
260
+ return out;
261
+ const placeholders = memoryIds.map(() => "?").join(", ");
262
+ const rows = db
263
+ .prepare(`SELECT memory_id, tag, weight FROM memory_tags
264
+ WHERE memory_id IN (${placeholders})
265
+ ORDER BY memory_id, (weight IS NULL) ASC, weight DESC, rowid ASC`)
266
+ .all(...memoryIds);
267
+ for (const r of rows) {
268
+ let arr = out.get(r.memory_id);
269
+ if (!arr) {
270
+ arr = [];
271
+ out.set(r.memory_id, arr);
272
+ }
273
+ arr.push({ tag: r.tag, weight: r.weight });
274
+ }
275
+ return out;
276
+ }
247
277
  /**
248
278
  * Read the stored embedding for a memory from memory_vectors.
249
279
  * Returns null when the row is missing (caller falls back to re-embedding).
@@ -286,14 +316,56 @@ function vectorSearch(db, queryEmbedding, limit = 10, excludeIds = []) {
286
316
  }
287
317
  return results;
288
318
  }
289
- // ---------------------------------------------------------------------------
290
- // FTS5 search
291
- // ---------------------------------------------------------------------------
319
+ const BM25_DEFAULTS = {
320
+ body: 1.0,
321
+ project: 2.0,
322
+ domain: 2.0,
323
+ };
324
+ let bm25Weights = { ...BM25_DEFAULTS };
292
325
  /**
293
- * Full-text search using FTS5 BM25 ranking.
294
- * Returns memories with a rank field (lower is better).
326
+ * Configure BM25F weights from config. Called by retrieval.configureScoring
327
+ * (which itself is called at server + nightly boot) so storage and retrieval
328
+ * rank identically. Invalid/out-of-range values keep the shipped default per
329
+ * key. Range [0, ∞) — a 0 weight effectively drops that field from the score;
330
+ * negative values are rejected (BM25F sign semantics break otherwise). Returns
331
+ * the resolved set for logging/tests.
295
332
  */
296
- function searchFts(db, query, limit = 10, privacy, sourceAgent, project) {
333
+ function configureBm25Fts(config) {
334
+ const num = (key, dflt) => {
335
+ const v = Number(config?.[key]);
336
+ return Number.isFinite(v) && v >= 0 ? v : dflt;
337
+ };
338
+ bm25Weights = {
339
+ body: num("bm25WeightBody", BM25_DEFAULTS.body),
340
+ project: num("bm25WeightProject", BM25_DEFAULTS.project),
341
+ domain: num("bm25WeightDomain", BM25_DEFAULTS.domain),
342
+ };
343
+ return { ...bm25Weights };
344
+ }
345
+ /** Current resolved weights (tests + status output). */
346
+ function getBm25Weights() {
347
+ return { ...bm25Weights };
348
+ }
349
+ /**
350
+ * Full-text search using FTS5 fielded BM25 (BM25F) ranking.
351
+ * Returns memories with a rank field (lower is better — see sign note below).
352
+ *
353
+ * `project` is NOT a filter here (#203): the hard project WHERE from #192 was
354
+ * removed — project is now a soft affinity boost in retrieval.computeScore AND
355
+ * a weighted field in BM25F (#205). `privacy` stays a hard filter (security
356
+ * boundary, not a relevance signal). `sourceAgent` stays a hard filter (kept
357
+ * for completeness; no production caller of retrieve() currently passes it).
358
+ *
359
+ * #205 sign handling: FTS5's `bm25(table, w0, w1, …)` returns a NEGATIVE score
360
+ * where MORE-negative = better match (it is 1 − the normalized BM25 score,
361
+ * which is itself positive — the negation is the FTS5 convention so that
362
+ * `ORDER BY bm25(…)` ASC gives best-first, matching the legacy `ORDER BY
363
+ * fts.rank` direction). Higher field weight ⇒ that column contributes MORE to
364
+ * the per-row score ⇒ matches in that field float up. We bind weights
365
+ * positionally as parameters (NOT string-interpolated) so query-planner
366
+ * caching is unaffected and the config path is the only editor.
367
+ */
368
+ function searchFts(db, query, limit = 10, privacy, sourceAgent) {
297
369
  const conditions = ["memories_fts MATCH ?"];
298
370
  const params = [query];
299
371
  if (privacy && privacy.length > 0) {
@@ -305,23 +377,27 @@ function searchFts(db, query, limit = 10, privacy, sourceAgent, project) {
305
377
  conditions.push("m.source_agent = ?");
306
378
  params.push(sourceAgent);
307
379
  }
308
- // #192: project is pushed into SQL (like privacy/sourceAgent) instead of
309
- // being post-filtered by the caller — post-filtering a global top-N against
310
- // a small project starves the result set regardless of corpus content.
311
- if (project) {
312
- conditions.push("m.project = ?");
313
- params.push(project);
314
- }
315
380
  const where = conditions.join(" AND ");
316
- params.push(limit);
381
+ // SQLite binds `?` parameters in LEXICAL SQL order (left-to-right) — the
382
+ // `bm25(memories_fts, ?, ?, ?)` in the SELECT clause comes BEFORE the WHERE
383
+ // and LIMIT `?`s, so the weights must be pushed FIRST. Get this order wrong
384
+ // and FTS5 ends up with a numeric weight as its MATCH expression (parsed as
385
+ // FTS5 query syntax → "syntax error near '.'" on the decimal point).
386
+ const boundParams = [
387
+ bm25Weights.body,
388
+ bm25Weights.project,
389
+ bm25Weights.domain,
390
+ ...params,
391
+ limit,
392
+ ];
317
393
  const rows = db
318
- .prepare(`SELECT m.*, fts.rank
394
+ .prepare(`SELECT m.*, bm25(memories_fts, ?, ?, ?) AS rank
319
395
  FROM memories_fts fts
320
396
  JOIN memories m ON m.rowid = fts.rowid
321
397
  WHERE ${where}
322
- ORDER BY fts.rank
398
+ ORDER BY rank
323
399
  LIMIT ?`)
324
- .all(...params);
400
+ .all(...boundParams);
325
401
  return rows.map((r) => {
326
402
  const rank = r.rank;
327
403
  const mem = rowToMemory(r);
package/dist/types.d.ts CHANGED
@@ -40,6 +40,10 @@ export interface MemorySearchResult {
40
40
  access_count: number;
41
41
  memory_type: string;
42
42
  project: string | null;
43
+ /** Origin agent (e.g. "hermes/atlas", "cc/mbp5") — surfaced in the recall
44
+ * one-liner so agents can calibrate trust (#202 provenance). Optional on the
45
+ * result type (matches how `domain` is threaded) to avoid breaking fixtures. */
46
+ source_agent?: string | null;
43
47
  created_at: string;
44
48
  connections: number;
45
49
  /** True cosine similarity to the query for vector-matched candidates; null
package/dist/uninstall.js CHANGED
@@ -101,14 +101,28 @@ async function runUninstall() {
101
101
  }
102
102
  catch { /* no settings */ }
103
103
  }
104
- // 3. Remove CC custom commands
104
+ // 3. Remove CC custom commands — only files we actually wrote. `learn.md` is
105
+ // a generic name a user may own; guard on the "hicortex" marker the installer
106
+ // always embedded, so uninstall never deletes an unrelated user command.
107
+ let removedCmds = 0;
105
108
  for (const cmd of ["learn.md", "hicortex-activate.md"]) {
106
109
  const cmdPath = (0, node_path_1.join)(CC_COMMANDS_DIR, cmd);
107
- if ((0, node_fs_1.existsSync)(cmdPath)) {
108
- (0, node_fs_1.unlinkSync)(cmdPath);
110
+ if (!(0, node_fs_1.existsSync)(cmdPath))
111
+ continue;
112
+ try {
113
+ if (!(0, node_fs_1.readFileSync)(cmdPath, "utf-8").toLowerCase().includes("hicortex")) {
114
+ console.log(` ⚠ Skipping ${cmd} — not a Hicortex file, left untouched`);
115
+ continue;
116
+ }
117
+ }
118
+ catch {
119
+ continue;
109
120
  }
121
+ (0, node_fs_1.unlinkSync)(cmdPath);
122
+ removedCmds++;
110
123
  }
111
- console.log(" ✓ Removed /learn and /hicortex-activate commands");
124
+ if (removedCmds > 0)
125
+ console.log(` ✓ Removed ${removedCmds} legacy CC command${removedCmds > 1 ? "s" : ""} (/learn, /hicortex-activate)`);
112
126
  // 4. Remove SessionStart hook (JSON merge — filter out entries containing "lessons-context")
113
127
  try {
114
128
  const raw = (0, node_fs_1.readFileSync)(CC_SETTINGS, "utf-8");