claude-mem-lite 3.77.0 → 3.78.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +2 -1
- package/hook.mjs +13 -34
- package/lib/citation-tracker.mjs +27 -6
- package/lib/error-recall-core.mjs +256 -0
- package/lib/inject-search-core.mjs +10 -2
- package/lib/relevance-floor.mjs +136 -0
- package/npm-shrinkwrap.json +2 -2
- package/package.json +3 -1
- package/schema.mjs +16 -1
- package/scripts/user-prompt-search.js +9 -96
- package/source-files.mjs +8 -0
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"plugins": [
|
|
11
11
|
{
|
|
12
12
|
"name": "claude-mem-lite",
|
|
13
|
-
"version": "3.
|
|
13
|
+
"version": "3.78.0",
|
|
14
14
|
"source": "./",
|
|
15
15
|
"description": "Persistent long-term memory for Claude Code via MCP — captures coding decisions, bugfixes, and context across sessions. Hybrid FTS5 + TF-IDF search with episode batching. Single SQLite DB, no external services. A lighter, lower-cost alternative to claude-mem (episode batching + a smaller model; cost savings are an internal estimate, not a measured benchmark)."
|
|
16
16
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "claude-mem-lite",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.78.0",
|
|
4
4
|
"description": "Persistent long-term memory for Claude Code via MCP — captures coding decisions, bugfixes, and context across sessions. Hybrid FTS5 + TF-IDF search with episode batching. Single SQLite DB, no external services. A lighter, lower-cost alternative to claude-mem (episode batching + a smaller model; cost savings are an internal estimate, not a measured benchmark).",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "sdsrss"
|
package/README.md
CHANGED
|
@@ -814,7 +814,8 @@ benchmark and A/B harness are calibrated against — changing them invalidates t
|
|
|
814
814
|
| `CLAUDE_MEM_UPS_BM25_MIN_FOLLOWUP` | Looser floor for follow-up prompts inside an already-injected session. | `5e-6` |
|
|
815
815
|
| `CLAUDE_MEM_UPS_OR_BM25_MIN` | Floor applied to the OR-fallback arm (looser query, needs a stricter floor). | `30` |
|
|
816
816
|
| `CLAUDE_MEM_UPS_TOP_MIN` | Minimum score for the top hit; `0` disables (useful on tiny test corpora). | `50` |
|
|
817
|
-
| `CLAUDE_MEM_UPS_FLOOR_REF_CORPUS` | Reference corpus size the score floors are normalized against, so a fresh install with few rows is not silently gated to zero injections. | `584` |
|
|
817
|
+
| `CLAUDE_MEM_UPS_FLOOR_REF_CORPUS` | Reference corpus size the score floors are normalized against, so a fresh install with few rows is not silently gated to zero injections. Shared by every floor-bearing surface, including error-recall below. | `584` |
|
|
818
|
+
| `CLAUDE_MEM_ERROR_RECALL_BM25_MIN` | Relevance floor for the error-recall surface (memories injected after a failed Bash command). **Off by default.** Setting it to `10.5` (the calibrated value) makes the surface stay silent when its best-matching memory is not actually about the failure — the whole set is dropped, never trimmed row-by-row. **It is a real trade, not a free win:** measured on a live database at that threshold, injections fall ~37% and ~39% of firings go silent, concentrated in projects with few memories. Off by default because nothing shows the dropped rows were noise. Explore with `node benchmark/error-recall-suite.mjs --sweep`. | `0` (off) |
|
|
818
819
|
| `CLAUDE_MEM_UPS_IDENTIFIER_BYPASS` | `0` disables the bypass that lets an exact identifier match skip the score floors. | _(on)_ |
|
|
819
820
|
| `CLAUDE_MEM_UPS_PROMPT_FALLBACK_LIMIT` | How many past-prompt rows the fallback arm may return. | `1` |
|
|
820
821
|
| `MEM_COVERAGE_THRESHOLD` | Fraction of query terms a memory must cover to qualify (∈ [0,1]). | `0.4` |
|
package/hook.mjs
CHANGED
|
@@ -24,9 +24,9 @@ import { readFileSync, writeFileSync, unlinkSync, readdirSync, renameSync, statS
|
|
|
24
24
|
import { homedir } from 'os';
|
|
25
25
|
import {
|
|
26
26
|
inferProject, detectBashSignificance,
|
|
27
|
-
|
|
27
|
+
extractFilePaths, isRelatedToEpisode,
|
|
28
28
|
makeEntryDesc, scrubSecrets, stripPrivate, EDIT_TOOLS, debugCatch, debugLog,
|
|
29
|
-
|
|
29
|
+
formatErrorRecallHints,
|
|
30
30
|
MAX_HOOK_STDIN_BYTES,
|
|
31
31
|
} from './utils.mjs';
|
|
32
32
|
import {
|
|
@@ -75,7 +75,8 @@ import { gcOldMetricShards, recordMetric } from './lib/metrics.mjs';
|
|
|
75
75
|
import { detectMemOverride } from './lib/mem-override.mjs';
|
|
76
76
|
import { injectedIdsFileName, keyContextIdsFileName } from './lib/injected-ids.mjs';
|
|
77
77
|
import { recordKeyContextInjection, touchKeyContextMarker } from './lib/keyctx-marker.mjs';
|
|
78
|
-
import { liveObsFilterSql
|
|
78
|
+
import { liveObsFilterSql } from './lib/inject-search-core.mjs';
|
|
79
|
+
import { selectErrorRecall } from './lib/error-recall-core.mjs';
|
|
79
80
|
import { buildAndSaveHandoff, detectContinuationIntent, renderHandoffInjection, pickHandoffToInject, extractUnfinishedSummary } from './hook-handoff.mjs';
|
|
80
81
|
import { checkForUpdate, getCachedUpdateBanner, isUpdateCheckDue } from './hook-update.mjs';
|
|
81
82
|
import { handleLLMOptimize } from './hook-optimize.mjs';
|
|
@@ -494,37 +495,15 @@ function triggerErrorRecall(db, toolInput, response) {
|
|
|
494
495
|
// query degraded to ['npm','run','build'] — the command's topic, not the failure.
|
|
495
496
|
// planErrorRecall still returns null when nothing usable survives (empty output, or
|
|
496
497
|
// only stop words), and then we stay silent rather than query the command's topic.
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
const
|
|
506
|
-
const rows = db.prepare(`
|
|
507
|
-
SELECT o.id, o.type, o.title, o.lesson_learned
|
|
508
|
-
FROM observations_fts
|
|
509
|
-
JOIN observations o ON observations_fts.rowid = o.id
|
|
510
|
-
WHERE observations_fts MATCH ? AND o.project = ?
|
|
511
|
-
-- Live-row invariant, same as every other model-facing retrieval path
|
|
512
|
-
-- (hook-context obsPool/fallbackObs/keyObs, hook-memory, search-engine,
|
|
513
|
-
-- recent/search/timeline/recall-core, pre-tool-recall, user-prompt-search).
|
|
514
|
-
-- This surface INLINES rows[0].lesson_learned into the model context, so an
|
|
515
|
-
-- unfiltered SELECT handed a retracted lesson to the agent verbatim while its
|
|
516
|
-
-- correction trailed as a bare pointer. compressed_into is filtered too, not
|
|
517
|
-
-- only for symmetry: the block's own footer is a mem_get(ids=...) pointer, and a
|
|
518
|
-
-- COMPRESSED_PENDING_PURGE row is queued for deletion by maintain purge_stale,
|
|
519
|
-
-- so that pointer would resolve to nothing.
|
|
520
|
-
AND ${liveObsFilterSql('o')}
|
|
521
|
-
AND ${notLowSignalTitleClause('o')}
|
|
522
|
-
-- Decay via the shared core (P2-11): the M-1 MAX(0,…) age clamp lives there.
|
|
523
|
-
-- Fixed 14d half-life (error recency matters more than obs type here).
|
|
524
|
-
ORDER BY ${OBS_BM25}
|
|
525
|
-
* ${recencyDecaySql({ tsExpr: 'o.created_at_epoch', halfLifeSql: '1209600000.0' })}
|
|
526
|
-
LIMIT 3
|
|
527
|
-
`).all(ftsQuery, project, nowR);
|
|
498
|
+
// Selection lives in lib/error-recall-core.mjs so the offline calibration suite
|
|
499
|
+
// scores THIS statement rather than a re-typed lookalike. null ⇒ do not inject.
|
|
500
|
+
const selected = selectErrorRecall(db, {
|
|
501
|
+
cmd: toolInput.command || '',
|
|
502
|
+
response,
|
|
503
|
+
project,
|
|
504
|
+
});
|
|
505
|
+
if (!selected) return;
|
|
506
|
+
const rows = selected.rows;
|
|
528
507
|
|
|
529
508
|
const out = formatErrorRecallHints(rows);
|
|
530
509
|
if (out) {
|
package/lib/citation-tracker.mjs
CHANGED
|
@@ -900,16 +900,35 @@ export function applyCitationDecay(db, project, injectedIds, citedIds, sessionId
|
|
|
900
900
|
// literal +1: a FIRST resolution counts the obs into the denominator (1); a
|
|
901
901
|
// cross-turn LATE upgrade of an already-resolved obs must NOT re-count it (0), else
|
|
902
902
|
// cite-rate reads N/2 for a single injected-then-cited obs instead of N/1.
|
|
903
|
+
// v46 (D#159): stamp decay_seen_at_first_cite on the FIRST citation only.
|
|
904
|
+
//
|
|
905
|
+
// `cited_count = 0` is evaluated against the row's PRE-UPDATE state (SQLite reads
|
|
906
|
+
// the old values on the right-hand side of every SET), so it identifies the first
|
|
907
|
+
// promote even though the same statement increments the counter. Once set, the
|
|
908
|
+
// CASE re-writes the column to itself and later citations cannot move it.
|
|
909
|
+
//
|
|
910
|
+
// The stamped value INCLUDES this resolution (`decay_seen_count + @seenInc`), i.e.
|
|
911
|
+
// "this memory was cited on the Nth time the decay loop saw it" — so 1 means cited
|
|
912
|
+
// immediately, and a large N means it was injected-and-ignored N-1 times first.
|
|
913
|
+
// That is exactly the quantity a future "stop injecting after K silent decays"
|
|
914
|
+
// gate must be validated against.
|
|
915
|
+
//
|
|
916
|
+
// NAMED parameters: this statement now binds the same value twice, and a positional
|
|
917
|
+
// list would silently renumber if a clause were ever reordered.
|
|
903
918
|
const updatePromote = db.prepare(`
|
|
904
919
|
UPDATE observations
|
|
905
|
-
SET importance = MIN(
|
|
920
|
+
SET importance = MIN(@cap, COALESCE(importance, 1) + 1),
|
|
906
921
|
cited_count = cited_count + 1,
|
|
907
922
|
uncited_streak = 0,
|
|
908
923
|
demoted_at = NULL,
|
|
909
|
-
last_decided_session_id =
|
|
910
|
-
last_cited_session_id =
|
|
911
|
-
decay_seen_count = decay_seen_count +
|
|
912
|
-
|
|
924
|
+
last_decided_session_id = @session,
|
|
925
|
+
last_cited_session_id = @session,
|
|
926
|
+
decay_seen_count = decay_seen_count + @seenInc,
|
|
927
|
+
decay_seen_at_first_cite = CASE
|
|
928
|
+
WHEN COALESCE(cited_count, 0) = 0 THEN decay_seen_count + @seenInc
|
|
929
|
+
ELSE decay_seen_at_first_cite
|
|
930
|
+
END
|
|
931
|
+
WHERE id = @id
|
|
913
932
|
`);
|
|
914
933
|
const updateStreakOnly = db.prepare(`
|
|
915
934
|
UPDATE observations
|
|
@@ -959,7 +978,9 @@ export function applyCitationDecay(db, project, injectedIds, citedIds, sessionId
|
|
|
959
978
|
// obs already in the injected denominator, so re-counting it would inflate both
|
|
960
979
|
// decay_seen_count and the funnel's injected_n (cite-rate would read N/2, not N/1).
|
|
961
980
|
const firstResolution = !decidedThisSession;
|
|
962
|
-
updatePromote.run(
|
|
981
|
+
updatePromote.run({
|
|
982
|
+
cap: IMPORTANCE_CAP, session: sessionId, seenInc: firstResolution ? 1 : 0, id,
|
|
983
|
+
});
|
|
963
984
|
promoted++;
|
|
964
985
|
if (firstResolution) touched++;
|
|
965
986
|
} else {
|
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
// Error-triggered recall — the SELECTION half of the surface.
|
|
2
|
+
//
|
|
3
|
+
// Why this is a shared core and not left inline in hook.mjs (project convention
|
|
4
|
+
// "shared by two or more faces → lib/"): the offline calibration suite
|
|
5
|
+
// (benchmark/error-recall-suite.mjs) must score THE QUERY THIS SURFACE ACTUALLY
|
|
6
|
+
// RUNS. A benchmark that re-types the SQL measures a second program that merely
|
|
7
|
+
// looks like the first — the failure mode recorded in
|
|
8
|
+
// reference_verify_cc_host_behavior_in_bundle ("copy a call, swap one argument,
|
|
9
|
+
// and you have measured something else"). One body, two consumers: the hook
|
|
10
|
+
// injects from it, the suite scores it.
|
|
11
|
+
//
|
|
12
|
+
// hook.mjs keeps what is genuinely its own: project inference, rendering
|
|
13
|
+
// (formatErrorRecallHints), metering, and the stdout envelope.
|
|
14
|
+
|
|
15
|
+
import { planErrorRecall } from '../bash-utils.mjs';
|
|
16
|
+
import { OBS_BM25, notLowSignalTitleClause } from '../scoring-sql.mjs';
|
|
17
|
+
import { liveObsFilterSql, recencyDecaySql } from './inject-search-core.mjs';
|
|
18
|
+
import { corpusFloorScale } from './relevance-floor.mjs';
|
|
19
|
+
|
|
20
|
+
/** Rows injected per fired error-recall. Historically a bare `LIMIT 3`. */
|
|
21
|
+
export const ERROR_RECALL_LIMIT = 3;
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Floor default. **0 = OFF**, and that is a measured decision, not an oversight.
|
|
25
|
+
* The calibrated value if you want to enable it is 10.5 (`CLAUDE_MEM_ERROR_RECALL_BM25_MIN`).
|
|
26
|
+
* See errorRecallBm25Floor for the numbers behind both halves of that sentence.
|
|
27
|
+
*/
|
|
28
|
+
export const DEFAULT_ERROR_RECALL_BM25_FLOOR = 0;
|
|
29
|
+
|
|
30
|
+
/** The value calibrated for this face, used when the floor is switched on. */
|
|
31
|
+
export const CALIBRATED_ERROR_RECALL_BM25_FLOOR = 10.5;
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Set-level |bm25| floor for this surface. **Default 0 = off.**
|
|
35
|
+
*
|
|
36
|
+
* WHY IT IS BUILT, CALIBRATED, AND STILL OFF. Short version: measured, the lever is
|
|
37
|
+
* small where it is safe and large where it is unmeasured, so nothing justifies moving
|
|
38
|
+
* it into everyone's default path. Long version, because whoever reaches for this next
|
|
39
|
+
* needs the numbers rather than the conclusion:
|
|
40
|
+
*
|
|
41
|
+
* The DISTRIBUTION suggested a floor, and then stopped suggesting it once the fixture
|
|
42
|
+
* got more honest. Measured over 7 well-served cases (618 rows), |bm25_raw| by class:
|
|
43
|
+
*
|
|
44
|
+
* class n min p25 med p75 max
|
|
45
|
+
* relevant 9 10.93 11.17 20.75 30.27 42.97
|
|
46
|
+
* negative 9 11.28 22.35 24.39 27.28 31.89
|
|
47
|
+
* filler 19 8.33 8.51 9.55 10.13 29.57
|
|
48
|
+
*
|
|
49
|
+
* 10.5 sits in that gap — filler p75 10.13, relevant min 10.93 — which is how the UPS
|
|
50
|
+
* face's OR floor got its 30 in its own 22->41 gap. But adding two NO-GOOD-MATCH cases
|
|
51
|
+
* (real failures the corpus cannot explain, which is the common case in production and
|
|
52
|
+
* which the fixture originally lacked entirely) moves it (9 cases, 620 rows):
|
|
53
|
+
*
|
|
54
|
+
* class n min p25 med p75 max
|
|
55
|
+
* relevant 9 10.93 11.18 20.77 29.99 43.00
|
|
56
|
+
* negative 11 10.59 20.87 24.41 29.25 31.91
|
|
57
|
+
* filler 30 8.16 8.89 9.56 10.99 29.25
|
|
58
|
+
*
|
|
59
|
+
* filler's p75 is now 10.99, ABOVE relevant's min of 10.93: the gap is gone. It was an
|
|
60
|
+
* artifact of a fixture where every case had something good to find. Note too that
|
|
61
|
+
* `relevant` and `negative` overlap almost entirely in both tables — per #8858 a
|
|
62
|
+
* magnitude gate can remove genuinely off-topic rows and cannot tell an explaining row
|
|
63
|
+
* from a merely topical one.
|
|
64
|
+
*
|
|
65
|
+
* The SWEEP says the achievable gain is small, because a floor only affects rows that
|
|
66
|
+
* reach the injection cap and weak rows mostly do not. On the 7-case fixture a PER-ROW
|
|
67
|
+
* floor at 10.5 moved 20 -> 18 injected rows, both removed rows off-topic filler,
|
|
68
|
+
* relevant and hit-rate unchanged: +3.3pp precision. That was the best case for this
|
|
69
|
+
* lever and it is worth 2 rows.
|
|
70
|
+
*
|
|
71
|
+
* The LIVE DATABASE says the cost is neither small nor measured. Pre-release review ran
|
|
72
|
+
* the per-row form over 8 projects x 10 real hard-error shapes on the maintainer's own
|
|
73
|
+
* DB: injected rows 221 -> 112 (-49%), and 25 of 80 firing cases (31%) went to
|
|
74
|
+
* injecting nothing at all. The loss sits entirely in SMALL projects (under ~500
|
|
75
|
+
* observations: -47..-87%; above ~800: -0..-3%), and corpusFloorScale cannot correct it
|
|
76
|
+
* — it normalises over the WHOLE observations table, which is right for FTS5's IDF and
|
|
77
|
+
* deliberately project-blind, whereas what collapses on a small project is how good the
|
|
78
|
+
* best available memory is. That is v3.61.0's failure mode relocated from install scope
|
|
79
|
+
* to project scope.
|
|
80
|
+
*
|
|
81
|
+
* The obvious repair — make the gate SET-LEVEL, matching what UPS actually does (read
|
|
82
|
+
* the top row, drop the whole set on failure) so a case with one strong row keeps its
|
|
83
|
+
* supporting rows — is the form implemented in selectErrorRecall. AND THE FIXTURE SAYS
|
|
84
|
+
* IT IS FREE, WHICH IS ALSO WRONG. On the ruler it is a no-op at 10.5 (26 injected rows
|
|
85
|
+
* at every floor from 0 to 20; it only bites at 25, costing hit-rate 85.7% -> 42.9%).
|
|
86
|
+
* On the live DB, same threshold, 8 projects x 9 shapes, 69 firing cases:
|
|
87
|
+
*
|
|
88
|
+
* base (off) 201 rows
|
|
89
|
+
* set-level 10.5 126 rows (-37%) 27 of 69 cases silenced (39%)
|
|
90
|
+
* per-row 10.5 97 rows (-52%) 26 of 69 cases silenced (38%)
|
|
91
|
+
*
|
|
92
|
+
* The set-level form trims fewer rows in the cases it spares, and silences just as many
|
|
93
|
+
* cases. The fixture cannot see this because ITS hard negatives are constructed to score
|
|
94
|
+
* high — every fixture case, including the no-good-match ones, has a top row above 10.5,
|
|
95
|
+
* while real small projects frequently have nothing above it. That is the same failure
|
|
96
|
+
* as the per-row measurement one level up: a fixture number standing in for a live one.
|
|
97
|
+
*
|
|
98
|
+
* So the per-row form buys +3.3pp on a fixture at a live cost nothing has shown to be
|
|
99
|
+
* noise (citation_surface_log stores counts, not ids, so the |bm25| of the 15 cited rows
|
|
100
|
+
* is unrecoverable), and the set-level form costs a third of the face for a benefit
|
|
101
|
+
* measured nowhere. Neither earns a default. The defect this face actually has is that
|
|
102
|
+
* command words dominate BM25 — semantic, and out of reach of any magnitude gate (D#167).
|
|
103
|
+
*
|
|
104
|
+
* Enable with CLAUDE_MEM_ERROR_RECALL_BM25_MIN=10.5 (see CALIBRATED_… above). Read at
|
|
105
|
+
* call time so tests and a redirected environment both take effect.
|
|
106
|
+
*/
|
|
107
|
+
export function errorRecallBm25Floor() {
|
|
108
|
+
const raw = process.env.CLAUDE_MEM_ERROR_RECALL_BM25_MIN;
|
|
109
|
+
if (raw === undefined || raw === '') return DEFAULT_ERROR_RECALL_BM25_FLOOR;
|
|
110
|
+
const n = Number(raw);
|
|
111
|
+
return Number.isFinite(n) && n >= 0 ? n : DEFAULT_ERROR_RECALL_BM25_FLOOR;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Fixed 14d half-life: for a failure, recency matters more than observation
|
|
116
|
+
* type, so this surface deliberately does NOT use TYPE_DECAY_CASE.
|
|
117
|
+
*/
|
|
118
|
+
const ERROR_RECALL_HALF_LIFE_MS = '1209600000.0';
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Build the error-recall SELECT. Exported so the calibration suite can assert it
|
|
122
|
+
* is scoring the same statement the hook runs, and so a future floor lands in
|
|
123
|
+
* exactly one place.
|
|
124
|
+
*
|
|
125
|
+
* @param {number} limit Row cap; coerced to a safe integer before interpolation.
|
|
126
|
+
* @returns {string} SQL taking NAMED parameters @q (MATCH), @project, @now
|
|
127
|
+
* (decay reference) and @floor (|bm25| minimum). Named rather than positional
|
|
128
|
+
* so the binding cannot silently renumber when the statement is rearranged.
|
|
129
|
+
*/
|
|
130
|
+
export function errorRecallSql(limit = ERROR_RECALL_LIMIT) {
|
|
131
|
+
// Number.isFinite before trunc: `Number('Infinity')` is a number and truncates to
|
|
132
|
+
// Infinity, which interpolates as `LIMIT Infinity` and throws at prepare(). Not an
|
|
133
|
+
// injection (every string form coerces to the default) but a crash where a fallback
|
|
134
|
+
// belongs.
|
|
135
|
+
const asNum = Number(limit);
|
|
136
|
+
const n = Number.isFinite(asNum) && asNum >= 1
|
|
137
|
+
? Math.max(1, Math.trunc(asNum))
|
|
138
|
+
: ERROR_RECALL_LIMIT;
|
|
139
|
+
// No floor in the SQL: the gate is SET-LEVEL and lives in selectErrorRecall. See
|
|
140
|
+
// its docblock for why. This statement is the pre-floor one plus a bm25_raw column.
|
|
141
|
+
return `
|
|
142
|
+
SELECT o.id, o.type, o.title, o.lesson_learned,
|
|
143
|
+
-- Raw match quality, exposed so the set-level floor can read the top row.
|
|
144
|
+
-- Deliberately the UNDECAYED bm25: an old-but-exact row should be able to clear
|
|
145
|
+
-- the floor, and gating on the decayed score would make the floor an age cutoff.
|
|
146
|
+
-- Note the gate reads the RANK-top row (ordered by bm25 x decay below), not the
|
|
147
|
+
-- highest |bm25| in the set — see selectErrorRecall.
|
|
148
|
+
${OBS_BM25} AS bm25_raw
|
|
149
|
+
FROM observations_fts
|
|
150
|
+
JOIN observations o ON observations_fts.rowid = o.id
|
|
151
|
+
WHERE observations_fts MATCH @q AND o.project = @project
|
|
152
|
+
-- Live-row invariant, same as every other model-facing retrieval path
|
|
153
|
+
-- (hook-context obsPool/fallbackObs/keyObs, hook-memory, search-engine,
|
|
154
|
+
-- recent/search/timeline/recall-core, pre-tool-recall, user-prompt-search).
|
|
155
|
+
-- This surface INLINES rows[0].lesson_learned into the model context, so an
|
|
156
|
+
-- unfiltered SELECT handed a retracted lesson to the agent verbatim while its
|
|
157
|
+
-- correction trailed as a bare pointer. compressed_into is filtered too, not
|
|
158
|
+
-- only for symmetry: the block's own footer is a mem_get(ids=...) pointer, and a
|
|
159
|
+
-- COMPRESSED_PENDING_PURGE row is queued for deletion by maintain purge_stale,
|
|
160
|
+
-- so that pointer would resolve to nothing.
|
|
161
|
+
AND ${liveObsFilterSql('o')}
|
|
162
|
+
AND ${notLowSignalTitleClause('o')}
|
|
163
|
+
-- Decay via the shared core (P2-11): the M-1 MAX(0,…) age clamp lives there.
|
|
164
|
+
ORDER BY ${OBS_BM25}
|
|
165
|
+
* ${recencyDecaySql({
|
|
166
|
+
tsExpr: 'o.created_at_epoch',
|
|
167
|
+
halfLifeSql: ERROR_RECALL_HALF_LIFE_MS,
|
|
168
|
+
// NAMED, not positional. The statement binds @q and @project ahead of this
|
|
169
|
+
// expression, and better-sqlite3 forbids mixing the two styles; an earlier
|
|
170
|
+
// revision moved this expression into the SELECT list with a positional `?`
|
|
171
|
+
// and MATCH silently received the project name (FTS5: `no such column`).
|
|
172
|
+
nowParam: '@now',
|
|
173
|
+
})}
|
|
174
|
+
LIMIT ${n}
|
|
175
|
+
`;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Turn planErrorRecall's terms into the OR-query this surface matches on.
|
|
180
|
+
* @returns {string} FTS5 MATCH expression, or '' when there is nothing to run.
|
|
181
|
+
*/
|
|
182
|
+
export function errorRecallFtsQuery(terms) {
|
|
183
|
+
return (terms || []).map((t) => `"${String(t).replace(/"/g, '""')}"`).join(' OR ');
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Decide whether error-recall fires, and select its rows.
|
|
188
|
+
*
|
|
189
|
+
* Returns null — meaning DO NOT INJECT — in the two cases the surface already
|
|
190
|
+
* treated as silence: planErrorRecall found no usable error term (see its own
|
|
191
|
+
* docblock for why silence beats querying the command's topic), or the terms
|
|
192
|
+
* produced an empty MATCH expression.
|
|
193
|
+
*
|
|
194
|
+
* @param {object} db Open better-sqlite3 handle.
|
|
195
|
+
* @param {{cmd: string, response: string, project: string, now?: number,
|
|
196
|
+
* limit?: number}} opts
|
|
197
|
+
* @returns {{rows: object[], terms: string[], ftsQuery: string}|null}
|
|
198
|
+
*/
|
|
199
|
+
export function selectErrorRecall(db, {
|
|
200
|
+
cmd, response, project, now = Date.now(), limit = ERROR_RECALL_LIMIT, floor,
|
|
201
|
+
}) {
|
|
202
|
+
const plan = planErrorRecall(cmd, response);
|
|
203
|
+
if (!plan) return null;
|
|
204
|
+
|
|
205
|
+
const ftsQuery = errorRecallFtsQuery(plan.terms);
|
|
206
|
+
if (!ftsQuery) return null;
|
|
207
|
+
|
|
208
|
+
// A null / NaN / '' floor means "not specified", same as omitting it — NOT "no
|
|
209
|
+
// floor". Disabling the gate must take an explicit 0. The env reader above is
|
|
210
|
+
// hardened the same way, and the asymmetry between the two was a review finding.
|
|
211
|
+
// Only a real, finite, non-negative NUMBER counts as specified. Coercing first would
|
|
212
|
+
// make `''` mean 0 — Number('') is 0 and is finite — i.e. an empty string would
|
|
213
|
+
// silently disable the gate. Same posture as the env reader above.
|
|
214
|
+
const base = (typeof floor === 'number' && Number.isFinite(floor) && floor >= 0)
|
|
215
|
+
? floor
|
|
216
|
+
: errorRecallBm25Floor();
|
|
217
|
+
// Scale by corpus size for the same reason UPS does: bm25 carries IDF, so a fixed
|
|
218
|
+
// magnitude means "weak match" on an established index and "small index" on a new
|
|
219
|
+
// one. v3.61.0 shipped an unscaled floor and injected 0/8 on fresh installs.
|
|
220
|
+
const effective = base > 0 ? base * corpusFloorScale(db) : 0;
|
|
221
|
+
|
|
222
|
+
const rows = db.prepare(errorRecallSql(limit)).all({ q: ftsQuery, project, now });
|
|
223
|
+
|
|
224
|
+
// SET-LEVEL gate, the shape UPS already uses (read the top row; on failure drop the
|
|
225
|
+
// WHOLE set) rather than filtering row by row.
|
|
226
|
+
//
|
|
227
|
+
// Why not per-row: measured in pre-release review against the maintainer's live DB,
|
|
228
|
+
// a per-row floor cut injected rows 221 → 112 (−49%) across 8 projects, and the loss
|
|
229
|
+
// was concentrated entirely in SMALL ones — projects under ~500 observations lost
|
|
230
|
+
// 47–87%, those above ~800 lost 0–3%. corpusFloorScale cannot see that by design: it
|
|
231
|
+
// normalises over the WHOLE observations table, which is correct for FTS5's IDF and
|
|
232
|
+
// is documented as deliberately project-blind. But what collapses on a small project
|
|
233
|
+
// is not IDF, it is how good the best available memory is. So a per-row floor also
|
|
234
|
+
// shortened sets that DID have a good top row — a different and unjustified
|
|
235
|
+
// behaviour from "this failure has nothing worth recalling".
|
|
236
|
+
//
|
|
237
|
+
// The set-level shape says exactly the intended thing and nothing more: if the best
|
|
238
|
+
// match is not about the failure, stay silent (D#136's stance); otherwise inject the
|
|
239
|
+
// set unchanged. Rows 2..n are never judged on their own, so a case with one strong
|
|
240
|
+
// row keeps its supporting rows.
|
|
241
|
+
//
|
|
242
|
+
// `rows[0]` is the RANK-top row (ordered by bm25 x decay), NOT the highest |bm25|.
|
|
243
|
+
// Those differ: with a 14-day half-life the multiplier reaches 2x, so a fresh weaker
|
|
244
|
+
// row can outrank an older stronger one and veto a set the stronger row would have
|
|
245
|
+
// admitted. Measured, this is why the set-level form silences marginally MORE cases
|
|
246
|
+
// than the per-row one (27 vs 26 of 69 on the live DB). Kept as-is rather than
|
|
247
|
+
// switched to max(): UPS gates on `ftsRows[0]` the same way, and one face quietly
|
|
248
|
+
// disagreeing with the other about what "the top hit" means is worse than the
|
|
249
|
+
// occasional veto. Stated here because the comment above used to claim otherwise.
|
|
250
|
+
if (effective > 0 && rows.length && Math.abs(rows[0].bm25_raw) < effective) {
|
|
251
|
+
return {
|
|
252
|
+
rows: [], terms: plan.terms, ftsQuery, floor: effective, suppressed: rows.length,
|
|
253
|
+
};
|
|
254
|
+
}
|
|
255
|
+
return { rows, terms: plan.terms, ftsQuery, floor: effective, suppressed: 0 };
|
|
256
|
+
}
|
|
@@ -61,10 +61,18 @@ export function liveObsFilterSql(alias = 'o') {
|
|
|
61
61
|
* (e.g. 'o.created_at_epoch', or the created/last-accessed MAX search-engine uses)
|
|
62
62
|
* @param {string} [opts.halfLifeSql=TYPE_DECAY_CASE] - SQL expression for the
|
|
63
63
|
* half-life in ms (constant or per-type CASE; TYPE_DECAY_CASE assumes alias 'o')
|
|
64
|
+
* @param {string} [opts.nowParam='?'] - placeholder text for `now`. Defaults to a
|
|
65
|
+
* POSITIONAL `?`, which makes the binding order depend on where this expression
|
|
66
|
+
* lands in the statement — moving it from ORDER BY into a SELECT list silently
|
|
67
|
+
* renumbers every other placeholder (observed 2026-08-24: the MATCH argument
|
|
68
|
+
* received a project name and FTS5 reported `no such column`). A caller that
|
|
69
|
+
* would rather not carry that coupling passes a NAMED placeholder such as
|
|
70
|
+
* '@now' and binds by object instead. better-sqlite3 does not allow mixing the
|
|
71
|
+
* two styles in one statement, so this is per-statement, all or nothing.
|
|
64
72
|
* @returns {string} SQL numeric expression (parenthesized)
|
|
65
73
|
*/
|
|
66
|
-
export function recencyDecaySql({ tsExpr, halfLifeSql = TYPE_DECAY_CASE }) {
|
|
67
|
-
return `(1.0 + EXP(-0.693 * MAX(0,
|
|
74
|
+
export function recencyDecaySql({ tsExpr, halfLifeSql = TYPE_DECAY_CASE, nowParam = '?' }) {
|
|
75
|
+
return `(1.0 + EXP(-0.693 * MAX(0, ${nowParam} - ${tsExpr}) / ${halfLifeSql}))`;
|
|
68
76
|
}
|
|
69
77
|
|
|
70
78
|
/**
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
// Corpus-size normalization for ABSOLUTE relevance floors.
|
|
2
|
+
//
|
|
3
|
+
// Extracted from scripts/user-prompt-search.js (v3.61.0) when a second injection
|
|
4
|
+
// face — error-recall — needed the same ramp. Two faces, one body: the project's
|
|
5
|
+
// "shared by two or more faces → lib/" rule, and specifically the rule that exists
|
|
6
|
+
// because absolute-floor logic re-typed per face is how v3.61.0 shipped a constant
|
|
7
|
+
// gating an IDF-bearing quantity and injected 0/8 on fresh installs.
|
|
8
|
+
//
|
|
9
|
+
// scripts/user-prompt-search.js re-exports corpusFloorScale so its own callers and
|
|
10
|
+
// tests (tests/ups-corpus-floor-scale.test.mjs) keep their import path.
|
|
11
|
+
//
|
|
12
|
+
// ONE DELIBERATE CHANGE ON EXTRACTION: the reference corpus is read at CALL time,
|
|
13
|
+
// not at module load. In production this is indistinguishable — a hook process reads
|
|
14
|
+
// a fixed environment — but it removes a trap for callers: a re-import with a
|
|
15
|
+
// cache-busting query string reloads the FACE, not this module, so a load-time
|
|
16
|
+
// constant here would have frozen at whatever the first import saw.
|
|
17
|
+
|
|
18
|
+
// Default reference corpus. Overridable per the historical UPS env name.
|
|
19
|
+
// Module-private: exported by habit in the first cut, and knip correctly flagged it —
|
|
20
|
+
// this project treats a new unused export as a defect to fix, not a baseline to carry.
|
|
21
|
+
const DEFAULT_FLOOR_REF_CORPUS = 584;
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* The reference corpus the absolute floors are calibrated against.
|
|
25
|
+
* @returns {number}
|
|
26
|
+
*/
|
|
27
|
+
function floorRefCorpus() {
|
|
28
|
+
return Number(process.env.CLAUDE_MEM_UPS_FLOOR_REF_CORPUS || DEFAULT_FLOOR_REF_CORPUS);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* FTS5's IDF term for the best case a query can hit: a term appearing in exactly
|
|
33
|
+
* one row (df=1) of an n-row index. SQLite computes
|
|
34
|
+
* `log((n - df + 0.5) / (df + 0.5))`, so this is the ceiling any single-term bm25
|
|
35
|
+
* contribution can reach at corpus size n — the quantity the absolute floors are
|
|
36
|
+
* implicitly denominated in. Clamped at 0: below n=2 the formula goes negative,
|
|
37
|
+
* which as a scale would flip the comparison rather than relax it.
|
|
38
|
+
* @param {number} n Row count.
|
|
39
|
+
* @returns {number} Max attainable IDF at this corpus size, ≥ 0.
|
|
40
|
+
*/
|
|
41
|
+
function maxIdf(n) {
|
|
42
|
+
// n <= 1 makes the numerator non-positive → Math.log returns NaN or -Infinity, and
|
|
43
|
+
// Math.max(0, NaN) is NaN, not 0. Short-circuit instead: a 0- or 1-row index has no
|
|
44
|
+
// term that can discriminate, so the max attainable IDF is 0.
|
|
45
|
+
if (!(n > 1)) return 0;
|
|
46
|
+
return Math.max(0, Math.log((n - 1 + 0.5) / 1.5));
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// ─── Why the ramp exists (v3.61.0 calibration history, moved with the code) ───
|
|
50
|
+
//
|
|
51
|
+
// The floors this scales are ABSOLUTE magnitudes (UPS: TOP_REL_FLOOR /
|
|
52
|
+
// OR_TOP_BM25_FLOOR; error-recall: ERROR_RECALL_BM25_FLOOR), but the quantity they
|
|
53
|
+
// gate is not scale-free: FTS5 bm25 carries an IDF term ≈ ln(N/df), so the SAME hit
|
|
54
|
+
// scores higher on a bigger index. Measured on one fixed query + one fixed target
|
|
55
|
+
// row, padding the corpus with distinct filler (2026-08-13 dogfood):
|
|
56
|
+
//
|
|
57
|
+
// totalObs 10 40 100 300
|
|
58
|
+
// top|bm25| 10.0 18.6 24.2 30.7 ← same row, same query
|
|
59
|
+
//
|
|
60
|
+
// The floors were calibrated at `projects--mem, 584 obs` (CHANGELOG v2.43.x /
|
|
61
|
+
// v2.34.3). Comparing a log-N quantity against that constant therefore does not
|
|
62
|
+
// mean "weak match" on a small index — it means "small index". A brand-new
|
|
63
|
+
// install measured 0/8 injections on a realistic first-day corpus (10 memories,
|
|
64
|
+
// 8 recall questions whose correct target ranked #1 in 4/5 scored cases): every
|
|
65
|
+
// one was dropped by the OR floor at |bm25| 3.8–15.2 < 30. The plugin is inert
|
|
66
|
+
// during exactly the window where a new user decides whether it earns its keep.
|
|
67
|
+
//
|
|
68
|
+
// Fix: scale the floors by the corpus's MAX ATTAINABLE IDF over the reference
|
|
69
|
+
// corpus's, capped at 1.0, so the SIGNAL↔NOISE separation the maintainer measured
|
|
70
|
+
// (signal ≥41, noise ≤22 at N_REF) is preserved proportionally at any N. At
|
|
71
|
+
// N ≥ N_REF the factor is exactly 1.0, so every established install keeps
|
|
72
|
+
// byte-identical behavior; only genuinely-new installs relax.
|
|
73
|
+
//
|
|
74
|
+
// 2026-08-17 e2e round — the ramp shape. The first cut used ln(N+1)/ln(N_REF+1), which has the
|
|
75
|
+
// right asymptotics but the wrong small-N behavior: FTS5's IDF term is
|
|
76
|
+
// `log((N - df + 0.5) / (df + 0.5))`, which is EXACTLY 0 at N=2/df=1 and stays far
|
|
77
|
+
// below ln(N+1) for the whole first-week window. Re-measured end-to-end through the
|
|
78
|
+
// production write path (lib/save-observation.mjs — a raw INSERT skips CJK bigram
|
|
79
|
+
// expansion and understates the corpus, which is how the first cut's ramp table was
|
|
80
|
+
// misread), 1 planted target + topically clustered filler, CJK prose prompt carrying
|
|
81
|
+
// no identifier for the bypass to rescue:
|
|
82
|
+
//
|
|
83
|
+
// N 2 3 4 5 6 10 25 80
|
|
84
|
+
// top|bm25| 0.0 5.1 7.0 9.5 11.4 15.5 22.2 30.3
|
|
85
|
+
// ln ramp floor 5.2 6.5 7.6 8.4 9.2 11.3 15.3 20.7 ← DROP at N≤4
|
|
86
|
+
// idf ramp floor 0.0 2.6 4.3 5.5 6.5 9.3 14.1 20.0 ← admits all
|
|
87
|
+
//
|
|
88
|
+
// The two ramps agree within 8% at N≥30 and within 2% at N≥200, so this re-shape is
|
|
89
|
+
// confined to the window it is meant to fix. Accepted tradeoff: on a ≤2-row corpus the
|
|
90
|
+
// scale is EXACTLY 0 (FTS5's max IDF is 0 there), which disables the set-level floors
|
|
91
|
+
// rather than lowering them, and just above that they are small; so the best lexical match
|
|
92
|
+
// is injected even when it is weak.
|
|
93
|
+
// That is the intended trade — the alternative measured behavior is total silence, and
|
|
94
|
+
// a 4-row corpus has no room to bury signal under noise. For UPS the upstream
|
|
95
|
+
// hasExplicitSignal gate, not these floors, is what suppresses noise prompts.
|
|
96
|
+
//
|
|
97
|
+
// N counts the WHOLE observations table, not the project: FTS5 computes IDF over
|
|
98
|
+
// the entire index and `o.project = ?` is a post-MATCH filter. Verified — a
|
|
99
|
+
// 2-row project on a 302-row install scores 31.5, matching the 300-row global
|
|
100
|
+
// baseline, not the 10-row one. So a new project on an established install is
|
|
101
|
+
// (correctly) unaffected by this ramp.
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Scale factor in [0, 1] for the absolute score floors, by total corpus size.
|
|
105
|
+
*
|
|
106
|
+
* Short-circuits with a bounded probe: if a row exists at offset N_REF-1 the
|
|
107
|
+
* corpus is at or above the reference and the factor is 1.0 — no COUNT scan on
|
|
108
|
+
* the large corpora where the answer is always 1.0 anyway.
|
|
109
|
+
*
|
|
110
|
+
* Note the ceiling: this only ever RELAXES a floor on a small corpus, never
|
|
111
|
+
* tightens one on a large corpus. A face calibrated at or above the reference
|
|
112
|
+
* therefore keeps its measured value everywhere, and a bigger index (higher IDF,
|
|
113
|
+
* higher scores) makes the same floor comparatively more permissive — the safe
|
|
114
|
+
* direction, since the failure this ramp exists to prevent is silence.
|
|
115
|
+
*
|
|
116
|
+
* @param {object} db Open better-sqlite3 handle.
|
|
117
|
+
* @returns {number} Multiplier for an absolute score floor.
|
|
118
|
+
*/
|
|
119
|
+
export function corpusFloorScale(db) {
|
|
120
|
+
const ref = floorRefCorpus();
|
|
121
|
+
if (ref <= 1) return 1;
|
|
122
|
+
try {
|
|
123
|
+
const atRef = db.prepare('SELECT 1 FROM observations LIMIT 1 OFFSET ?').get(ref - 1);
|
|
124
|
+
if (atRef) return 1;
|
|
125
|
+
const { c = 0 } = db.prepare('SELECT count(*) AS c FROM observations').get() || {};
|
|
126
|
+
const refIdf = maxIdf(ref);
|
|
127
|
+
// Degenerate reference (CLAUDE_MEM_UPS_FLOOR_REF_CORPUS set to 2 or 3, where
|
|
128
|
+
// maxIdf is 0 or near it): division would blow up or divide by zero. Treat the
|
|
129
|
+
// floors as fully calibrated, matching the ref <= 1 guard above.
|
|
130
|
+
if (!(refIdf > 0)) return 1;
|
|
131
|
+
return Math.min(1, maxIdf(c) / refIdf);
|
|
132
|
+
} catch {
|
|
133
|
+
// Any probe failure → behave exactly as before the ramp existed.
|
|
134
|
+
return 1;
|
|
135
|
+
}
|
|
136
|
+
}
|
package/npm-shrinkwrap.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "claude-mem-lite",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.78.0",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "claude-mem-lite",
|
|
9
|
-
"version": "3.
|
|
9
|
+
"version": "3.78.0",
|
|
10
10
|
"dependencies": {
|
|
11
11
|
"@modelcontextprotocol/sdk": "^1.26.0",
|
|
12
12
|
"better-sqlite3": "^12.6.2",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "claude-mem-lite",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.78.0",
|
|
4
4
|
"description": "Persistent long-term memory for Claude Code via MCP — captures coding decisions, bugfixes, and context across sessions. Hybrid FTS5 + TF-IDF search with episode batching. Single SQLite DB, no external services. A lighter, lower-cost alternative to claude-mem (episode batching + a smaller model; cost savings are an internal estimate, not a measured benchmark).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"packageManager": "npm@10.9.2",
|
|
@@ -91,6 +91,8 @@
|
|
|
91
91
|
"lib/time-constants.mjs",
|
|
92
92
|
"lib/keyctx-marker.mjs",
|
|
93
93
|
"lib/inject-search-core.mjs",
|
|
94
|
+
"lib/error-recall-core.mjs",
|
|
95
|
+
"lib/relevance-floor.mjs",
|
|
94
96
|
"lib/get-core.mjs",
|
|
95
97
|
"lib/browse-core.mjs",
|
|
96
98
|
"lib/save-observation.mjs",
|
package/schema.mjs
CHANGED
|
@@ -152,7 +152,7 @@ export const CODE_DIR = join(homedir(), '.claude-mem-lite');
|
|
|
152
152
|
// (citation_surface_log.surface) in LATEST_MIGRATION_COLUMNS: a table that only
|
|
153
153
|
// the forced pass can create is unreachable forever once the version row says
|
|
154
154
|
// "done", which is not a hypothetical — see the note there.
|
|
155
|
-
export const CURRENT_SCHEMA_VERSION =
|
|
155
|
+
export const CURRENT_SCHEMA_VERSION = 46;
|
|
156
156
|
|
|
157
157
|
// Sentinel columns for the LATEST migration set(s). The fast-path uses these
|
|
158
158
|
// to self-heal half-migrated DBs — schema_version bumped but column ALTERs
|
|
@@ -173,6 +173,7 @@ export const CURRENT_SCHEMA_VERSION = 45;
|
|
|
173
173
|
// pragma_table_info on a missing table returns zero rows (it does not throw), so
|
|
174
174
|
// naming any column of the new table is a table-presence check.
|
|
175
175
|
const LATEST_MIGRATION_COLUMNS = [
|
|
176
|
+
{ table: 'observations', column: 'decay_seen_at_first_cite' }, // v46
|
|
176
177
|
{ table: 'citation_surface_log', column: 'surface' }, // v45
|
|
177
178
|
{ table: 'observations', column: 'scope' }, // v44
|
|
178
179
|
{ table: 'observation_files', column: 'last_cited_session_id' }, // v43
|
|
@@ -380,6 +381,20 @@ const MIGRATIONS = [
|
|
|
380
381
|
// makes pre-tool-recall skip environment-scoped rows on file-triggered
|
|
381
382
|
// injection; NULL always passes the filter.
|
|
382
383
|
'ALTER TABLE observations ADD COLUMN scope TEXT DEFAULT NULL',
|
|
384
|
+
// v46 (D#159): decay_seen_count AS IT STOOD when this observation was cited for
|
|
385
|
+
// the FIRST time. The lifetime counters already on the row cannot answer the
|
|
386
|
+
// question the "stop injecting a long-uncited memory" gate needs: measured
|
|
387
|
+
// 2026-08-22, a candidate gate of `decay_seen >= 20 AND cited_count = 0` matched
|
|
388
|
+
// 631 rows, while 331 of the 510 rows that HAVE been cited also carry a lifetime
|
|
389
|
+
// decay_seen >= 20 — whether those crossed 20 before or after their first citation
|
|
390
|
+
// is unrecoverable from cumulative counters, so the gate's false-kill rate is not
|
|
391
|
+
// computable. This column makes it computable going forward.
|
|
392
|
+
//
|
|
393
|
+
// NULLABLE ON PURPOSE, and NULL is not 0: NULL means "never cited", 1 means "cited
|
|
394
|
+
// on its very first decay resolution". A DEFAULT 0 would merge those two states and
|
|
395
|
+
// destroy the distinction the column exists to record. Legacy rows stay NULL — they
|
|
396
|
+
// are not evidence of anything and must not be read as first-cite-at-0.
|
|
397
|
+
'ALTER TABLE observations ADD COLUMN decay_seen_at_first_cite INTEGER DEFAULT NULL',
|
|
383
398
|
];
|
|
384
399
|
|
|
385
400
|
/**
|
|
@@ -9,6 +9,7 @@ import { liveObsFilterSql, injectionRelevanceSql } from '../lib/inject-search-co
|
|
|
9
9
|
import { fileMatchClause, fileMatchParams, basenameAnySep } from '../lib/file-edge-match.mjs';
|
|
10
10
|
import { cjkPrecisionOk } from '../nlp.mjs';
|
|
11
11
|
import { upsFtsQuery } from '../lib/ups-query.mjs';
|
|
12
|
+
import { corpusFloorScale } from '../lib/relevance-floor.mjs';
|
|
12
13
|
import { writeFileSync, readFileSync, existsSync, renameSync } from 'fs';
|
|
13
14
|
import { join, sep } from 'path';
|
|
14
15
|
import { pathToFileURL } from 'url';
|
|
@@ -130,103 +131,15 @@ const OR_TOP_BM25_FLOOR = TOP_REL_FLOOR === 0
|
|
|
130
131
|
|
|
131
132
|
// ─── Corpus-size normalization of the absolute floors (v3.61.0) ─────────────
|
|
132
133
|
//
|
|
133
|
-
//
|
|
134
|
-
//
|
|
135
|
-
//
|
|
136
|
-
//
|
|
134
|
+
// Moved to lib/relevance-floor.mjs when error-recall became the second injection
|
|
135
|
+
// face needing the same ramp — one body rather than two hand-mirrored copies, the
|
|
136
|
+
// drift class this project keeps paying for. The calibration history that explains
|
|
137
|
+
// the ramp SHAPE (the 0/8 fresh-install measurement, the ln-vs-idf re-measure)
|
|
138
|
+
// moved with the code; read it there before changing a floor.
|
|
137
139
|
//
|
|
138
|
-
//
|
|
139
|
-
//
|
|
140
|
-
|
|
141
|
-
// The floors were calibrated at `projects--mem, 584 obs` (CHANGELOG v2.43.x /
|
|
142
|
-
// v2.34.3). Comparing a log-N quantity against that constant therefore does not
|
|
143
|
-
// mean "weak match" on a small index — it means "small index". A brand-new
|
|
144
|
-
// install measured 0/8 injections on a realistic first-day corpus (10 memories,
|
|
145
|
-
// 8 recall questions whose correct target ranked #1 in 4/5 scored cases): every
|
|
146
|
-
// one was dropped by the OR floor at |bm25| 3.8–15.2 < 30. The plugin is inert
|
|
147
|
-
// during exactly the window where a new user decides whether it earns its keep.
|
|
148
|
-
//
|
|
149
|
-
// Fix: scale both floors by the corpus's MAX ATTAINABLE IDF over the reference
|
|
150
|
-
// corpus's, capped at 1.0, so the SIGNAL↔NOISE separation the maintainer measured
|
|
151
|
-
// (signal ≥41, noise ≤22 at N_REF) is preserved proportionally at any N. At
|
|
152
|
-
// N ≥ N_REF the factor is exactly 1.0, so every established install keeps
|
|
153
|
-
// byte-identical behavior; only genuinely-new installs relax.
|
|
154
|
-
//
|
|
155
|
-
// 2026-08-17 e2e round — the ramp shape. The first cut used ln(N+1)/ln(N_REF+1), which has the
|
|
156
|
-
// right asymptotics but the wrong small-N behavior: FTS5's IDF term is
|
|
157
|
-
// `log((N - df + 0.5) / (df + 0.5))`, which is EXACTLY 0 at N=2/df=1 and stays far
|
|
158
|
-
// below ln(N+1) for the whole first-week window. Re-measured end-to-end through the
|
|
159
|
-
// production write path (lib/save-observation.mjs — a raw INSERT skips CJK bigram
|
|
160
|
-
// expansion and understates the corpus, which is how the first cut's ramp table was
|
|
161
|
-
// misread), 1 planted target + topically clustered filler, CJK prose prompt carrying
|
|
162
|
-
// no identifier for the bypass to rescue:
|
|
163
|
-
//
|
|
164
|
-
// N 2 3 4 5 6 10 25 80
|
|
165
|
-
// top|bm25| 0.0 5.1 7.0 9.5 11.4 15.5 22.2 30.3
|
|
166
|
-
// ln ramp floor 5.2 6.5 7.6 8.4 9.2 11.3 15.3 20.7 ← DROP at N≤4
|
|
167
|
-
// idf ramp floor 0.0 2.6 4.3 5.5 6.5 9.3 14.1 20.0 ← admits all
|
|
168
|
-
//
|
|
169
|
-
// The two ramps agree within 8% at N≥30 and within 2% at N≥200, so this re-shape is
|
|
170
|
-
// confined to the window it is meant to fix. Accepted tradeoff: on a ≤2-row corpus the
|
|
171
|
-
// scale is EXACTLY 0 (FTS5's max IDF is 0 there), which disables both set-level floors
|
|
172
|
-
// rather than lowering them, and just above that they are small; so the best lexical match
|
|
173
|
-
// is injected even when it is weak.
|
|
174
|
-
// That is the intended trade — the alternative measured behavior is total silence, and
|
|
175
|
-
// a 4-row corpus has no room to bury signal under noise. The upstream
|
|
176
|
-
// hasExplicitSignal gate, not these floors, is what suppresses noise prompts.
|
|
177
|
-
//
|
|
178
|
-
// N counts the WHOLE observations table, not the project: FTS5 computes IDF over
|
|
179
|
-
// the entire index and `o.project = ?` is a post-MATCH filter. Verified — a
|
|
180
|
-
// 2-row project on a 302-row install scores 31.5, matching the 300-row global
|
|
181
|
-
// baseline, not the 10-row one. So a new project on an established install is
|
|
182
|
-
// (correctly) unaffected by this ramp.
|
|
183
|
-
const FLOOR_REF_CORPUS = Number(process.env.CLAUDE_MEM_UPS_FLOOR_REF_CORPUS || 584);
|
|
184
|
-
|
|
185
|
-
/**
|
|
186
|
-
* FTS5's IDF term for the best case a query can hit: a term appearing in exactly
|
|
187
|
-
* one row (df=1) of an n-row index. SQLite computes
|
|
188
|
-
* `log((n - df + 0.5) / (df + 0.5))`, so this is the ceiling any single-term bm25
|
|
189
|
-
* contribution can reach at corpus size n — the quantity the absolute floors are
|
|
190
|
-
* implicitly denominated in. Clamped at 0: below n=2 the formula goes negative,
|
|
191
|
-
* which as a scale would flip the comparison rather than relax it.
|
|
192
|
-
* @param {number} n Row count.
|
|
193
|
-
* @returns {number} Max attainable IDF at this corpus size, ≥ 0.
|
|
194
|
-
*/
|
|
195
|
-
function maxIdf(n) {
|
|
196
|
-
// n <= 1 makes the numerator non-positive → Math.log returns NaN or -Infinity, and
|
|
197
|
-
// Math.max(0, NaN) is NaN, not 0. Short-circuit instead: a 0- or 1-row index has no
|
|
198
|
-
// term that can discriminate, so the max attainable IDF is 0.
|
|
199
|
-
if (!(n > 1)) return 0;
|
|
200
|
-
return Math.max(0, Math.log((n - 1 + 0.5) / 1.5));
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
/**
|
|
204
|
-
* Scale factor in [0, 1] for the absolute score floors, by total corpus size.
|
|
205
|
-
*
|
|
206
|
-
* Short-circuits with a bounded probe: if a row exists at offset N_REF-1 the
|
|
207
|
-
* corpus is at or above the reference and the factor is 1.0 — no COUNT scan on
|
|
208
|
-
* the large corpora where the answer is always 1.0 anyway.
|
|
209
|
-
*
|
|
210
|
-
* @param {object} db Open better-sqlite3 handle.
|
|
211
|
-
* @returns {number} Multiplier for TOP_REL_FLOOR / OR_TOP_BM25_FLOOR.
|
|
212
|
-
*/
|
|
213
|
-
export function corpusFloorScale(db) {
|
|
214
|
-
if (FLOOR_REF_CORPUS <= 1) return 1;
|
|
215
|
-
try {
|
|
216
|
-
const atRef = db.prepare('SELECT 1 FROM observations LIMIT 1 OFFSET ?').get(FLOOR_REF_CORPUS - 1);
|
|
217
|
-
if (atRef) return 1;
|
|
218
|
-
const { c = 0 } = db.prepare('SELECT count(*) AS c FROM observations').get() || {};
|
|
219
|
-
const refIdf = maxIdf(FLOOR_REF_CORPUS);
|
|
220
|
-
// Degenerate reference (CLAUDE_MEM_UPS_FLOOR_REF_CORPUS set to 2 or 3, where
|
|
221
|
-
// maxIdf is 0 or near it): division would blow up or divide by zero. Treat the
|
|
222
|
-
// floors as fully calibrated, matching the FLOOR_REF_CORPUS <= 1 guard above.
|
|
223
|
-
if (!(refIdf > 0)) return 1;
|
|
224
|
-
return Math.min(1, maxIdf(c) / refIdf);
|
|
225
|
-
} catch {
|
|
226
|
-
// Any probe failure → behave exactly as before the ramp existed.
|
|
227
|
-
return 1;
|
|
228
|
-
}
|
|
229
|
-
}
|
|
140
|
+
// Re-exported so callers here and tests/ups-corpus-floor-scale.test.mjs keep the
|
|
141
|
+
// existing import path.
|
|
142
|
+
export { corpusFloorScale };
|
|
230
143
|
|
|
231
144
|
function isFollowUpSession(injectedIdsFile) {
|
|
232
145
|
try {
|
package/source-files.mjs
CHANGED
|
@@ -149,6 +149,14 @@ export const SOURCE_FILES = [
|
|
|
149
149
|
// search-engine.mjs AND the standalone hook scripts — missing it from the
|
|
150
150
|
// manifest kills every retrieval surface on auto-update.
|
|
151
151
|
'lib/inject-search-core.mjs',
|
|
152
|
+
// Error-triggered recall selection. Statically imported by hook.mjs (PostToolUse
|
|
153
|
+
// injection) and by benchmark/error-recall-suite.mjs (offline calibration) — the
|
|
154
|
+
// hook is the one that breaks on a missing manifest entry.
|
|
155
|
+
'lib/error-recall-core.mjs',
|
|
156
|
+
// Corpus-size ramp for absolute relevance floors. Imported by BOTH floor-bearing
|
|
157
|
+
// injection faces: scripts/user-prompt-search.js (standalone hook) and
|
|
158
|
+
// lib/error-recall-core.mjs. Missing here = UserPromptSubmit dies on auto-update.
|
|
159
|
+
'lib/relevance-floor.mjs',
|
|
152
160
|
// Shared UserPromptSubmit query caps — imported by BOTH hooks that event fires
|
|
153
161
|
// (scripts/user-prompt-search.js and hook.mjs user-prompt via hook-memory.mjs).
|
|
154
162
|
'lib/ups-query.mjs',
|