claude-mem-lite 3.91.0 → 3.92.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +1 -0
- package/README.zh-CN.md +1 -0
- package/claudemd.mjs +9 -6
- package/hook-context.mjs +9 -8
- package/hook-llm.mjs +56 -28
- package/hook-optimize.mjs +47 -28
- package/hook-shared.mjs +56 -11
- package/hook.mjs +94 -25
- package/install.mjs +9 -2
- package/lib/atomic-write.mjs +14 -3
- package/lib/cite-back-hint.mjs +3 -3
- package/lib/cite-recall-path.mjs +52 -0
- package/lib/compress-core.mjs +31 -15
- package/lib/frontmatter.mjs +67 -0
- package/lib/injected-ids.mjs +93 -4
- package/lib/maintain-core.mjs +191 -5
- package/lib/observation-write.mjs +71 -13
- package/lib/registry-core.mjs +117 -1
- package/lib/save-enrich.mjs +10 -6
- package/lib/save-observation.mjs +44 -0
- package/lib/transcript-scan.mjs +59 -8
- package/mem-cli.mjs +75 -193
- package/memdir.mjs +6 -6
- package/npm-shrinkwrap.json +20 -2
- package/package.json +9 -1
- package/registry-importer.mjs +4 -34
- package/registry.mjs +2 -1
- package/scripts/post-tool-recall.js +12 -4
- package/scripts/pre-tool-recall.js +9 -33
- package/scripts/user-prompt-search.js +13 -36
- package/search-scoring.mjs +10 -0
- package/server.mjs +68 -170
- package/source-files.mjs +2 -0
package/lib/maintain-core.mjs
CHANGED
|
@@ -18,6 +18,7 @@ import { DEDUP_JACCARD_THRESHOLD, MINHASH_PRE_THRESHOLD as MINHASH_PRE_THRESHOLD
|
|
|
18
18
|
import { liveObsFilterSql } from './inject-search-core.mjs';
|
|
19
19
|
|
|
20
20
|
import { DAY_MS } from './time-constants.mjs';
|
|
21
|
+
import { snapshotDb } from './db-backup.mjs';
|
|
21
22
|
export const STALE_AGE_MS = 30 * DAY_MS;
|
|
22
23
|
export const OP_CAP = 1000;
|
|
23
24
|
export const SCAN_LIMIT = 500;
|
|
@@ -494,8 +495,13 @@ export function mergeDuplicates(db, groups) {
|
|
|
494
495
|
return cur; // an id with no outgoing redirect is a keeper
|
|
495
496
|
};
|
|
496
497
|
|
|
497
|
-
|
|
498
|
-
|
|
498
|
+
// Liveness here is `liveObsFilterSql`, not compressed_into alone (audit 2026-09-02 P0-2):
|
|
499
|
+
// a superseded row keeps compressed_into=0, so the narrower predicate accepted a TOMBSTONE
|
|
500
|
+
// as a keeper and pointed a live row at it — the row then vanishes from every read face
|
|
501
|
+
// (all of which filter superseded_at) with no recovery path, because recoverOrphanedChildren
|
|
502
|
+
// only resurfaces children whose keeper row is GONE, and a tombstone still exists.
|
|
503
|
+
const isLive = db.prepare(`SELECT 1 FROM observations WHERE id = ? AND ${liveObsFilterSql('')}`);
|
|
504
|
+
const mergeStmt = db.prepare(`UPDATE observations SET compressed_into = ? WHERE id = ? AND ${liveObsFilterSql('')}`);
|
|
499
505
|
let merged = 0;
|
|
500
506
|
for (const removeId of redirect.keys()) {
|
|
501
507
|
const keeper = resolveKeeper(removeId);
|
|
@@ -569,7 +575,12 @@ export function findDuplicates(db, { projectFilter, baseParams, limit = SCAN_LIM
|
|
|
569
575
|
const recent = db.prepare(`
|
|
570
576
|
SELECT id, title, project, importance, access_count, created_at_epoch
|
|
571
577
|
FROM observations
|
|
572
|
-
|
|
578
|
+
-- liveObsFilterSql, not compressed_into alone (audit 2026-09-02 P0-2): this scan renders a
|
|
579
|
+
-- ready-to-paste "maintain execute --ops dedup --merge-ids A,B", and the keeper rule is
|
|
580
|
+
-- importance-based (mem-cli.mjs / server.mjs), so a superseded tombstone in the pair could
|
|
581
|
+
-- be nominated keeper. Same predicate the hook-side fuzzy dedup candidate query uses
|
|
582
|
+
-- (hook.mjs), and the same one mergeDuplicates now enforces at write time.
|
|
583
|
+
WHERE ${liveObsFilterSql('')} ${projectFilter}
|
|
573
584
|
ORDER BY created_at_epoch DESC LIMIT ${limit}
|
|
574
585
|
`).all(...baseParams);
|
|
575
586
|
|
|
@@ -652,8 +663,15 @@ export function rebuildVectors(db) {
|
|
|
652
663
|
return { ok: true, terms: vocab.terms.size, updated, total: allObs.length };
|
|
653
664
|
}
|
|
654
665
|
|
|
655
|
-
/**
|
|
656
|
-
|
|
666
|
+
/**
|
|
667
|
+
* VACUUM the whole DB, reporting freelist reclaim. Must run OUTSIDE any transaction.
|
|
668
|
+
*
|
|
669
|
+
* Module-private since P1-5: its only two consumers were mem-cli.mjs and server.mjs, and
|
|
670
|
+
* both now reach it through `runMaintainOps`. Exporting it again would add a name to the
|
|
671
|
+
* knip baseline for nobody (the v3.70.0 precedent, #9675) — export it the day something
|
|
672
|
+
* outside this module needs it.
|
|
673
|
+
*/
|
|
674
|
+
function vacuum(db) {
|
|
657
675
|
const pageSize = db.pragma('page_size', { simple: true });
|
|
658
676
|
const freeBefore = db.pragma('freelist_count', { simple: true });
|
|
659
677
|
db.exec('VACUUM');
|
|
@@ -661,3 +679,171 @@ export function vacuum(db) {
|
|
|
661
679
|
const reclaimedMB = ((Math.max(0, freeBefore - freeAfter) * pageSize) / 1048576).toFixed(1);
|
|
662
680
|
return { reclaimedMB, freeBefore, freeAfter };
|
|
663
681
|
}
|
|
682
|
+
|
|
683
|
+
// ─── The `maintain execute` sequence, once ──────────────────────────────────
|
|
684
|
+
//
|
|
685
|
+
// Audit 2026-09-02 P1-5. Every OPERATION above was already shared; the SEQUENCE was
|
|
686
|
+
// hand-copied into mem-cli.mjs and server.mjs, and had drifted three ways:
|
|
687
|
+
//
|
|
688
|
+
// * `server.mjs` rendered the demote_pinned line with a hardcoded `inj>=8` while
|
|
689
|
+
// mem-cli.mjs interpolated PINNED_INJ_THRESHOLD — change the constant and the MCP
|
|
690
|
+
// surface reports a threshold the code no longer uses.
|
|
691
|
+
// * the cap hint was a named `capHint()` on one face and the same ternary inlined
|
|
692
|
+
// four times on the other.
|
|
693
|
+
// * the comment explaining WHY demote_pinned must physically follow boost (boostAccessed
|
|
694
|
+
// lifts any access_count>3 row under importance 3, so demoting first and boosting
|
|
695
|
+
// second hands the row straight back inside one run) existed only on the CLI, i.e.
|
|
696
|
+
// the face where it was already right.
|
|
697
|
+
//
|
|
698
|
+
// The order is the contract, so it lives here with its reason. What stays with the
|
|
699
|
+
// surfaces is what genuinely differs: how a caller spells `merge_ids`, and what a purge
|
|
700
|
+
// preview tells the user to type next. Those arrive as explicit parameters rather than
|
|
701
|
+
// being re-implemented, which makes the differences reviewable in one place.
|
|
702
|
+
|
|
703
|
+
/**
|
|
704
|
+
* Run the `maintain execute` operation sequence and return the report lines.
|
|
705
|
+
*
|
|
706
|
+
* Owns the snapshot, the transaction boundary and the three post-transaction ops, because
|
|
707
|
+
* all three are part of the ordering contract: the pre-transaction snapshot counts only
|
|
708
|
+
* PRE-EXISTING pending rows, so purge must run before decay or a row is marked and deleted
|
|
709
|
+
* in one call with the backup skipped (audit HIGH-1), and VACUUM cannot run inside a
|
|
710
|
+
* transaction at all.
|
|
711
|
+
*
|
|
712
|
+
* @param {object} db
|
|
713
|
+
* @param {object} ctx { projectFilter, baseParams, staleAge, opCap }
|
|
714
|
+
* @param {string[]} ops
|
|
715
|
+
* @param {object} opts
|
|
716
|
+
* @param {number} [opts.retainDays=30]
|
|
717
|
+
* @param {number} opts.retainCutoff
|
|
718
|
+
* @param {boolean} [opts.confirmed=false] purge_stale is the only DELETE; unconfirmed previews
|
|
719
|
+
* @param {Array<number[]>|null} [opts.mergeGroups] parsed dedup groups, already validated
|
|
720
|
+
* @param {boolean} [opts.mergeIdsProvided] true when the caller passed merge ids at all
|
|
721
|
+
* @param {string[]} [opts.invalidMergeSegments] malformed segments to warn about (CLI only)
|
|
722
|
+
* @param {string} [opts.mergeIdsFlagName] how this surface spells the flag, for the warning
|
|
723
|
+
* @param {Function} opts.renderPurgePreview (previewRow, retainDays) => string
|
|
724
|
+
* @param {Function} [opts.onError] (err, scope) => void, for the two catch arms
|
|
725
|
+
* @returns {string[]} report lines, in order
|
|
726
|
+
*/
|
|
727
|
+
export function runMaintainOps(db, ctx, ops, {
|
|
728
|
+
retainDays = 30,
|
|
729
|
+
retainCutoff,
|
|
730
|
+
confirmed = false,
|
|
731
|
+
mergeGroups = null,
|
|
732
|
+
mergeIdsProvided = false,
|
|
733
|
+
invalidMergeSegments = [],
|
|
734
|
+
mergeIdsFlagName = 'merge_ids',
|
|
735
|
+
renderPurgePreview,
|
|
736
|
+
onError = () => {},
|
|
737
|
+
} = {}) {
|
|
738
|
+
const results = [];
|
|
739
|
+
const opCap = ctx.opCap ?? OP_CAP;
|
|
740
|
+
const capHint = (changes) => (changes >= opCap ? ' (cap reached, re-run for more)' : '');
|
|
741
|
+
|
|
742
|
+
// Snapshot before the irreversible hard deletes, and only when rows will actually be
|
|
743
|
+
// removed. OUTSIDE the transaction below — VACUUM cannot run inside one, and a snapshot
|
|
744
|
+
// taken inside would not see a consistent pre-state anyway. Best-effort; never throws.
|
|
745
|
+
const willPurge = ops.includes('purge_stale') && confirmed;
|
|
746
|
+
if (hardDeleteCandidateCount(db, ctx, { cleanup: ops.includes('cleanup'), purge: willPurge }) > 0) {
|
|
747
|
+
snapshotDb(db, { tag: 'pre-maintain' });
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
db.transaction(() => {
|
|
751
|
+
// PURGE FIRST — same order as handleAutoMaintain. Running decay before purge in one
|
|
752
|
+
// transaction marks a stale row pending-purge AND deletes it in the SAME call (zero
|
|
753
|
+
// grace), and the snapshot above counts only PRE-EXISTING pending rows, so it skips the
|
|
754
|
+
// backup: permanent, unrecoverable loss of notable imp-2/3 memories (audit HIGH-1).
|
|
755
|
+
// Purging first deletes only rows a PRIOR run marked — which the guard saw and backed
|
|
756
|
+
// up — and rows decay marks below wait for the next run, regaining the grace cycle.
|
|
757
|
+
if (ops.includes('purge_stale')) {
|
|
758
|
+
if (!confirmed) {
|
|
759
|
+
results.push(renderPurgePreview(purgeStalePreview(db, ctx, retainCutoff), retainDays));
|
|
760
|
+
} else {
|
|
761
|
+
const purged = purgeStale(db, ctx, retainCutoff);
|
|
762
|
+
results.push(`Purged ${purged} stale observations (retained last ${retainDays} days)${capHint(purged)}`);
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
if (ops.includes('cleanup')) {
|
|
767
|
+
const deleted = cleanupBroken(db, ctx);
|
|
768
|
+
results.push(`Cleaned up ${deleted} broken observations${capHint(deleted)}`);
|
|
769
|
+
// Self-heal legacy orphans (keeper hard-deleted pre-recoverChildrenOf): resurface
|
|
770
|
+
// unreachable children. Non-destructive — un-hide only, no delete.
|
|
771
|
+
const orphans = recoverOrphanedChildren(db, ctx);
|
|
772
|
+
if (orphans > 0) results.push(`Recovered ${orphans} orphaned compression children`);
|
|
773
|
+
// Heal lesson rows citation-decay buried at importance 0 (pre floor=1). 0→1 on
|
|
774
|
+
// lesson-bearing rows only; idempotent no-op once none remain.
|
|
775
|
+
const lessonsHealed = recoverBuriedLessons(db, ctx);
|
|
776
|
+
if (lessonsHealed > 0) results.push(`Healed ${lessonsHealed} lesson rows buried at importance 0`);
|
|
777
|
+
// Heal deferred_work rows whose closing obs / source prompt was hard-deleted while FK
|
|
778
|
+
// was OFF (dangling refs foreign_key_check flags). Applies the FK's ON DELETE SET NULL.
|
|
779
|
+
const deferredHealed = sweepDeferredWorkOrphans(db, ctx);
|
|
780
|
+
if (deferredHealed > 0) results.push(`Healed ${deferredHealed} deferred-work rows with dangling references`);
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
if (ops.includes('decay')) {
|
|
784
|
+
// injection_count>0 protected (decayAndMarkIdle) — an obs Claude was shown 8× is
|
|
785
|
+
// contextually proven. The MCP copy lacked this clause before the extraction.
|
|
786
|
+
const { decayed, idleMarked } = decayAndMarkIdle(db, ctx);
|
|
787
|
+
results.push(`Decayed ${decayed} stale observations, marked ${idleMarked} idle as pending-purge${capHint(Math.max(decayed, idleMarked))}`);
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
if (ops.includes('boost')) {
|
|
791
|
+
const boosted = boostAccessed(db, ctx);
|
|
792
|
+
results.push(`Boosted ${boosted} frequently-accessed observations${capHint(boosted)}`);
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
// AFTER boost, and the order is load-bearing in one direction only: boostAccessed lifts
|
|
796
|
+
// any access_count>3 row with importance<3, so demoting a pinned row to 1 and then
|
|
797
|
+
// boosting hands it straight back at 2 — the demotion silently undone inside a single
|
|
798
|
+
// run. DEFAULT_MAINTAIN_OPS pins the order; this block has to physically follow the
|
|
799
|
+
// boost block for that order to be real.
|
|
800
|
+
if (ops.includes('demote_pinned')) {
|
|
801
|
+
// Repairs the citation-decay blind spot: decay protects injection_count>0, so a
|
|
802
|
+
// heavily-injected-but-uncited memory stays pinned at max importance forever.
|
|
803
|
+
// Floor, not purge: no lesson_learned → 1, lesson-bearing → 2 (v3.76.1 dual floor).
|
|
804
|
+
const demoted = demotePinned(db, ctx);
|
|
805
|
+
results.push(`Demoted ${demoted} pinned-but-uncited observations (inj>=${PINNED_INJ_THRESHOLD}, cited=0; no lesson → importance 1, lesson → 2)${capHint(demoted)}`);
|
|
806
|
+
}
|
|
807
|
+
|
|
808
|
+
if (ops.includes('dedup') && mergeIdsProvided) {
|
|
809
|
+
const totalMerged = mergeDuplicates(db, mergeGroups || []);
|
|
810
|
+
if (invalidMergeSegments.length) {
|
|
811
|
+
results.push(`Warning: ignored ${invalidMergeSegments.length} malformed ${mergeIdsFlagName} segment(s): ${invalidMergeSegments.join(', ')} (expected keepId:removeId[:removeId...] with positive integers)`);
|
|
812
|
+
}
|
|
813
|
+
results.push(`Merged ${totalMerged} duplicate observations`);
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
if (!ops.includes('dedup') && mergeIdsProvided) {
|
|
817
|
+
results.push(`Warning: ${mergeIdsFlagName} provided but "dedup" not in operations — ${mergeIdsFlagName} ignored`);
|
|
818
|
+
}
|
|
819
|
+
})();
|
|
820
|
+
|
|
821
|
+
// FTS5 optimize — outside the transaction.
|
|
822
|
+
db.exec("INSERT INTO observations_fts(observations_fts) VALUES('optimize')");
|
|
823
|
+
results.push('FTS5 index optimized');
|
|
824
|
+
|
|
825
|
+
if (ops.includes('rebuild_vectors')) {
|
|
826
|
+
try {
|
|
827
|
+
const r = rebuildVectors(db);
|
|
828
|
+
results.push(r.ok
|
|
829
|
+
? `Vectors: rebuilt vocabulary (${r.terms} terms), updated ${r.updated}/${r.total} vectors`
|
|
830
|
+
: `Vectors: ${r.reason}`);
|
|
831
|
+
} catch (e) {
|
|
832
|
+
onError(e, 'rebuild_vectors');
|
|
833
|
+
results.push(`Vectors: rebuild failed — ${e.message}`);
|
|
834
|
+
}
|
|
835
|
+
}
|
|
836
|
+
|
|
837
|
+
// VACUUM: reclaim freelist pages left by DELETEs. Whole-DB, outside any transaction.
|
|
838
|
+
if (ops.includes('vacuum')) {
|
|
839
|
+
try {
|
|
840
|
+
const v = vacuum(db);
|
|
841
|
+
results.push(`VACUUM: reclaimed ~${v.reclaimedMB}MB (freelist ${v.freeBefore} → ${v.freeAfter} pages)`);
|
|
842
|
+
} catch (e) {
|
|
843
|
+
onError(e, 'vacuum');
|
|
844
|
+
results.push(`VACUUM failed — ${e.message}`);
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
|
|
848
|
+
return results;
|
|
849
|
+
}
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
// Statement-only: callers own the transaction boundary (both wrap the row + files
|
|
9
9
|
// + vector writes in one db.transaction so a failure can't leave a partial row).
|
|
10
10
|
|
|
11
|
-
import { getVocabulary, computeVector, vectorsEnabled } from '../tfidf.mjs';
|
|
11
|
+
import { getVocabulary, computeVector, vectorsEnabled, vecTextForRow } from '../tfidf.mjs';
|
|
12
12
|
import { debugCatch, cjkBigrams, scrubSecrets } from '../utils.mjs';
|
|
13
13
|
|
|
14
14
|
// Canonical column order — must mirror the observations schema (schema.mjs).
|
|
@@ -69,21 +69,79 @@ export function insertObservationFiles(db, obsId, files) {
|
|
|
69
69
|
for (const f of files) if (typeof f === 'string' && f.length > 0) stmt.run(obsId, f);
|
|
70
70
|
}
|
|
71
71
|
|
|
72
|
+
// ─── The observation_vectors upsert, once ───────────────────────────────────
|
|
73
|
+
//
|
|
74
|
+
// Audit 2026-09-02 P1-4: this one statement shipped FIVE times — here, hook-optimize's
|
|
75
|
+
// rebuildVector, hook-llm's enrich path, lib/compress-core's summary write and
|
|
76
|
+
// maintain-core's bulk rebuild — and it had already drifted once: hook-optimize wrote the
|
|
77
|
+
// column as `computed_at` instead of `created_at_epoch`, silently swallowed by its own
|
|
78
|
+
// catch until an experiment surfaced it.
|
|
79
|
+
//
|
|
80
|
+
// Four of the five now come through `upsertObservationVector`. The fifth,
|
|
81
|
+
// `maintain-core.rebuildVectors`, deliberately does NOT: it rebuilds the vocabulary
|
|
82
|
+
// itself (so it must pass its own, not read the cache), reuses one prepared statement
|
|
83
|
+
// across every live row, and lets a throw abort the whole rebuild rather than skipping a
|
|
84
|
+
// row. Forcing it through a per-row best-effort helper would change all three. Its SQL is
|
|
85
|
+
// the one remaining copy, and it is a copy on purpose.
|
|
86
|
+
//
|
|
87
|
+
// The `vectorsEnabled()` gate is a PARAMETER, not a decision made here, because the five
|
|
88
|
+
// sites did not agree and this refactor preserves each one's behaviour rather than
|
|
89
|
+
// picking a winner. Worth knowing before reading that as a live defect: `getVocabulary`
|
|
90
|
+
// returns null whenever the arm is disabled, and every path guards on the vocab, so the
|
|
91
|
+
// gate difference has no behavioural consequence today — it is a redundancy, not a hole.
|
|
92
|
+
const VECTOR_UPSERT_SQL =
|
|
93
|
+
'INSERT OR REPLACE INTO observation_vectors (observation_id, vector, vocab_version, created_at_epoch) VALUES (?, ?, ?, ?)';
|
|
94
|
+
|
|
72
95
|
/**
|
|
73
|
-
*
|
|
74
|
-
*
|
|
75
|
-
*
|
|
96
|
+
* The text a vector is computed from, for the three shapes callers hold.
|
|
97
|
+
*
|
|
98
|
+
* @param {string|string[]|object} textOrRow a ready string, parts to join, or an
|
|
99
|
+
* observations row (which goes through `vecTextForRow`, so a rebuild derives exactly
|
|
100
|
+
* what the save path derived — a second concatenation here would be the next drift).
|
|
76
101
|
*/
|
|
77
|
-
|
|
78
|
-
if (
|
|
102
|
+
function vectorTextFor(textOrRow) {
|
|
103
|
+
if (typeof textOrRow === 'string') return textOrRow;
|
|
104
|
+
// `filter(Boolean)` is inherited from rebuildVector and is INERT, measured: a null or
|
|
105
|
+
// empty entry joins to extra whitespace, which the tokenizer collapses, so the vector
|
|
106
|
+
// is byte-identical with or without it. Kept rather than deleted because it is
|
|
107
|
+
// defensive code that costs nothing — but do not write a test for it, because no
|
|
108
|
+
// mutation of it can fail one.
|
|
109
|
+
if (Array.isArray(textOrRow)) return textOrRow.filter(Boolean).join(' ');
|
|
110
|
+
return vecTextForRow(textOrRow);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Best-effort TF-IDF vector write. Non-critical: vocab may be uninitialized on a fresh DB,
|
|
115
|
+
* so failures are swallowed (the caller's transaction must NOT roll back an observation
|
|
116
|
+
* over a missing vector).
|
|
117
|
+
*
|
|
118
|
+
* @param {object} db
|
|
119
|
+
* @param {number} obsId
|
|
120
|
+
* @param {string|string[]|object} textOrRow
|
|
121
|
+
* @param {object} [opts]
|
|
122
|
+
* @param {boolean} [opts.gate=true] apply `vectorsEnabled()` — see the note above on why
|
|
123
|
+
* this is a parameter
|
|
124
|
+
* @param {object} [opts.vocab] use this vocabulary instead of `getVocabulary(db)`
|
|
125
|
+
* @param {number} [opts.at] created_at_epoch (compress-core stamps the summary's
|
|
126
|
+
* median date, not now)
|
|
127
|
+
* @param {string} [opts.scope] debugCatch scope, so a failure still names its caller
|
|
128
|
+
* @returns {boolean} whether a row was written
|
|
129
|
+
*/
|
|
130
|
+
export function upsertObservationVector(db, obsId, textOrRow, { gate = true, vocab = null, at = null, scope = 'upsertObservationVector' } = {}) {
|
|
131
|
+
if (gate && !vectorsEnabled()) return false; // Phase-1: vector arm off by default (audit 2026-06-27)
|
|
79
132
|
try {
|
|
80
|
-
const
|
|
81
|
-
if (!
|
|
82
|
-
const vec = computeVector(
|
|
83
|
-
if (!vec) return;
|
|
84
|
-
db.prepare(
|
|
85
|
-
|
|
86
|
-
} catch (e) { debugCatch(e,
|
|
133
|
+
const v = vocab ?? getVocabulary(db);
|
|
134
|
+
if (!v) return false;
|
|
135
|
+
const vec = computeVector(vectorTextFor(textOrRow), v);
|
|
136
|
+
if (!vec) return false;
|
|
137
|
+
db.prepare(VECTOR_UPSERT_SQL).run(obsId, Buffer.from(vec.buffer), v.version, at ?? Date.now());
|
|
138
|
+
return true;
|
|
139
|
+
} catch (e) { debugCatch(e, scope); return false; }
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** The save path's vector write. Thin wrapper — kept as a name because it has consumers. */
|
|
143
|
+
export function insertObservationVector(db, obsId, vecText) {
|
|
144
|
+
upsertObservationVector(db, obsId, vecText, { scope: 'insertObservationVector' });
|
|
87
145
|
}
|
|
88
146
|
|
|
89
147
|
/**
|
package/lib/registry-core.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// Shared body for the registry write actions — import / remove / reindex.
|
|
1
|
+
// Shared body for the registry write actions — import / remove / reindex / enrich.
|
|
2
2
|
//
|
|
3
3
|
// These were the last un-collapsed CLI/MCP twin (audit 2026-08-22 P1-3): mem-cli.mjs and
|
|
4
4
|
// server.mjs each wrote their own SQL for the same three actions, and had already drifted —
|
|
@@ -11,7 +11,10 @@
|
|
|
11
11
|
// format output or touch process state — rendering (and CLI-only concerns like bare-flag
|
|
12
12
|
// rejection or the "add --capability-summary" tip) stays in the surface.
|
|
13
13
|
|
|
14
|
+
import { readFileSync } from 'fs';
|
|
15
|
+
|
|
14
16
|
import { upsertResource } from '../registry.mjs';
|
|
17
|
+
import { isPathConfined } from '../utils.mjs';
|
|
15
18
|
|
|
16
19
|
/**
|
|
17
20
|
* String columns an import may set, in canonical snake_case. Single source: the CLI derives
|
|
@@ -92,3 +95,116 @@ export function reindexResources(db) {
|
|
|
92
95
|
const row = db.prepare('SELECT COUNT(*) as c FROM resources WHERE status = ?').get('active');
|
|
93
96
|
return { activeCount: row.c };
|
|
94
97
|
}
|
|
98
|
+
|
|
99
|
+
// ─── Enrichment (four legs, one gate) ───────────────────────────────────────
|
|
100
|
+
//
|
|
101
|
+
// `resources.local_path` is a filesystem path stored in a DB row, and every enrichment leg
|
|
102
|
+
// ends in `readFileSync(local_path)`. Before audit 2026-09-02 P1-3 the confinement check
|
|
103
|
+
// guarded exactly ONE of the four legs — the MCP `enrich` action — while the MCP
|
|
104
|
+
// `import_url --enrich` leg, the CLI `enrich <name>` leg and the CLI `enrich --all` leg read
|
|
105
|
+
// the path bare. That is this repo's first-listed 病类 in its narrowest form: not "CLI vs
|
|
106
|
+
// MCP" but "one branch of one face". Counting the legs is what showed the MCP side was
|
|
107
|
+
// itself half-guarded, so a CLI-only patch would have left the shape alive on both faces.
|
|
108
|
+
//
|
|
109
|
+
// Every leg now funnels through `enrichResourceRow`, so a new leg cannot be written without
|
|
110
|
+
// passing `confineTo` — and omitting it is a visible `undefined`, not a silently absent `if`.
|
|
111
|
+
|
|
112
|
+
/** Env var that disables the confinement gate (escape hatch for relocated managed dirs). */
|
|
113
|
+
export const REGISTRY_CONFINE_ENV = 'CLAUDE_MEM_REGISTRY_CONFINE';
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Whether the confinement gate is active. Only `off` disables it — case-insensitively and
|
|
117
|
+
* ignoring surrounding whitespace, so `OFF` and ` off\n` also open the gate. Every OTHER
|
|
118
|
+
* value, including an unset one and every near-miss (`of`, `0`, `false`, `disable`), keeps
|
|
119
|
+
* the gate ON, because the failure mode of a typo must be "refuses a path it could have
|
|
120
|
+
* read", never "reads a path it should have refused". Said precisely because the first
|
|
121
|
+
* version of this sentence claimed "the exact string `off`" in three places, which is the
|
|
122
|
+
* wrong description of a `.trim().toLowerCase()` — the safety property held, the wording
|
|
123
|
+
* did not.
|
|
124
|
+
*/
|
|
125
|
+
export function registryConfineEnabled(env = process.env) {
|
|
126
|
+
return String(env[REGISTRY_CONFINE_ENV] ?? '').trim().toLowerCase() !== 'off';
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Enrich one already-resolved resource row.
|
|
131
|
+
*
|
|
132
|
+
* @param {object} db open resource-registry.db handle
|
|
133
|
+
* @param {object} row must carry {name, type, local_path}
|
|
134
|
+
* @param {object} opts
|
|
135
|
+
* @param {string} opts.confineTo base dir `local_path` must stay within
|
|
136
|
+
* @param {Function} opts.enrichResource injected `(db,name,type,content) => Promise<boolean>`;
|
|
137
|
+
* injected rather than imported so this module stays a
|
|
138
|
+
* leaf and the Anthropic SDK stays off the CLI's
|
|
139
|
+
* cold-start path (both faces already lazy-import it)
|
|
140
|
+
* @param {object} [opts.env] env source for the escape hatch (tests pass their own)
|
|
141
|
+
* @returns {Promise<{status:'enriched'|'failed'|'no-path'|'denied'|'unreadable', error?:Error}>}
|
|
142
|
+
* `error` is carried on 'unreadable' only: a stale local_path (the file was moved or
|
|
143
|
+
* deleted after import) is the common case, and the errno is the whole diagnosis —
|
|
144
|
+
* collapsing it into a bare status would make this refactor a diagnosability
|
|
145
|
+
* regression on the one leg that already surfaced the message.
|
|
146
|
+
*/
|
|
147
|
+
export async function enrichResourceRow(db, row, { confineTo, enrichResource, env } = {}) {
|
|
148
|
+
// `confineTo` is REQUIRED, and a missing one throws rather than defaulting to "no gate".
|
|
149
|
+
// The defect this replaces was three call sites that simply had no `if` — invisible in
|
|
150
|
+
// review because absence has no syntax. A fifth leg written without the gate now fails
|
|
151
|
+
// loudly on its first call instead of shipping as a silent hole; the static sweep in
|
|
152
|
+
// tests/registry-enrich-confinement.test.mjs catches it earlier still.
|
|
153
|
+
if (!confineTo) throw new TypeError('enrichResourceRow: confineTo is required');
|
|
154
|
+
if (!row?.local_path) return { status: 'no-path' };
|
|
155
|
+
if (registryConfineEnabled(env) && !isPathConfined(row.local_path, confineTo)) {
|
|
156
|
+
return { status: 'denied' };
|
|
157
|
+
}
|
|
158
|
+
let content;
|
|
159
|
+
try {
|
|
160
|
+
content = readFileSync(row.local_path, 'utf8');
|
|
161
|
+
} catch (e) {
|
|
162
|
+
return { status: 'unreadable', error: e };
|
|
163
|
+
}
|
|
164
|
+
// Enrichment itself is an LLM round-trip; a throw here is a failed enrichment, not a
|
|
165
|
+
// refused read, and the two must stay distinguishable in the counts the surfaces render.
|
|
166
|
+
try {
|
|
167
|
+
return { status: (await enrichResource(db, row.name, row.type, content)) ? 'enriched' : 'failed' };
|
|
168
|
+
} catch (e) {
|
|
169
|
+
return { status: 'failed', error: e };
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* Enrich the rows an import just produced (MCP `import_url --enrich` + CLI `import --enrich`).
|
|
175
|
+
*
|
|
176
|
+
* `results` carries only {id, name, type}; local_path is read back per row because
|
|
177
|
+
* importFromGitHub writes it during the import.
|
|
178
|
+
*
|
|
179
|
+
* @returns {Promise<{ok:number, denied:number, total:number}>}
|
|
180
|
+
*/
|
|
181
|
+
export async function enrichImportedResources(db, results, { confineTo, enrichResource, env } = {}) {
|
|
182
|
+
// Checked here too, not only in the delegate: an empty `results` never reaches
|
|
183
|
+
// enrichResourceRow, so an ungated caller would otherwise pass on the empty input a test
|
|
184
|
+
// is most likely to use and throw only in production.
|
|
185
|
+
if (!confineTo) throw new TypeError('enrichImportedResources: confineTo is required');
|
|
186
|
+
let ok = 0, denied = 0;
|
|
187
|
+
for (const r of results) {
|
|
188
|
+
const row = db.prepare('SELECT name, type, local_path FROM resources WHERE id = ?').get(r.id);
|
|
189
|
+
const { status } = await enrichResourceRow(db, row, { confineTo, enrichResource, env });
|
|
190
|
+
if (status === 'enriched') ok++;
|
|
191
|
+
else if (status === 'denied') denied++;
|
|
192
|
+
}
|
|
193
|
+
return { ok, denied, total: results.length };
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* Resolve one active resource by name and enrich it (MCP `enrich` + CLI `enrich <name>`).
|
|
198
|
+
* @returns {Promise<{status:'not-found'|'enriched'|'failed'|'no-path'|'denied'|'unreadable', name:string, error?:Error}>}
|
|
199
|
+
*/
|
|
200
|
+
export async function enrichNamedResource(db, name, { confineTo, enrichResource, env } = {}) {
|
|
201
|
+
// Same reason as enrichImportedResources: the 'not-found' return short-circuits above the
|
|
202
|
+
// delegate, so the gate check cannot be left to it.
|
|
203
|
+
if (!confineTo) throw new TypeError('enrichNamedResource: confineTo is required');
|
|
204
|
+
const row = db
|
|
205
|
+
.prepare("SELECT name, type, local_path FROM resources WHERE name = ? AND status = 'active'")
|
|
206
|
+
.get(name);
|
|
207
|
+
if (!row) return { status: 'not-found', name };
|
|
208
|
+
const { status, error } = await enrichResourceRow(db, row, { confineTo, enrichResource, env });
|
|
209
|
+
return { status, name: row.name, error };
|
|
210
|
+
}
|
package/lib/save-enrich.mjs
CHANGED
|
@@ -33,7 +33,7 @@ import { join, dirname } from 'path';
|
|
|
33
33
|
import { fileURLToPath } from 'url';
|
|
34
34
|
import { scrubSecrets, cjkBigrams, truncate, debugCatch } from '../utils.mjs';
|
|
35
35
|
import { scrubRecord } from './scrub-record.mjs';
|
|
36
|
-
import { normalizeScope, SCOPE_PROMPT_LEGEND } from './observation-write.mjs';
|
|
36
|
+
import { normalizeScope, SCOPE_PROMPT_LEGEND, upsertObservationVector } from './observation-write.mjs';
|
|
37
37
|
|
|
38
38
|
export const ENRICH_OBLIGATED_TYPES = new Set(['bugfix', 'decision']);
|
|
39
39
|
|
|
@@ -170,13 +170,17 @@ scope: ${SCOPE_PROMPT_LEGEND}`;
|
|
|
170
170
|
}
|
|
171
171
|
|
|
172
172
|
if (enriched) {
|
|
173
|
-
// Refresh the TF-IDF vector so backfilled lesson/aliases reach the vector
|
|
174
|
-
//
|
|
175
|
-
//
|
|
173
|
+
// Refresh the TF-IDF vector so backfilled lesson/aliases reach the vector arm too.
|
|
174
|
+
//
|
|
175
|
+
// This used to `await import('../hook-optimize.mjs')` for its `rebuildVector` — the
|
|
176
|
+
// ONLY lib -> hook-layer edge in the tree, and a lazy one specifically so the save
|
|
177
|
+
// surfaces would not drag in a 1139-line optimize stack to write one row (audit
|
|
178
|
+
// 2026-09-02 P1-4). The function's body now lives in lib/observation-write.mjs, so the
|
|
179
|
+
// edge is gone and the import is static and cheap. `gate: false` matches what
|
|
180
|
+
// rebuildVector did.
|
|
176
181
|
try {
|
|
177
|
-
const { rebuildVector } = await import('../hook-optimize.mjs');
|
|
178
182
|
const full = db.prepare('SELECT * FROM observations WHERE id = ?').get(id);
|
|
179
|
-
if (full)
|
|
183
|
+
if (full) upsertObservationVector(db, id, full, { gate: false, scope: 'save-enrich-vector' });
|
|
180
184
|
} catch (e) { debugCatch(e, 'save-enrich-vector'); }
|
|
181
185
|
}
|
|
182
186
|
return { enriched };
|
package/lib/save-observation.mjs
CHANGED
|
@@ -14,6 +14,9 @@
|
|
|
14
14
|
import { jaccardSimilarity, scrubSecrets, computeMinHash, cjkBigrams, getCurrentBranch } from '../utils.mjs';
|
|
15
15
|
import { DEDUP_JACCARD_THRESHOLD } from './dedup-constants.mjs';
|
|
16
16
|
import { insertObservationRow, insertObservationFiles, insertObservationVector } from './observation-write.mjs';
|
|
17
|
+
// Imported, not injected: `allowStatuses` below is the POLICY this function exists to hold,
|
|
18
|
+
// and a caller free to pass its own resolver could reinstate the one-way gate D#195 closed.
|
|
19
|
+
import { resolveDeferredIds, closeDeferredItems } from './deferred-work.mjs';
|
|
17
20
|
|
|
18
21
|
const DEDUP_WINDOW_MS = 5 * 60 * 1000;
|
|
19
22
|
const DEDUP_RECENT_LIMIT = 50;
|
|
@@ -385,3 +388,44 @@ export function saveObservation(db, params) {
|
|
|
385
388
|
supersedeSkipped: [...malformedSupersedes, ...skipped],
|
|
386
389
|
};
|
|
387
390
|
}
|
|
391
|
+
|
|
392
|
+
/**
|
|
393
|
+
* Save one observation and, on a non-duplicate save, close the deferred items it closes —
|
|
394
|
+
* in ONE transaction.
|
|
395
|
+
*
|
|
396
|
+
* Audit 2026-09-02 P1-6. `server.mjs`'s mem_save and `mem-cli.mjs`'s cmdSave each wrote
|
|
397
|
+
* this transaction body out, and each carried a comment saying it was "kept in sync with"
|
|
398
|
+
* the other. Two things inside it are policy rather than plumbing, which is why a comment
|
|
399
|
+
* was never enough:
|
|
400
|
+
*
|
|
401
|
+
* * the dedup SHORT-CIRCUIT must happen before the resolver, so replaying the same save
|
|
402
|
+
* is idempotent — a duplicate must not re-close (or fail to re-close) a deferred item
|
|
403
|
+
* that has already transitioned out of 'open';
|
|
404
|
+
* * `allowStatuses: ['open', 'dropped']` (D#195) — `defer drop` on an item that was
|
|
405
|
+
* actually fixed used to be a one-way gate that permanently lost the obs link.
|
|
406
|
+
*
|
|
407
|
+
* The CALLER keeps its own catch: the two surfaces report a failure differently (MCP
|
|
408
|
+
* re-throws with a prefix so the tool response names the contract failure, the CLI writes
|
|
409
|
+
* to stderr and exits), and both need `closesTokens` to decide which message to use.
|
|
410
|
+
*
|
|
411
|
+
* @param {object} db
|
|
412
|
+
* @param {object} params as `saveObservation`
|
|
413
|
+
* @param {object} opts
|
|
414
|
+
* @param {string[]|null} [opts.closesTokens] deferred tokens the caller asked to close
|
|
415
|
+
* @param {string} opts.project
|
|
416
|
+
* @returns {{result: object, closesIds: number[]|null}}
|
|
417
|
+
*/
|
|
418
|
+
export function saveWithClosures(db, params, { closesTokens, project }) {
|
|
419
|
+
let closesIds = null;
|
|
420
|
+
const result = db.transaction(() => {
|
|
421
|
+
const r = saveObservation(db, params);
|
|
422
|
+
// BEFORE the resolver, deliberately — see the docblock.
|
|
423
|
+
if (r.kind === 'duplicate') return r;
|
|
424
|
+
if (closesTokens && closesTokens.length > 0) {
|
|
425
|
+
closesIds = resolveDeferredIds(db, project, closesTokens, { allowStatuses: ['open', 'dropped'] });
|
|
426
|
+
closeDeferredItems(db, closesIds, r.id);
|
|
427
|
+
}
|
|
428
|
+
return r;
|
|
429
|
+
})();
|
|
430
|
+
return { result, closesIds };
|
|
431
|
+
}
|
package/lib/transcript-scan.mjs
CHANGED
|
@@ -12,13 +12,56 @@
|
|
|
12
12
|
// This is a memo, not a rewrite: each scanner keeps its own per-entry logic exactly as
|
|
13
13
|
// it was, and only stops re-reading and re-parsing the file to get at it.
|
|
14
14
|
import { readFileSync, existsSync, statSync } from 'fs';
|
|
15
|
+
import { getHeapStatistics } from 'v8';
|
|
15
16
|
|
|
16
|
-
//
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
//
|
|
20
|
-
//
|
|
17
|
+
// Parsed entries cost ~3.45× the file size in heap (measured: 5.7MB file → 19.5MB).
|
|
18
|
+
export const TRANSCRIPT_ENTRY_HEAP_FACTOR = 3.45;
|
|
19
|
+
|
|
20
|
+
// Retention cap, kept as the FLOOR of the heap-aware budget below.
|
|
21
|
+
//
|
|
22
|
+
// Audit 2026-09-02 P2-12: above this cap the memo is declined and every caller parses for
|
|
23
|
+
// itself, so `handleStop` — which asks this file twelve questions — degrades to twelve full
|
|
24
|
+
// parses at exactly the size where one parse is already expensive. The audit's proposed fix
|
|
25
|
+
// was to cache a projected subset of each entry instead. That was NOT taken: the twelve
|
|
26
|
+
// scanners read arbitrary fields, so a projection is a hand-maintained field manifest, and
|
|
27
|
+
// a scanner reading a field nobody remembered to project sees `undefined` and silently
|
|
28
|
+
// answers a narrower question — this repo's most-repeated defect, traded for a path no
|
|
29
|
+
// transcript here reaches (112 transcripts on the machine that filed the audit, largest
|
|
30
|
+
// 4.9MB, ZERO above the cap; the report's "50MB ≈ 3.8s" is an extrapolation from a 4.37MB
|
|
31
|
+
// measurement, not an observation).
|
|
32
|
+
//
|
|
33
|
+
// What changed instead: the cap is no longer a bare constant. The thing it is protecting is
|
|
34
|
+
// the heap, so it is expressed against the heap — a quarter of this process's limit, capped
|
|
35
|
+
// at 256MB and floored at the old 24MB. On an ordinary 64-bit Node (~4GB limit) that admits
|
|
36
|
+
// transcripts up to ~74MB, i.e. the degenerate case the audit described now takes one parse
|
|
37
|
+
// instead of twelve; under a constrained limit (a 512MB CI container) the budget shrinks
|
|
38
|
+
// with it and can fall BELOW the old constant, which is the correct direction and is
|
|
39
|
+
// precisely what a fixed 24MB could not do. No caller sees any difference in what it gets.
|
|
21
40
|
export const TRANSCRIPT_CACHE_MAX_BYTES = 24 * 1024 * 1024;
|
|
41
|
+
const TRANSCRIPT_CACHE_HEAP_SHARE = 0.25;
|
|
42
|
+
const TRANSCRIPT_CACHE_CEILING_BYTES = 256 * 1024 * 1024;
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Largest transcript whose parsed form this process is willing to retain.
|
|
46
|
+
*
|
|
47
|
+
* @param {number} [heapLimitBytes] test seam; defaults to this process's V8 limit
|
|
48
|
+
* @returns {number} bytes
|
|
49
|
+
*/
|
|
50
|
+
export function transcriptCacheBudgetBytes(heapLimitBytes) {
|
|
51
|
+
// "Not supplied" and "supplied but unusable" are different, and collapsing them would
|
|
52
|
+
// make a bad explicit argument silently read the real heap instead of failing safe.
|
|
53
|
+
let limit;
|
|
54
|
+
if (heapLimitBytes === undefined) {
|
|
55
|
+
try { limit = getHeapStatistics().heap_size_limit; } catch { limit = 0; }
|
|
56
|
+
} else {
|
|
57
|
+
limit = heapLimitBytes;
|
|
58
|
+
}
|
|
59
|
+
// An unreadable or nonsensical limit must not widen the budget — fall back to the
|
|
60
|
+
// constant, which is the value this cap had before it became heap-derived.
|
|
61
|
+
if (!Number.isFinite(limit) || limit <= 0) return TRANSCRIPT_CACHE_MAX_BYTES;
|
|
62
|
+
const budget = Math.floor((limit * TRANSCRIPT_CACHE_HEAP_SHARE) / TRANSCRIPT_ENTRY_HEAP_FACTOR);
|
|
63
|
+
return Math.min(Math.max(budget, 0), TRANSCRIPT_CACHE_CEILING_BYTES);
|
|
64
|
+
}
|
|
22
65
|
|
|
23
66
|
let cacheKey = '';
|
|
24
67
|
let cacheEntries = null;
|
|
@@ -31,13 +74,21 @@ let cacheEntries = null;
|
|
|
31
74
|
* must see the same fresh data it would have read for itself.
|
|
32
75
|
*
|
|
33
76
|
* @param {string|null|undefined} transcriptPath
|
|
77
|
+
* @param {object} [opts]
|
|
78
|
+
* @param {number} [opts.maxBytes] Retention budget. A TEST SEAM — production passes
|
|
79
|
+
* nothing and gets `transcriptCacheBudgetBytes()`. Exercising the over-budget branch for
|
|
80
|
+
* real would mean writing a 256MB fixture; the alternative (asserting only the default
|
|
81
|
+
* budget's value) would leave the branch that actually declines the memo untested.
|
|
82
|
+
* It is part of the cache KEY, not just the decision: two callers in one process passing
|
|
83
|
+
* different budgets must not read each other's entry.
|
|
34
84
|
* @returns {object[]} entries (empty array when the path is missing or unreadable)
|
|
35
85
|
*/
|
|
36
|
-
export function readTranscriptEntries(transcriptPath) {
|
|
86
|
+
export function readTranscriptEntries(transcriptPath, { maxBytes } = {}) {
|
|
37
87
|
if (!transcriptPath || !existsSync(transcriptPath)) return [];
|
|
38
88
|
let st;
|
|
39
89
|
try { st = statSync(transcriptPath); } catch { return []; }
|
|
40
|
-
const
|
|
90
|
+
const budget = Number.isFinite(maxBytes) ? maxBytes : transcriptCacheBudgetBytes();
|
|
91
|
+
const key = `${transcriptPath} ${st.size} ${st.mtimeMs} ${budget}`;
|
|
41
92
|
if (key === cacheKey && cacheEntries) return cacheEntries;
|
|
42
93
|
|
|
43
94
|
let raw;
|
|
@@ -47,7 +98,7 @@ export function readTranscriptEntries(transcriptPath) {
|
|
|
47
98
|
if (!line.trim()) continue;
|
|
48
99
|
try { entries.push(JSON.parse(line)); } catch { /* a partially written tail line */ }
|
|
49
100
|
}
|
|
50
|
-
if (st.size <=
|
|
101
|
+
if (st.size <= budget) {
|
|
51
102
|
cacheKey = key;
|
|
52
103
|
cacheEntries = entries;
|
|
53
104
|
} else {
|