claude-mem-lite 3.42.0 → 3.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/hook-optimize.mjs +17 -1
- package/hook-update.mjs +26 -3
- package/hook.mjs +6 -1
- package/install.mjs +16 -2
- package/lib/export-columns.mjs +26 -0
- package/lib/maintain-core.mjs +24 -0
- package/lib/search-core.mjs +65 -2
- package/lib/stats-quality.mjs +15 -0
- package/mem-cli.mjs +29 -15
- package/package.json +2 -1
- package/project-utils.mjs +13 -5
- package/registry-recommend.mjs +15 -2
- package/schema.mjs +77 -1
- package/scoring-sql.mjs +4 -0
- package/search-engine.mjs +34 -2
- package/server.mjs +76 -53
- package/source-files.mjs +34 -6
- package/tool-schemas.mjs +1 -1
- package/utils.mjs +12 -7
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"plugins": [
|
|
11
11
|
{
|
|
12
12
|
"name": "claude-mem-lite",
|
|
13
|
-
"version": "3.
|
|
13
|
+
"version": "3.44.0",
|
|
14
14
|
"source": "./",
|
|
15
15
|
"description": "Persistent long-term memory for Claude Code via MCP — captures coding decisions, bugfixes, and context across sessions. Hybrid FTS5 + TF-IDF search with episode batching. Single SQLite DB, no external services. A lighter, lower-cost alternative to claude-mem (episode batching + a smaller model; cost savings are an internal estimate, not a measured benchmark)."
|
|
16
16
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "claude-mem-lite",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.44.0",
|
|
4
4
|
"description": "Persistent long-term memory for Claude Code via MCP — captures coding decisions, bugfixes, and context across sessions. Hybrid FTS5 + TF-IDF search with episode batching. Single SQLite DB, no external services. A lighter, lower-cost alternative to claude-mem (episode batching + a smaller model; cost savings are an internal estimate, not a measured benchmark).",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "sdsrss"
|
package/hook-optimize.mjs
CHANGED
|
@@ -929,7 +929,23 @@ export async function optimizeRun(db, { tasks, maxItems = 15, force = false, ree
|
|
|
929
929
|
try {
|
|
930
930
|
switch (task) {
|
|
931
931
|
case 're-enrich':
|
|
932
|
-
|
|
932
|
+
if (reenrichScope === 'narrow') {
|
|
933
|
+
// P1-2: the default maintenance pass covers BOTH narrow (fill lesson/concepts on
|
|
934
|
+
// fully-degraded rows) AND aliases (backfill search_aliases on lesson-bearing
|
|
935
|
+
// manual saves that narrow+wide both skip — mem_save writes no aliases, so without
|
|
936
|
+
// this they stay paraphrase-unfindable). Split the budget so neither starves.
|
|
937
|
+
// An explicit --scope wide|aliases still runs exactly that one scope (below).
|
|
938
|
+
const aliasBudget = Math.max(1, Math.floor(budget.reenrich / 2));
|
|
939
|
+
const narrowRes = await executeReenrich(db, budget.reenrich - aliasBudget, { scope: 'narrow', project });
|
|
940
|
+
const aliasRes = await executeReenrich(db, aliasBudget, { scope: 'aliases', project });
|
|
941
|
+
results.reenrich = {
|
|
942
|
+
processed: (narrowRes.processed || 0) + (aliasRes.processed || 0),
|
|
943
|
+
skipped: (narrowRes.skipped || 0) + (aliasRes.skipped || 0),
|
|
944
|
+
byScope: { narrow: narrowRes, aliases: aliasRes },
|
|
945
|
+
};
|
|
946
|
+
} else {
|
|
947
|
+
results.reenrich = await executeReenrich(db, budget.reenrich, { scope: reenrichScope, project });
|
|
948
|
+
}
|
|
933
949
|
break;
|
|
934
950
|
case 'normalize':
|
|
935
951
|
results.normalize = await executeNormalize(db, force, { project });
|
package/hook-update.mjs
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// Skips in dev mode (symlinked installs). Silent on network failure.
|
|
4
4
|
|
|
5
5
|
import { execSync, execFileSync } from 'node:child_process';
|
|
6
|
-
import { readFileSync, writeFileSync, copyFileSync, cpSync, readdirSync, existsSync, lstatSync, mkdirSync, rmSync, renameSync, chmodSync } from 'node:fs';
|
|
6
|
+
import { readFileSync, writeFileSync, copyFileSync, cpSync, readdirSync, existsSync, lstatSync, mkdirSync, mkdtempSync, rmSync, renameSync, chmodSync } from 'node:fs';
|
|
7
7
|
import { join, dirname, resolve } from 'node:path';
|
|
8
8
|
import { pathToFileURL } from 'node:url';
|
|
9
9
|
import { tmpdir, homedir } from 'node:os';
|
|
@@ -305,12 +305,22 @@ async function loadReleaseManifest(sourceDir) {
|
|
|
305
305
|
}
|
|
306
306
|
}
|
|
307
307
|
|
|
308
|
+
// Create a private (0700), unpredictably-named staging dir under the system tmpdir.
|
|
309
|
+
// mkdtempSync is atomic and owner-only, closing the predictable-name TOCTOU the old
|
|
310
|
+
// `join(tmpdir(), \`...-${Date.now()}\`)` + mkdirSync(recursive) left open: in a
|
|
311
|
+
// world-writable /tmp a local user could pre-create or symlink that guessable path before
|
|
312
|
+
// we downloaded the tarball / ran validate→install into it. mkdirSync(recursive) succeeds
|
|
313
|
+
// on an existing (attacker-owned) dir; mkdtempSync fails closed unless it creates a fresh
|
|
314
|
+
// one. Mirrors the repair() path (install.mjs) which already uses mkdtempSync. (P3-4)
|
|
315
|
+
export function createUpdateTmpDir() {
|
|
316
|
+
return mkdtempSync(join(tmpdir(), 'claude-mem-lite-update-'));
|
|
317
|
+
}
|
|
318
|
+
|
|
308
319
|
// ── Download & Install ─────────────────────────────────────
|
|
309
320
|
// Direct file copy instead of running old install.mjs (avoids symlink overwrite in dev)
|
|
310
321
|
async function downloadAndInstall(tarballUrl, expectedVersion, assets = []) {
|
|
311
|
-
const tmpDir =
|
|
322
|
+
const tmpDir = createUpdateTmpDir();
|
|
312
323
|
try {
|
|
313
|
-
mkdirSync(tmpDir, { recursive: true });
|
|
314
324
|
|
|
315
325
|
// Download tarball via curl (available on all supported platforms)
|
|
316
326
|
// Validate URL to prevent command injection via crafted tarball URLs
|
|
@@ -389,6 +399,19 @@ export function validateExtractedTarball(sourceDir, expectedVersion, expectedNam
|
|
|
389
399
|
return { ok: true };
|
|
390
400
|
}
|
|
391
401
|
|
|
402
|
+
// Pure downgrade-guard predicate (exported for unit testing). True when `relVersion` is
|
|
403
|
+
// strictly OLDER than the installed `localVersion`. Under signing an attacker cannot forge a
|
|
404
|
+
// release, but CAN replay an older validly-signed one (a since-patched version) by pinning the
|
|
405
|
+
// GitHub "latest" API response — so repair(), which installs whatever fetchLatestRelease()
|
|
406
|
+
// resolves, must refuse to move BACKWARD (parity with checkForUpdate, which only installs when
|
|
407
|
+
// compareVersions(latest,current) > 0). A null/unknown local version (a broken install that
|
|
408
|
+
// genuinely needs repair, or an unreadable package.json) is allowed through — fail toward
|
|
409
|
+
// recoverability, since the signature check downstream still gates authenticity. (P3-3)
|
|
410
|
+
export function isRepairDowngrade(relVersion, localVersion) {
|
|
411
|
+
if (!relVersion || !localVersion) return false;
|
|
412
|
+
return compareVersions(relVersion, localVersion) < 0;
|
|
413
|
+
}
|
|
414
|
+
|
|
392
415
|
// ── Release signature verification (P1 supply-chain hardening) ──────────────
|
|
393
416
|
// Embedded Ed25519 PUBLIC key (SPKI PEM). ACTIVE since v3.20.0 — auto-update now
|
|
394
417
|
// FAILS CLOSED: a release missing valid signature assets is refused (the matching
|
package/hook.mjs
CHANGED
|
@@ -47,7 +47,7 @@ import { handleLLMEpisode, handleLLMSummary, saveObservation, buildImmediateObse
|
|
|
47
47
|
import { scrubRecord } from './lib/scrub-record.mjs';
|
|
48
48
|
import { formatHookError } from './lib/native-binding-hint.mjs';
|
|
49
49
|
import { selectCompressionCandidates, groupByProjectWeek, compressGroup } from './lib/compress-core.mjs';
|
|
50
|
-
import { cleanupBroken, decayAndMarkIdle, boostAccessed, selectFuzzyDedupeIds, hardDeleteCandidateCount, purgeStale, recoverOrphanedChildren, recoverBuriedLessons } from './lib/maintain-core.mjs';
|
|
50
|
+
import { cleanupBroken, decayAndMarkIdle, boostAccessed, selectFuzzyDedupeIds, hardDeleteCandidateCount, purgeStale, recoverOrphanedChildren, recoverBuriedLessons, sweepDeferredWorkOrphans } from './lib/maintain-core.mjs';
|
|
51
51
|
import { snapshotDb } from './lib/db-backup.mjs';
|
|
52
52
|
import {
|
|
53
53
|
extractCitationsFromTranscript,
|
|
@@ -851,6 +851,11 @@ function runSessionStartAutoMaintain(db) {
|
|
|
851
851
|
const lessonsHealed = recoverBuriedLessons(db, mctx);
|
|
852
852
|
if (lessonsHealed > 0) debugLog('DEBUG', 'auto-maintain', `healed ${lessonsHealed} lesson rows buried at importance 0`);
|
|
853
853
|
|
|
854
|
+
// Heal deferred_work rows whose closing obs / source prompt was deleted while FK was OFF
|
|
855
|
+
// (dangling ref foreign_key_check flags). Applies the ON DELETE SET NULL the FK would.
|
|
856
|
+
const deferredHealed = sweepDeferredWorkOrphans(db, mctx);
|
|
857
|
+
if (deferredHealed > 0) debugLog('DEBUG', 'auto-maintain', `healed ${deferredHealed} deferred-work rows with dangling references`);
|
|
858
|
+
|
|
854
859
|
const { decayed, idleMarked } = decayAndMarkIdle(db, mctx);
|
|
855
860
|
if (decayed > 0) debugLog('DEBUG', 'auto-maintain', `decayed ${decayed} stale observations`);
|
|
856
861
|
if (idleMarked > 0) debugLog('DEBUG', 'auto-maintain', `marked ${idleMarked} idle as pending-purge`);
|
package/install.mjs
CHANGED
|
@@ -1996,14 +1996,23 @@ async function repair() {
|
|
|
1996
1996
|
// repo/TLS-MITM compromise achieved RCE, bypassing the Ed25519 signed-release control that
|
|
1997
1997
|
// the manual `update` path enforces. Lazy import so a missing/broken hook-update dependency
|
|
1998
1998
|
// degrades to the manual fallback (fail-closed) rather than to unverified auto-install.
|
|
1999
|
-
let fetchLatestRelease, verifyReleaseAuthenticity;
|
|
1999
|
+
let fetchLatestRelease, verifyReleaseAuthenticity, validateExtractedTarball, isRepairDowngrade, getCurrentVersion;
|
|
2000
2000
|
try {
|
|
2001
|
-
({ fetchLatestRelease, verifyReleaseAuthenticity } = await import('./hook-update.mjs'));
|
|
2001
|
+
({ fetchLatestRelease, verifyReleaseAuthenticity, validateExtractedTarball, isRepairDowngrade, getCurrentVersion } = await import('./hook-update.mjs'));
|
|
2002
2002
|
} catch (e) {
|
|
2003
2003
|
throw new Error(`cannot load the verified-update path (${e.message}) — refusing to auto-install unverified code`, { cause: e });
|
|
2004
2004
|
}
|
|
2005
2005
|
const rel = await fetchLatestRelease();
|
|
2006
2006
|
if (!rel || !rel.tarballUrl) throw new Error('could not resolve the latest release (network / rate-limit)');
|
|
2007
|
+
// Rollback guard: refuse to repair BACKWARD onto an older validly-signed release replayed
|
|
2008
|
+
// as "latest" (the only attack signing leaves open). Skipped when the local version is
|
|
2009
|
+
// unreadable — a broken install still needs repair; the signature check below still gates
|
|
2010
|
+
// authenticity either way.
|
|
2011
|
+
let localVersion = null;
|
|
2012
|
+
try { localVersion = getCurrentVersion(); } catch { /* broken install → allow repair */ }
|
|
2013
|
+
if (isRepairDowngrade(rel.version, localVersion)) {
|
|
2014
|
+
throw new Error(`refusing to repair BACKWARD: resolved release v${rel.version} is older than installed v${localVersion} (possible signed-release rollback)`);
|
|
2015
|
+
}
|
|
2007
2016
|
// URL allow-list mirrors hook-update.downloadAndInstall — only github.com tarball URLs.
|
|
2008
2017
|
if (!/^https:\/\/(?:api\.)?github\.com\/[a-zA-Z0-9./_-]+$/.test(rel.tarballUrl)) {
|
|
2009
2018
|
throw new Error(`refusing suspicious tarball URL: ${rel.tarballUrl}`);
|
|
@@ -2015,6 +2024,11 @@ async function repair() {
|
|
|
2015
2024
|
log('Extracting...');
|
|
2016
2025
|
execFileSync('tar', ['xzf', tarballPath, '-C', stagingDir, '--strip-components=1'],
|
|
2017
2026
|
{ timeout: 30000, stdio: ['ignore', 'pipe', 'inherit'] });
|
|
2027
|
+
// Defense-in-depth on the extracted tarball (name + version === resolved tag + entry
|
|
2028
|
+
// points) BEFORE the signature check — parity with hook-update.downloadAndInstall. Catches
|
|
2029
|
+
// a wrong-version / truncated / squatter artifact whose package.json doesn't match the tag.
|
|
2030
|
+
const tarballValid = validateExtractedTarball(stagingDir, rel.version);
|
|
2031
|
+
if (!tarballValid.ok) throw new Error(`extracted tarball failed validation: ${tarballValid.reason}`);
|
|
2018
2032
|
// Verify the Ed25519 signature BEFORE running the downloaded install.mjs. Fail-closed:
|
|
2019
2033
|
// any tampering / missing-signature / fetch-failure aborts to the manual fallback.
|
|
2020
2034
|
log('Verifying release signature...');
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
// Single source of truth for the observation columns that `export` emits and `restore`
|
|
2
|
+
// reads back. Both the CLI (`cmdExport` in mem-cli.mjs) and the MCP `mem_export` tool
|
|
3
|
+
// (server.mjs) build their SELECT from this list so the two surfaces can never drift.
|
|
4
|
+
//
|
|
5
|
+
// v3.42 audit HIGH-2: the MCP export handler had its own narrower 16-column SELECT while
|
|
6
|
+
// the CLI export carried 24 — missing exactly the columns cmdRestore reads back
|
|
7
|
+
// (text / files_read / search_aliases / cited_count / uncited_streak / injection_count /
|
|
8
|
+
// decay_seen_count / last_accessed_at). A backup taken via MCP export then restored via CLI
|
|
9
|
+
// silently collapsed every empty-`narrative` row (import-jsonl / cold-start bodies live in
|
|
10
|
+
// `text`) to its bare title — unrecoverable AND unsearchable. Sharing one list closes that
|
|
11
|
+
// twin-drift class permanently.
|
|
12
|
+
//
|
|
13
|
+
// Full round-trippable set: content + value-signals (access/cited/uncited/injection/decay)
|
|
14
|
+
// + branch + timing. `id` + `memory_session_id` are informational (restore remaps id and
|
|
15
|
+
// buckets under a synthetic restore session). Session-idempotency keys (last_decided/
|
|
16
|
+
// last_cited_session_id, demoted_at, optimized_at) are intentionally NOT exported — they are
|
|
17
|
+
// meaningless after a row is re-bucketed under a restore session.
|
|
18
|
+
export const EXPORT_COLUMNS = [
|
|
19
|
+
'id', 'memory_session_id', 'project', 'type', 'title', 'subtitle', 'narrative', 'text',
|
|
20
|
+
'concepts', 'facts', 'files_read', 'files_modified', 'lesson_learned', 'search_aliases',
|
|
21
|
+
'importance', 'branch', 'access_count', 'cited_count', 'uncited_streak', 'injection_count',
|
|
22
|
+
'decay_seen_count', 'last_accessed_at', 'created_at', 'created_at_epoch',
|
|
23
|
+
];
|
|
24
|
+
|
|
25
|
+
// The SELECT column fragment (comma-joined) — drop into `SELECT ${EXPORT_COLUMNS_SQL} FROM …`.
|
|
26
|
+
export const EXPORT_COLUMNS_SQL = EXPORT_COLUMNS.join(', ');
|
package/lib/maintain-core.mjs
CHANGED
|
@@ -140,6 +140,30 @@ export function recoverBuriedLessons(db, { projectFilter = '', baseParams = [] }
|
|
|
140
140
|
`).run(...baseParams).changes;
|
|
141
141
|
}
|
|
142
142
|
|
|
143
|
+
// Heal deferred_work rows whose closing observation / source prompt was hard-deleted while
|
|
144
|
+
// foreign_keys was OFF. The warm-start fast-path deliberately runs with FK disabled (schema.mjs
|
|
145
|
+
// early migrations require cascade off), so the column's `ON DELETE SET NULL` never fired and a
|
|
146
|
+
// dangling closed_by_obs_id / source_prompt_id survives — exactly what `PRAGMA foreign_key_check`
|
|
147
|
+
// flags. This applies the SET NULL the FK would have. Closure state lives in status/
|
|
148
|
+
// closed_at_epoch, NOT the back-ref, so nulling the id does NOT reopen a done item — it only drops
|
|
149
|
+
// a pointer to a row that no longer exists. NON-DESTRUCTIVE + idempotent (a no-op once no dangling
|
|
150
|
+
// refs remain), so safe to run unconditionally alongside recoverOrphanedChildren. (P3-5)
|
|
151
|
+
export function sweepDeferredWorkOrphans(db, { projectFilter = '', baseParams = [] } = {}) {
|
|
152
|
+
const obs = db.prepare(`
|
|
153
|
+
UPDATE deferred_work SET closed_by_obs_id = NULL
|
|
154
|
+
WHERE closed_by_obs_id IS NOT NULL
|
|
155
|
+
AND NOT EXISTS (SELECT 1 FROM observations o WHERE o.id = deferred_work.closed_by_obs_id)
|
|
156
|
+
${projectFilter}
|
|
157
|
+
`).run(...baseParams).changes;
|
|
158
|
+
const prompt = db.prepare(`
|
|
159
|
+
UPDATE deferred_work SET source_prompt_id = NULL
|
|
160
|
+
WHERE source_prompt_id IS NOT NULL
|
|
161
|
+
AND NOT EXISTS (SELECT 1 FROM user_prompts p WHERE p.id = deferred_work.source_prompt_id)
|
|
162
|
+
${projectFilter}
|
|
163
|
+
`).run(...baseParams).changes;
|
|
164
|
+
return obs + prompt;
|
|
165
|
+
}
|
|
166
|
+
|
|
143
167
|
export function cleanupBroken(db, { projectFilter, baseParams, opCap = OP_CAP }) {
|
|
144
168
|
const doomed = db.prepare(`
|
|
145
169
|
SELECT id FROM observations
|
package/lib/search-core.mjs
CHANGED
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
// keys (date=created_at, text=prompt_text, session=content_session_id) so each
|
|
25
25
|
// surface's renderer reads its own field names off a single row shape.
|
|
26
26
|
|
|
27
|
-
import { sanitizeFtsQuery, relaxFtsQueryToOr, SESS_BM25, DEFAULT_DECAY_HALF_LIFE_MS } from '../utils.mjs';
|
|
27
|
+
import { sanitizeFtsQuery, relaxFtsQueryToOr, SESS_BM25, EVT_BM25, DEFAULT_DECAY_HALF_LIFE_MS } from '../utils.mjs';
|
|
28
28
|
import { cjkPrecisionOk, extractCjkLikePatterns } from '../nlp.mjs';
|
|
29
29
|
import { computeTier } from '../tier.mjs';
|
|
30
30
|
import { countSearchTotal, attachBodyTokens } from '../search-engine.mjs';
|
|
@@ -213,6 +213,35 @@ export function searchPromptsFts(db, { query, ftsQuery, project = null, epochFro
|
|
|
213
213
|
.map((r) => ({ ...r, score: 0 }));
|
|
214
214
|
}
|
|
215
215
|
|
|
216
|
+
/**
|
|
217
|
+
* Event FTS search (events table via events_fts) with recency decay + same-project boost —
|
|
218
|
+
* the mirror of searchSessionsFts for the events source. Events are the CANONICAL store for
|
|
219
|
+
* event-typed memories (persistHaikuSummary upgrade-deletes the pre-saved observations row and
|
|
220
|
+
* inserts here), so without this leg the bugfix/feature/decision history is unreachable by
|
|
221
|
+
* mem_search. Excludes superseded events (superseded_at_epoch); events has no low-signal/noise
|
|
222
|
+
* column, so no noise gate (parity with searchEvents in lib/activity.mjs). Returns raw rows
|
|
223
|
+
* { id, event_type, title, body, project, importance, file_paths, created_at_epoch, score }.
|
|
224
|
+
*/
|
|
225
|
+
export function searchEventsFts(db, { ftsQuery, project = null, projectBoost = null, epochFrom = null, epochTo = null, perSourceLimit, perSourceOffset = 0 }) {
|
|
226
|
+
const wheres = ['events_fts MATCH ?', 'e.superseded_at_epoch IS NULL'];
|
|
227
|
+
const params = [Date.now(), projectBoost, projectBoost, ftsQuery];
|
|
228
|
+
if (project) { wheres.push('e.project = ?'); params.push(project); }
|
|
229
|
+
if (epochFrom !== null) { wheres.push('e.created_at_epoch >= ?'); params.push(epochFrom); }
|
|
230
|
+
if (epochTo !== null) { wheres.push('e.created_at_epoch <= ?'); params.push(epochTo); }
|
|
231
|
+
params.push(perSourceLimit, perSourceOffset);
|
|
232
|
+
return db.prepare(`
|
|
233
|
+
SELECT e.id, e.event_type, e.title, e.body, e.project, e.importance, e.file_paths, e.created_at_epoch,
|
|
234
|
+
${EVT_BM25}
|
|
235
|
+
* (1.0 + EXP(-0.693 * MAX(0, ? - e.created_at_epoch) / ${DEFAULT_DECAY_HALF_LIFE_MS}.0))
|
|
236
|
+
* (CASE WHEN ? IS NOT NULL AND e.project = ? THEN 2.0 ELSE 1.0 END) as score
|
|
237
|
+
FROM events_fts
|
|
238
|
+
JOIN events e ON events_fts.rowid = e.id
|
|
239
|
+
WHERE ${wheres.join(' AND ')}
|
|
240
|
+
ORDER BY score
|
|
241
|
+
LIMIT ? OFFSET ?
|
|
242
|
+
`).all(...params);
|
|
243
|
+
}
|
|
244
|
+
|
|
216
245
|
/**
|
|
217
246
|
* Normalize each source's BM25 scores to [-1, 0] before cross-source merge.
|
|
218
247
|
* Prevents observations (BM25 can reach -40) from systematically outranking
|
|
@@ -221,7 +250,7 @@ export function searchPromptsFts(db, { query, ftsQuery, project = null, epochFro
|
|
|
221
250
|
* -1.0. Mutates `results` in place; callers re-sort afterwards.
|
|
222
251
|
*/
|
|
223
252
|
export function normalizeCrossSourceScores(results, sourceKey) {
|
|
224
|
-
for (const src of ['obs', 'session', 'prompt']) {
|
|
253
|
+
for (const src of ['obs', 'session', 'prompt', 'event']) {
|
|
225
254
|
const srcResults = results.filter((r) => r[sourceKey] === src && r.score !== null && r.score !== undefined);
|
|
226
255
|
if (srcResults.length < 2) continue;
|
|
227
256
|
const maxAbs = Math.max(...srcResults.map((r) => Math.abs(r.score)));
|
|
@@ -457,6 +486,40 @@ export async function coreRunSearchPipeline(ctx, opts) {
|
|
|
457
486
|
if (tolerateMissingFts) { try { pushPrompts(); } catch { /* prompt FTS may not exist in older DBs */ } } else pushPrompts();
|
|
458
487
|
}
|
|
459
488
|
|
|
489
|
+
// ── Events (FTS via shared helper; events are the CANONICAL event-typed store) ──
|
|
490
|
+
if ((!effectiveSource || effectiveSource === 'events') && !isDeep) {
|
|
491
|
+
const toIso = (epoch) => (epoch ? new Date(epoch).toISOString() : null);
|
|
492
|
+
// events has no created_at ISO column (only *_epoch) and no subtitle; body carries the
|
|
493
|
+
// distilled lesson (persistHaikuSummary writes lesson_learned||narrative here), surfaced
|
|
494
|
+
// as lesson_learned (obs parity) AND text (attachBodyTokens / CLI JSON body field).
|
|
495
|
+
const shapeEvent = (r) => ({
|
|
496
|
+
source: 'event', id: r.id, type: r.event_type, title: r.title,
|
|
497
|
+
project: r.project, importance: r.importance, files_modified: r.file_paths,
|
|
498
|
+
created_at_epoch: r.created_at_epoch, created_at: toIso(r.created_at_epoch), date: toIso(r.created_at_epoch),
|
|
499
|
+
lesson_learned: r.body, text: r.body, score: r.score ?? 0, snippet: '',
|
|
500
|
+
});
|
|
501
|
+
const pushEvents = () => {
|
|
502
|
+
if (ftsQuery) {
|
|
503
|
+
const rows = searchEventsFts(db, { ftsQuery, project, projectBoost: project ? null : currentProject, epochFrom, epochTo, perSourceLimit, perSourceOffset });
|
|
504
|
+
for (const r of rows) results.push(shapeEvent(r));
|
|
505
|
+
} else if (recentListingNoFts && effectiveSource === 'events') {
|
|
506
|
+
const params = []; const wheres = ['superseded_at_epoch IS NULL'];
|
|
507
|
+
if (project) { wheres.push('project = ?'); params.push(project); }
|
|
508
|
+
if (epochFrom !== null) { wheres.push('created_at_epoch >= ?'); params.push(epochFrom); }
|
|
509
|
+
if (epochTo !== null) { wheres.push('created_at_epoch <= ?'); params.push(epochTo); }
|
|
510
|
+
params.push(perSourceLimit, perSourceOffset);
|
|
511
|
+
const rows = db.prepare(`
|
|
512
|
+
SELECT id, event_type, title, body, project, importance, file_paths, created_at_epoch
|
|
513
|
+
FROM events WHERE ${wheres.join(' AND ')}
|
|
514
|
+
ORDER BY created_at_epoch DESC
|
|
515
|
+
LIMIT ? OFFSET ?
|
|
516
|
+
`).all(...params);
|
|
517
|
+
for (const r of rows) results.push(shapeEvent(r));
|
|
518
|
+
}
|
|
519
|
+
};
|
|
520
|
+
if (tolerateMissingFts) { try { pushEvents(); } catch { /* events_fts may not exist in older DBs */ } } else pushEvents();
|
|
521
|
+
}
|
|
522
|
+
|
|
460
523
|
// ── Type-list fallback (MCP): obs_type set + 0 matches → list recent of that type ──
|
|
461
524
|
if (obsTypeFallback && results.length === 0 && obsType) {
|
|
462
525
|
const typeWheres = ['COALESCE(compressed_into, 0) = 0', 'superseded_at IS NULL', 'type = ?'];
|
package/lib/stats-quality.mjs
CHANGED
|
@@ -7,6 +7,21 @@ import { notLowSignalTitleClause } from '../scoring-sql.mjs';
|
|
|
7
7
|
import { truncate } from '../format-utils.mjs';
|
|
8
8
|
import { COMPRESSED_PENDING_PURGE } from '../utils.mjs';
|
|
9
9
|
|
|
10
|
+
// v3.42 F7: noise/low-signal ratios must divide by the LIVE (non-compressed) observation
|
|
11
|
+
// count, NOT the all-rows total. Both numerators (lowVal, lowSignalTitle) already exclude
|
|
12
|
+
// compressed rows (`compressed_into = 0`); dividing by an all-inclusive denominator
|
|
13
|
+
// systematically UNDER-reports noise on a compress-heavy mature store (the store looks
|
|
14
|
+
// cleaner than it is right before a maintain/compress decision). Shared by the CLI `stats`
|
|
15
|
+
// and MCP `mem_stats` main gauges so the two can't drift. Returns 0 when there is no live
|
|
16
|
+
// corpus (all-compressed / empty).
|
|
17
|
+
export function computeNoiseGauge({ liveTotal, lowValCount, lowSignalCount }) {
|
|
18
|
+
const denom = liveTotal > 0 ? liveTotal : 0;
|
|
19
|
+
return {
|
|
20
|
+
noiseRatio: denom ? lowValCount / denom : 0,
|
|
21
|
+
lowSignalRatio: denom ? lowSignalCount / denom : 0,
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
|
|
10
25
|
export function computeQualityStats(db, { project, days }) {
|
|
11
26
|
const projectFilter = project ? 'AND project = ?' : '';
|
|
12
27
|
const baseParams = project ? [project] : [];
|
package/mem-cli.mjs
CHANGED
|
@@ -18,7 +18,7 @@ import { selectCompressionCandidates, groupByProjectWeek, compressGroup } from '
|
|
|
18
18
|
import { buildNotLowSignalSql } from './lib/low-signal-patterns.mjs';
|
|
19
19
|
import {
|
|
20
20
|
cleanupBroken, decayAndMarkIdle, boostAccessed, demotePinned, mergeDuplicates,
|
|
21
|
-
recoverOrphanedChildren, recoverBuriedLessons,
|
|
21
|
+
recoverOrphanedChildren, recoverBuriedLessons, sweepDeferredWorkOrphans,
|
|
22
22
|
purgeStale, purgeStalePreview, findDuplicates, maintenanceStats, rebuildVectors, vacuum,
|
|
23
23
|
recoverChildrenOf, hardDeleteCandidateCount,
|
|
24
24
|
OP_CAP, STALE_AGE_MS, PINNED_INJ_THRESHOLD,
|
|
@@ -40,6 +40,8 @@ import { readFileSync, existsSync, readdirSync } from 'fs';
|
|
|
40
40
|
import { parseArgs, out, fail, relativeTime, fmtDateShort, parseIdToken, formatProbeHints, rejectBareStringFlags, suggestUnknownFlags, OBS_TIME_FIELDS, formatObsFieldValue } from './cli/common.mjs';
|
|
41
41
|
import { saveObservation } from './lib/save-observation.mjs';
|
|
42
42
|
import { rebuildObservationDerived } from './lib/observation-write.mjs';
|
|
43
|
+
import { EXPORT_COLUMNS_SQL } from './lib/export-columns.mjs';
|
|
44
|
+
import { computeNoiseGauge } from './lib/stats-quality.mjs';
|
|
43
45
|
import { recallByFile } from './lib/recall-core.mjs';
|
|
44
46
|
import { resolveAnchorToken, formatAnchorError, resolveQueryAnchor, fetchRecentTimeline, fetchTimelineWindow } from './lib/timeline-core.mjs';
|
|
45
47
|
import { buildSearchFtsQuery, parseDateBounds, parseDuration, coreRunSearchPipeline } from './lib/search-core.mjs';
|
|
@@ -133,8 +135,8 @@ async function cmdSearch(db, args, { llm } = {}) {
|
|
|
133
135
|
process.stderr.write('[mem] Note: --rerank requires --deep (it reranks deep-search candidates); ignored\n');
|
|
134
136
|
}
|
|
135
137
|
|
|
136
|
-
if (source && !['observations', 'sessions', 'prompts'].includes(source)) {
|
|
137
|
-
fail(`[mem] Invalid --source "${source}". Use: observations, sessions, prompts`);
|
|
138
|
+
if (source && !['observations', 'sessions', 'prompts', 'events'].includes(source)) {
|
|
139
|
+
fail(`[mem] Invalid --source "${source}". Use: observations, sessions, prompts, events`);
|
|
138
140
|
return;
|
|
139
141
|
}
|
|
140
142
|
|
|
@@ -242,7 +244,7 @@ async function cmdSearch(db, args, { llm } = {}) {
|
|
|
242
244
|
|
|
243
245
|
// "N of M" total when paged < total (paired-path with server.mjs formatSearchOutput, #8198).
|
|
244
246
|
const showTime = sort === 'time';
|
|
245
|
-
const hasMixed = paged.some(r => r.source === 'session' || r.source === 'prompt');
|
|
247
|
+
const hasMixed = paged.some(r => r.source === 'session' || r.source === 'prompt' || r.source === 'event');
|
|
246
248
|
// Suppressed when --or was explicit — user already asked for OR, no "fallback" there.
|
|
247
249
|
const fallbackHint = orFallbackFired && !useOr ? ' (relaxed AND→OR)' : '';
|
|
248
250
|
|
|
@@ -288,7 +290,7 @@ async function cmdSearch(db, args, { llm } = {}) {
|
|
|
288
290
|
const countLabel = total > paged.length ? `${paged.length} of ${total}` : `${paged.length}`;
|
|
289
291
|
// Pluralize on total — "Found 1 of 44 result" reads wrong; the population (44) drives
|
|
290
292
|
// grammatical number, not the page slice (1).
|
|
291
|
-
out(`[mem] Found ${countLabel} result${total !== 1 ? 's' : ''} for "${query}"${fallbackHint}:${hasMixed ? ' (# observation, S# session, P# prompt)' : ''}`);
|
|
293
|
+
out(`[mem] Found ${countLabel} result${total !== 1 ? 's' : ''} for "${query}"${fallbackHint}:${hasMixed ? ' (# observation, S# session, P# prompt, E# event)' : ''}`);
|
|
292
294
|
// `~Nt` = est. tokens to fetch this row's full body via mem_get (attachBodyTokens, paired with
|
|
293
295
|
// MCP). Conditional so a row that skipped enrichment renders cleanly, not "~undefinedt".
|
|
294
296
|
const tok = r => (r.bodyTokens ? ` ~${r.bodyTokens}t` : '');
|
|
@@ -300,6 +302,12 @@ async function cmdSearch(db, args, { llm } = {}) {
|
|
|
300
302
|
} else if (r.source === 'prompt') {
|
|
301
303
|
const date = fmtDateShort(r.created_at);
|
|
302
304
|
out(`P#${r.id} 💬 ${date}${timeStr} ${truncate(r.prompt_text || '(empty)', 80)}${tok(r)}`);
|
|
305
|
+
} else if (r.source === 'event') {
|
|
306
|
+
const date = fmtDateShort(r.created_at);
|
|
307
|
+
out(`E#${r.id} ${typeIcon(r.type)} ${date}${timeStr} ${truncate(r.title || '(untitled)', 80)}${tok(r)}`);
|
|
308
|
+
if (r.lesson_learned) {
|
|
309
|
+
out(` -> ${truncate(r.lesson_learned, 80)}`);
|
|
310
|
+
}
|
|
303
311
|
} else {
|
|
304
312
|
const date = fmtDateShort(r.created_at);
|
|
305
313
|
const title = truncate(r.title || r.subtitle || '(untitled)', 80);
|
|
@@ -1133,7 +1141,6 @@ async function cmdStats(db, args) {
|
|
|
1133
1141
|
AND COALESCE(compressed_into, 0) = 0
|
|
1134
1142
|
AND created_at_epoch < ? ${projectFilter}
|
|
1135
1143
|
`).get(thirtyDaysAgo, ...baseParams);
|
|
1136
|
-
const noiseRatio = obsTotal.c > 0 ? lowVal.c / obsTotal.c : 0;
|
|
1137
1144
|
// Low-signal-title population: template / tool-log titles (Modified, Worked on,
|
|
1138
1145
|
// Error while working, Error:, node/npm/npx …) that the retrieval layer already
|
|
1139
1146
|
// filters out by default. The imp=1 "Low-value" metric above structurally can't
|
|
@@ -1145,7 +1152,13 @@ async function cmdStats(db, args) {
|
|
|
1145
1152
|
WHERE NOT ${buildNotLowSignalSql()}
|
|
1146
1153
|
AND COALESCE(compressed_into, 0) = 0 ${projectFilter}
|
|
1147
1154
|
`).get(...baseParams);
|
|
1148
|
-
|
|
1155
|
+
// F7: both noise numerators exclude compressed rows → divide by the LIVE count, not
|
|
1156
|
+
// obsTotal (all rows), so a compress-heavy store isn't reported cleaner than it is.
|
|
1157
|
+
// Shared with the MCP mem_stats gauge via computeNoiseGauge (lib/stats-quality.mjs).
|
|
1158
|
+
const liveTotal = db.prepare(
|
|
1159
|
+
`SELECT COUNT(*) as c FROM observations WHERE COALESCE(compressed_into, 0) = 0 ${projectFilter}`
|
|
1160
|
+
).get(...baseParams);
|
|
1161
|
+
const { noiseRatio, lowSignalRatio } = computeNoiseGauge({ liveTotal: liveTotal.c, lowValCount: lowVal.c, lowSignalCount: lowSignalTitle.c });
|
|
1149
1162
|
const compressedCount = db.prepare(
|
|
1150
1163
|
`SELECT COUNT(*) as c FROM observations WHERE compressed_into IS NOT NULL ${projectFilter}`
|
|
1151
1164
|
).get(...baseParams);
|
|
@@ -1677,15 +1690,12 @@ function cmdExport(db, args) {
|
|
|
1677
1690
|
// content + value-signals (access/cited/uncited/injection/decay) + branch + timing.
|
|
1678
1691
|
// `search_aliases` is an FTS5-indexed column (BM25 weight 5) — dropping it on
|
|
1679
1692
|
// export silently lost the LLM-generated alternate query terms on restore, so a
|
|
1680
|
-
// restored memory became unfindable by its aliases.
|
|
1681
|
-
//
|
|
1682
|
-
//
|
|
1683
|
-
//
|
|
1693
|
+
// restored memory became unfindable by its aliases. id + memory_session_id are
|
|
1694
|
+
// informational (restore remaps id and buckets under a restore session).
|
|
1695
|
+
// EXPORT_COLUMNS_SQL is the single source of truth shared with the MCP mem_export
|
|
1696
|
+
// tool (server.mjs) so the two export surfaces can never drift (v3.42 HIGH-2).
|
|
1684
1697
|
const rows = db.prepare(`
|
|
1685
|
-
SELECT
|
|
1686
|
-
files_read, files_modified, lesson_learned, search_aliases, importance, branch,
|
|
1687
|
-
access_count, cited_count, uncited_streak, injection_count, decay_seen_count,
|
|
1688
|
-
last_accessed_at, created_at, created_at_epoch
|
|
1698
|
+
SELECT ${EXPORT_COLUMNS_SQL}
|
|
1689
1699
|
FROM observations WHERE ${wheres.join(' AND ')}
|
|
1690
1700
|
ORDER BY created_at_epoch DESC LIMIT ?
|
|
1691
1701
|
`).all(...params, limit);
|
|
@@ -2040,6 +2050,10 @@ function cmdMaintain(db, args) {
|
|
|
2040
2050
|
// lesson-bearing rows only; idempotent no-op once none remain.
|
|
2041
2051
|
const lessonsHealed = recoverBuriedLessons(db, mctx);
|
|
2042
2052
|
if (lessonsHealed > 0) results.push(`Healed ${lessonsHealed} lesson rows buried at importance 0`);
|
|
2053
|
+
// Heal deferred_work rows whose closing obs / source prompt was hard-deleted while FK was
|
|
2054
|
+
// OFF (dangling ref foreign_key_check flags). Applies the ON DELETE SET NULL the FK would.
|
|
2055
|
+
const deferredHealed = sweepDeferredWorkOrphans(db, mctx);
|
|
2056
|
+
if (deferredHealed > 0) results.push(`Healed ${deferredHealed} deferred-work rows with dangling references`);
|
|
2043
2057
|
}
|
|
2044
2058
|
|
|
2045
2059
|
if (ops.includes('decay')) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "claude-mem-lite",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.44.0",
|
|
4
4
|
"description": "Persistent long-term memory for Claude Code via MCP — captures coding decisions, bugfixes, and context across sessions. Hybrid FTS5 + TF-IDF search with episode batching. Single SQLite DB, no external services. A lighter, lower-cost alternative to claude-mem (episode batching + a smaller model; cost savings are an internal estimate, not a measured benchmark).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"packageManager": "npm@10.9.2",
|
|
@@ -68,6 +68,7 @@
|
|
|
68
68
|
"lib/err-sampler.mjs",
|
|
69
69
|
"lib/hook-telemetry.mjs",
|
|
70
70
|
"lib/resolve-data-dir.mjs",
|
|
71
|
+
"lib/export-columns.mjs",
|
|
71
72
|
"lib/file-intel.mjs",
|
|
72
73
|
"lib/reread-guard.mjs",
|
|
73
74
|
"lib/metrics.mjs",
|
package/project-utils.mjs
CHANGED
|
@@ -49,11 +49,19 @@ export function resolveProject(db, name) {
|
|
|
49
49
|
).get(`%--${name}%`);
|
|
50
50
|
if (prefixed) { _cache.set(name, prefixed.project); return prefixed.project; }
|
|
51
51
|
|
|
52
|
-
// 3)
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
52
|
+
// 3) Whole-token match: the name is a complete hyphen-delimited component of the base
|
|
53
|
+
// (e.g. "graph" → "projects--code-graph-mcp", "mcp" → "…-mcp"). Steps 1/2 already cover
|
|
54
|
+
// the exact base and the base *prefix*; this adds interior/trailing whole tokens ONLY.
|
|
55
|
+
// v3.42 F3: the old `%name%` substring fallback matched mid-token ("test" inside
|
|
56
|
+
// "loop-testing") and returned the highest-COUNT unrelated project, so `--project test`
|
|
57
|
+
// silently queried the wrong project. Require a hyphen boundary: an interior token
|
|
58
|
+
// (`%-name-%`) or a trailing token (`%-name`) so "test" no longer matches "testing".
|
|
59
|
+
const token = db.prepare(
|
|
60
|
+
`SELECT project FROM observations
|
|
61
|
+
WHERE (project LIKE '%-' || ? || '-%' OR project LIKE '%-' || ?)
|
|
62
|
+
GROUP BY project ORDER BY COUNT(*) DESC LIMIT 1`
|
|
63
|
+
).get(name, name);
|
|
64
|
+
if (token) { _cache.set(name, token.project); return token.project; }
|
|
57
65
|
|
|
58
66
|
// 4) Fallback: synthesize canonical form from current directory
|
|
59
67
|
const inferred = inferProject();
|
package/registry-recommend.mjs
CHANGED
|
@@ -349,16 +349,29 @@ export function formatFunnel(s) {
|
|
|
349
349
|
if (typeof s.sessions === 'number') {
|
|
350
350
|
const mp = s.matched && s.matched.pass
|
|
351
351
|
? `${s.matched.adopt}/${s.matched.pass} (${(100 * s.matched.precision).toFixed(0)}%)` : 'n/a';
|
|
352
|
-
|
|
352
|
+
// v3.42 F5: lift = adoptGivenPass / baseRate is a ratio over `passSessions` observations.
|
|
353
|
+
// Below LIFT_MIN_PASS_SESSIONS the numerator is 1-2 sessions of noise (a single
|
|
354
|
+
// pass→adopt yields lift 7.0 at passSessions=1) — displaying it top-ranked next to a
|
|
355
|
+
// 30-session skill misleads the shadow→live flip decision. Gate the DISPLAY on a minimum
|
|
356
|
+
// observation count and report how many were suppressed (never a silent cap).
|
|
357
|
+
const finite = Object.entries(s.lift || {}).filter(([, v]) => Number.isFinite(v.lift));
|
|
358
|
+
const shown = finite.filter(([, v]) => (v.passSessions || 0) >= LIFT_MIN_PASS_SESSIONS);
|
|
359
|
+
const suppressed = finite.length - shown.length;
|
|
360
|
+
const liftRows = shown
|
|
353
361
|
.sort((a, b) => b[1].lift - a[1].lift).slice(0, 5)
|
|
354
362
|
.map(([k, v]) => `${k}(lift ${v.lift.toFixed(2)}; ${v.hitSessions}/${v.passSessions} pass→adopt vs base ${(100 * v.baseRate).toFixed(0)}%)`)
|
|
355
363
|
.join(', ') || '(none)';
|
|
364
|
+
const suppressNote = suppressed > 0 ? ` [${suppressed} hidden: passSessions<${LIFT_MIN_PASS_SESSIONS}]` : '';
|
|
356
365
|
lines.push(` sessions=${s.sessions} matched precision (in-session PASS→adopt): ${mp}`);
|
|
357
|
-
lines.push(` targeting lift (>1 = gate beats organic base rate): ${liftRows}`);
|
|
366
|
+
lines.push(` targeting lift (>1 = gate beats organic base rate, min n=${LIFT_MIN_PASS_SESSIONS}): ${liftRows}${suppressNote}`);
|
|
358
367
|
}
|
|
359
368
|
return lines.join('\n');
|
|
360
369
|
}
|
|
361
370
|
|
|
371
|
+
// Minimum passSessions before a per-skill lift is trustworthy enough to display. Below this,
|
|
372
|
+
// adoptGivenPass is computed over 1-2 sessions and the lift is noise (F5).
|
|
373
|
+
export const LIFT_MIN_PASS_SESSIONS = 3;
|
|
374
|
+
|
|
362
375
|
/** Human-readable threshold sweep for `registry recommend-stats --sweep`. */
|
|
363
376
|
export function formatSweep(grid) {
|
|
364
377
|
const lines = ['gate threshold sweep (floor × margin → pass / matched precision):'];
|
package/schema.mjs
CHANGED
|
@@ -104,7 +104,15 @@ export const CODE_DIR = join(homedir(), '.claude-mem-lite');
|
|
|
104
104
|
// previously-uncited obs (see applyCitationDecay). REAL new column, so unlike v38-v40
|
|
105
105
|
// this DOES advance LATEST_MIGRATION_COLUMN (→ observations.last_cited_session_id);
|
|
106
106
|
// existing DBs reach the ALTER because version 40 != 41 falls through the fast-path.
|
|
107
|
-
|
|
107
|
+
// v42 (events_fts self-heal): events_fts was the one FTS table outside ensureFTS's
|
|
108
|
+
// column-aware recreation (its DDL is non-standard — UNINDEXED cols + custom tokenizer +
|
|
109
|
+
// events_fts_* trigger names — so it can't use the generic ensureFTS). Adds a dedicated
|
|
110
|
+
// ensureEventsFTS run in the migration body so a future events column addition self-heals
|
|
111
|
+
// instead of leaving a stale narrow index whose (wider) triggers throw "no column" and
|
|
112
|
+
// silently drop event writes. NO new column, so LATEST_MIGRATION_COLUMN is unchanged — the
|
|
113
|
+
// forced pass alone carries it (same pattern as v35/v36/v38/v39/v40); existing DBs run it
|
|
114
|
+
// because version 41 != 42 falls through the fast-path.
|
|
115
|
+
export const CURRENT_SCHEMA_VERSION = 42;
|
|
108
116
|
|
|
109
117
|
// Sentinel column for the LATEST migration set. The fast-path uses this to
|
|
110
118
|
// self-heal half-migrated DBs — schema_version bumped but column ALTERs rolled
|
|
@@ -555,6 +563,12 @@ export function initSchema(db) {
|
|
|
555
563
|
END;
|
|
556
564
|
`);
|
|
557
565
|
|
|
566
|
+
// v42: column-aware self-heal for events_fts (the one FTS table the generic ensureFTS can't
|
|
567
|
+
// manage — UNINDEXED cols + custom tokenizer + events_fts_* triggers). The CREATE ... IF NOT
|
|
568
|
+
// EXISTS above never widens a drifted (older, narrower) events_fts; this drops+recreates it on
|
|
569
|
+
// column drift and repopulates. No-op on a healthy DB.
|
|
570
|
+
ensureEventsFTS(db);
|
|
571
|
+
|
|
558
572
|
// Observation files junction table for normalized file lookups (replaces LIKE scans on files_modified JSON)
|
|
559
573
|
db.exec(`
|
|
560
574
|
CREATE TABLE IF NOT EXISTS observation_files (
|
|
@@ -1067,3 +1081,65 @@ export function ensureFTS(db, ftsName, tableName, columns) {
|
|
|
1067
1081
|
} catch { /* non-critical — index repopulates lazily on next write */ }
|
|
1068
1082
|
}
|
|
1069
1083
|
}
|
|
1084
|
+
|
|
1085
|
+
// Column-aware self-heal for events_fts — the events table's FTS index. events_fts is NOT
|
|
1086
|
+
// managed by the generic ensureFTS() above because its DDL is non-standard: event_type and
|
|
1087
|
+
// project are UNINDEXED, it uses a custom unicode61 tokenizer with '_-' tokenchars, and its
|
|
1088
|
+
// triggers are named events_fts_* (not events_*). Routing it through ensureFTS would recreate
|
|
1089
|
+
// it WITHOUT the UNINDEXED cols / tokenizer AND install a SECOND, differently-named trigger set
|
|
1090
|
+
// (events_*) that double-writes the index. This dedicated guard mirrors ensureFTS's drift
|
|
1091
|
+
// detection while preserving the exact events_fts DDL (schema.mjs events block) — closing the
|
|
1092
|
+
// F8/P2-4 gap: events_fts was the one FTS table outside self-heal, so a future events column
|
|
1093
|
+
// addition would leave a stale narrow index whose (wider) triggers throw "no column" and
|
|
1094
|
+
// silently drop event writes. Idempotent: a no-op once the column set matches.
|
|
1095
|
+
const EVENTS_FTS_COLUMNS = ['title', 'body', 'event_type', 'project']; // full set (drift check)
|
|
1096
|
+
export function ensureEventsFTS(db) {
|
|
1097
|
+
const ftsRow = db.prepare(`SELECT 1 FROM sqlite_master WHERE type='table' AND name='events_fts'`).get();
|
|
1098
|
+
let recreated = false;
|
|
1099
|
+
if (ftsRow) {
|
|
1100
|
+
let existingCols = [];
|
|
1101
|
+
try { existingCols = db.prepare(`PRAGMA table_info(events_fts)`).all().map(c => c.name); } catch { /* unreadable → recreate */ }
|
|
1102
|
+
const drifted = existingCols.length !== EVENTS_FTS_COLUMNS.length || EVENTS_FTS_COLUMNS.some(c => !existingCols.includes(c));
|
|
1103
|
+
if (drifted) {
|
|
1104
|
+
db.exec(`DROP TRIGGER IF EXISTS events_fts_ai`);
|
|
1105
|
+
db.exec(`DROP TRIGGER IF EXISTS events_fts_ad`);
|
|
1106
|
+
db.exec(`DROP TRIGGER IF EXISTS events_fts_au`);
|
|
1107
|
+
db.exec(`DROP TABLE IF EXISTS events_fts`);
|
|
1108
|
+
recreated = true;
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
if (!ftsRow || recreated) {
|
|
1112
|
+
db.exec(`
|
|
1113
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS events_fts USING fts5(
|
|
1114
|
+
title, body, event_type UNINDEXED, project UNINDEXED,
|
|
1115
|
+
content='events', content_rowid='id',
|
|
1116
|
+
tokenize="unicode61 remove_diacritics 2 tokenchars '_-'"
|
|
1117
|
+
);
|
|
1118
|
+
`);
|
|
1119
|
+
}
|
|
1120
|
+
// Triggers reinstated from the canonical template (INSERT all 4 columns; AFTER UPDATE OF the
|
|
1121
|
+
// two INDEXED columns only, so non-indexed bumps don't thrash the index). IF NOT EXISTS so an
|
|
1122
|
+
// unchanged definition is a no-op — byte-identical to the events block in initSchema.
|
|
1123
|
+
db.exec(`
|
|
1124
|
+
CREATE TRIGGER IF NOT EXISTS events_fts_ai AFTER INSERT ON events BEGIN
|
|
1125
|
+
INSERT INTO events_fts(rowid, title, body, event_type, project)
|
|
1126
|
+
VALUES (new.id, COALESCE(new.title,''), COALESCE(new.body,''), new.event_type, new.project);
|
|
1127
|
+
END;
|
|
1128
|
+
CREATE TRIGGER IF NOT EXISTS events_fts_ad AFTER DELETE ON events BEGIN
|
|
1129
|
+
INSERT INTO events_fts(events_fts, rowid, title, body, event_type, project)
|
|
1130
|
+
VALUES ('delete', old.id, COALESCE(old.title,''), COALESCE(old.body,''), old.event_type, old.project);
|
|
1131
|
+
END;
|
|
1132
|
+
CREATE TRIGGER IF NOT EXISTS events_fts_au AFTER UPDATE OF title, body ON events BEGIN
|
|
1133
|
+
INSERT INTO events_fts(events_fts, rowid, title, body, event_type, project)
|
|
1134
|
+
VALUES ('delete', old.id, COALESCE(old.title,''), COALESCE(old.body,''), old.event_type, old.project);
|
|
1135
|
+
INSERT INTO events_fts(rowid, title, body, event_type, project)
|
|
1136
|
+
VALUES (new.id, COALESCE(new.title,''), COALESCE(new.body,''), new.event_type, new.project);
|
|
1137
|
+
END;
|
|
1138
|
+
`);
|
|
1139
|
+
if (recreated) {
|
|
1140
|
+
try {
|
|
1141
|
+
const cnt = db.prepare(`SELECT COUNT(*) AS c FROM events`).get();
|
|
1142
|
+
if (cnt.c > 0) db.exec(`INSERT INTO events_fts(events_fts) VALUES('rebuild')`);
|
|
1143
|
+
} catch { /* non-critical — index repopulates lazily on next write */ }
|
|
1144
|
+
}
|
|
1145
|
+
}
|
package/scoring-sql.mjs
CHANGED
|
@@ -51,6 +51,10 @@ export const OBS_BM25 = 'bm25(observations_fts, 10, 5, 5, 3, 3, 2, 8, 5)';
|
|
|
51
51
|
/** session_summaries_fts BM25 weights: request=5, investigated=3, learned=3, completed=3, next_steps=2, notes=1, remaining_items=1 */
|
|
52
52
|
export const SESS_BM25 = 'bm25(session_summaries_fts, 5, 3, 3, 3, 2, 1, 1)';
|
|
53
53
|
|
|
54
|
+
/** events_fts BM25 weights: title=5, body=2 (event_type/project are UNINDEXED — weight irrelevant).
|
|
55
|
+
* Title-weighted like OBS/SESS so a title hit outranks a body-only hit. */
|
|
56
|
+
export const EVT_BM25 = 'bm25(events_fts, 5, 2)';
|
|
57
|
+
|
|
54
58
|
/** FTS5 columns for observations (must match BM25 weight order) */
|
|
55
59
|
export const OBS_FTS_COLUMNS = ['title', 'subtitle', 'narrative', 'text', 'facts', 'concepts', 'lesson_learned', 'search_aliases'];
|
|
56
60
|
|
package/search-engine.mjs
CHANGED
|
@@ -35,6 +35,15 @@ const SIMPLE_SCORE = `${OBS_BM25}
|
|
|
35
35
|
* (0.5 + 0.5 * COALESCE(o.importance, 1))
|
|
36
36
|
* (1.0 + 0.3 * (o.lesson_learned IS NOT NULL AND o.lesson_learned NOT IN ('', 'none')))`;
|
|
37
37
|
|
|
38
|
+
// Shared column set for fetching an observation surfaced by the vector arm — used by BOTH
|
|
39
|
+
// the RRF-merge branch (FTS also had results) and the FTS-empty fallback branch. Single
|
|
40
|
+
// source so the two can't drift. v3.42 F4: the fallback branch's SELECT had dropped
|
|
41
|
+
// lesson_learned while its RRF twin kept it, so a vector-only hit returned
|
|
42
|
+
// lesson_learned: undefined — losing the lesson content AND the 1.5× lesson scoring boost
|
|
43
|
+
// downstream. Both branches build `{ …, date: obs.created_at, lesson_learned: obs.lesson_learned }`
|
|
44
|
+
// so the SELECT must carry created_at + lesson_learned.
|
|
45
|
+
export const VEC_HIT_OBS_COLS = 'id, type, title, subtitle, project, created_at, created_at_epoch, importance, files_modified, branch, lesson_learned';
|
|
46
|
+
|
|
38
47
|
export function buildObsFtsQuery(scoring, { multiplier, withSnippet, withOffset, includeNoise } = {}) {
|
|
39
48
|
const scoreExpr = scoring === 'full' ? FULL_SCORE : SIMPLE_SCORE;
|
|
40
49
|
const mult = multiplier ? ` * ${multiplier}` : '';
|
|
@@ -158,6 +167,24 @@ export function countPromptFtsMatches(db, { ftsQuery, project = null, epochFrom
|
|
|
158
167
|
} catch { return 0; }
|
|
159
168
|
}
|
|
160
169
|
|
|
170
|
+
export function countEventFtsMatches(db, { ftsQuery, project = null, epochFrom = null, epochTo = null }) {
|
|
171
|
+
if (!ftsQuery) return 0;
|
|
172
|
+
try {
|
|
173
|
+
const wheres = ['events_fts MATCH ?', 'e.superseded_at_epoch IS NULL'];
|
|
174
|
+
const params = [ftsQuery];
|
|
175
|
+
if (project) { wheres.push('e.project = ?'); params.push(project); }
|
|
176
|
+
if (epochFrom) { wheres.push('e.created_at_epoch >= ?'); params.push(epochFrom); }
|
|
177
|
+
if (epochTo) { wheres.push('e.created_at_epoch <= ?'); params.push(epochTo); }
|
|
178
|
+
const row = db.prepare(`
|
|
179
|
+
SELECT COUNT(*) as c
|
|
180
|
+
FROM events_fts
|
|
181
|
+
JOIN events e ON events_fts.rowid = e.id
|
|
182
|
+
WHERE ${wheres.join(' AND ')}
|
|
183
|
+
`).get(...params);
|
|
184
|
+
return row?.c ?? 0;
|
|
185
|
+
} catch { return 0; }
|
|
186
|
+
}
|
|
187
|
+
|
|
161
188
|
/**
|
|
162
189
|
* Sum true match counts across the sources that contribute to a cross-source (or
|
|
163
190
|
* source-restricted) search. `obsFtsQuery` lets callers pass the OR-relaxed query
|
|
@@ -178,6 +205,9 @@ export function countSearchTotal(db, {
|
|
|
178
205
|
if (!effectiveSource || effectiveSource === 'prompts') {
|
|
179
206
|
total += countPromptFtsMatches(db, { ftsQuery, project, epochFrom, epochTo });
|
|
180
207
|
}
|
|
208
|
+
if (!effectiveSource || effectiveSource === 'events') {
|
|
209
|
+
total += countEventFtsMatches(db, { ftsQuery, project, epochFrom, epochTo });
|
|
210
|
+
}
|
|
181
211
|
return total;
|
|
182
212
|
}
|
|
183
213
|
|
|
@@ -226,6 +256,8 @@ export function attachBodyTokens(db, results) {
|
|
|
226
256
|
parts = [r.title, r.subtitle, r.lesson_learned, row.narrative, row.facts, row.text];
|
|
227
257
|
} else if (src === 'session') {
|
|
228
258
|
parts = [r.request, r.completed, r.working_on];
|
|
259
|
+
} else if (src === 'event') {
|
|
260
|
+
parts = [r.title, r.lesson_learned]; // events carry title + body(=lesson_learned) on the row
|
|
229
261
|
} else {
|
|
230
262
|
parts = [r.text, r.prompt_text];
|
|
231
263
|
}
|
|
@@ -432,7 +464,7 @@ export function searchObservationsHybrid(db, ctx) {
|
|
|
432
464
|
const resultMap = new Map(results.map(r => [r.id, r]));
|
|
433
465
|
for (const vr of vecResults) {
|
|
434
466
|
if (!resultMap.has(vr.id)) {
|
|
435
|
-
const obs = db.prepare(
|
|
467
|
+
const obs = db.prepare(`SELECT ${VEC_HIT_OBS_COLS} FROM observations WHERE id = ?`).get(vr.id);
|
|
436
468
|
if (!obs) continue;
|
|
437
469
|
if (epochFrom !== null && obs.created_at_epoch < epochFrom) continue;
|
|
438
470
|
if (epochTo !== null && obs.created_at_epoch > epochTo) continue;
|
|
@@ -450,7 +482,7 @@ export function searchObservationsHybrid(db, ctx) {
|
|
|
450
482
|
} else {
|
|
451
483
|
// FTS5 found nothing but vector found results
|
|
452
484
|
for (const vr of vecResults) {
|
|
453
|
-
const obs = db.prepare(
|
|
485
|
+
const obs = db.prepare(`SELECT ${VEC_HIT_OBS_COLS} FROM observations WHERE id = ?`).get(vr.id);
|
|
454
486
|
if (!obs) continue;
|
|
455
487
|
if (epochFrom !== null && obs.created_at_epoch < epochFrom) continue;
|
|
456
488
|
if (epochTo !== null && obs.created_at_epoch > epochTo) continue;
|
package/server.mjs
CHANGED
|
@@ -17,7 +17,7 @@ import { resolveAnchorToken, formatAnchorError, resolveQueryAnchor, fetchRecentT
|
|
|
17
17
|
import { buildSearchFtsQuery, parseDateBounds, parseDuration, coreRunSearchPipeline } from './lib/search-core.mjs';
|
|
18
18
|
import {
|
|
19
19
|
cleanupBroken, decayAndMarkIdle, boostAccessed, demotePinned, mergeDuplicates,
|
|
20
|
-
recoverOrphanedChildren, recoverBuriedLessons,
|
|
20
|
+
recoverOrphanedChildren, recoverBuriedLessons, sweepDeferredWorkOrphans,
|
|
21
21
|
purgeStale, purgeStalePreview, findDuplicates, maintenanceStats, rebuildVectors, vacuum,
|
|
22
22
|
recoverChildrenOf, hardDeleteCandidateCount,
|
|
23
23
|
OP_CAP, STALE_AGE_MS,
|
|
@@ -44,6 +44,8 @@ import { searchResources } from './registry-retriever.mjs';
|
|
|
44
44
|
import { probeOtherSources as probeIdSources, bucketIdTokens } from './lib/id-routing.mjs';
|
|
45
45
|
import { saveObservation } from './lib/save-observation.mjs';
|
|
46
46
|
import { rebuildObservationDerived } from './lib/observation-write.mjs';
|
|
47
|
+
import { EXPORT_COLUMNS_SQL } from './lib/export-columns.mjs';
|
|
48
|
+
import { computeNoiseGauge } from './lib/stats-quality.mjs';
|
|
47
49
|
import { recallByFile } from './lib/recall-core.mjs';
|
|
48
50
|
import { AUTO_MERGE_THRESHOLD } from './lib/dedup-constants.mjs';
|
|
49
51
|
import {
|
|
@@ -216,7 +218,7 @@ function formatSearchOutput(paginatedResults, args, ftsQuery, totalCount, orFall
|
|
|
216
218
|
const countLabel = totalCount > paginatedResults.length
|
|
217
219
|
? `${paginatedResults.length} of ${totalCount}`
|
|
218
220
|
: `${paginatedResults.length}`;
|
|
219
|
-
const hasMixed = paginatedResults.some(r => r.source === 'session' || r.source === 'prompt');
|
|
221
|
+
const hasMixed = paginatedResults.some(r => r.source === 'session' || r.source === 'prompt' || r.source === 'event');
|
|
220
222
|
// P2-6: empty/omitted query falls through to a "listing recent" path — label it explicitly
|
|
221
223
|
// so callers don't mistake BM25-less results for relevance-ranked ones.
|
|
222
224
|
const qLabel = args.query ? ` for "${args.query}"` : ' (no query — listing recent)';
|
|
@@ -224,7 +226,7 @@ function formatSearchOutput(paginatedResults, args, ftsQuery, totalCount, orFall
|
|
|
224
226
|
// query actually matched only a subset of the terms. Suppressed when the caller
|
|
225
227
|
// explicitly requested OR semantics — there's no "fallback" in that path.
|
|
226
228
|
const fallbackHint = orFallbackFired && !args.or ? ' (relaxed AND→OR)' : '';
|
|
227
|
-
lines.push(`Found ${countLabel} result(s)${qLabel}${fallbackHint}:${hasMixed ? ' (# observation, S# session, P# prompt)' : ''}\n`);
|
|
229
|
+
lines.push(`Found ${countLabel} result(s)${qLabel}${fallbackHint}:${hasMixed ? ' (# observation, S# session, P# prompt, E# event)' : ''}\n`);
|
|
228
230
|
|
|
229
231
|
// `~Nt` = estimated tokens to fetch this row's full body via mem_get (attachBodyTokens).
|
|
230
232
|
// Conditional so a result that skipped enrichment renders cleanly, not "~undefinedt".
|
|
@@ -239,6 +241,8 @@ function formatSearchOutput(paginatedResults, args, ftsQuery, totalCount, orFall
|
|
|
239
241
|
lines.push(`S#${r.id} 📋 ${truncate(r.request || r.completed || '(no summary)')} | ${r.project} | ${fmtDate(r.date)}${tok(r)}`);
|
|
240
242
|
} else if (r.source === 'prompt') {
|
|
241
243
|
lines.push(`P#${r.id} 💬 ${truncate(r.text)} | ${fmtDate(r.date)}${tok(r)}`);
|
|
244
|
+
} else if (r.source === 'event') {
|
|
245
|
+
lines.push(`E#${r.id} ${typeIcon(r.type)} [${r.type}] ${truncate(r.title || '(untitled)')} | ${r.project} | ${fmtDate(r.date)}${tok(r)}`);
|
|
242
246
|
}
|
|
243
247
|
}
|
|
244
248
|
|
|
@@ -251,15 +255,14 @@ function formatSearchOutput(paginatedResults, args, ftsQuery, totalCount, orFall
|
|
|
251
255
|
// Exported for tests: runs the full mem_search pipeline against an explicit db
|
|
252
256
|
// with an optional injected llm (deepSearch dependency). The MCP tool handler
|
|
253
257
|
// calls this with the module db and the default llm.
|
|
254
|
-
//
|
|
255
|
-
//
|
|
256
|
-
// resolveProject() against the real (module) DB, not the test DB.
|
|
258
|
+
// v3.42 F3: resolveProject now runs against the injected `db` param (not the module db), so
|
|
259
|
+
// a project: arg through this seam resolves against the TEST db — real test isolation.
|
|
257
260
|
export async function handleSearchForTest(db, args, { llm, rerankLlm } = {}) {
|
|
258
261
|
return runSearchPipeline(db, args, { llm, rerankLlm });
|
|
259
262
|
}
|
|
260
263
|
|
|
261
264
|
async function runSearchPipeline(db, args, { llm, rerankLlm } = {}) {
|
|
262
|
-
if (args.project) args = { ...args, project:
|
|
265
|
+
if (args.project) args = { ...args, project: _resolveProjectShared(db, args.project) };
|
|
263
266
|
const limit = args.limit ?? 20;
|
|
264
267
|
const offset = args.offset ?? 0;
|
|
265
268
|
// args.or: force OR from the start (CLI `search --or` parity). The default path
|
|
@@ -352,15 +355,15 @@ server.registerTool(
|
|
|
352
355
|
|
|
353
356
|
// ─── Tool: mem_recent ────────────────────────────────────────────────────────
|
|
354
357
|
|
|
355
|
-
// In-process test seam (mirrors handleSearchForTest, #8743): threads an injected
|
|
356
|
-
//
|
|
357
|
-
//
|
|
358
|
+
// In-process test seam (mirrors handleSearchForTest, #8743): threads an injected db through
|
|
359
|
+
// the SAME body the registered handler runs. v3.42 F3: a `project` arg now resolves against
|
|
360
|
+
// the injected db (not the module db), so :memory: test isolation is actually achieved.
|
|
358
361
|
export async function handleRecentForTest(db, args) {
|
|
359
362
|
return runRecent(db, args);
|
|
360
363
|
}
|
|
361
364
|
|
|
362
365
|
async function runRecent(db, args) {
|
|
363
|
-
if (args.project) args = { ...args, project:
|
|
366
|
+
if (args.project) args = { ...args, project: _resolveProjectShared(db, args.project) };
|
|
364
367
|
const limit = args.limit ?? 10;
|
|
365
368
|
const project = args.project || inferProject();
|
|
366
369
|
|
|
@@ -906,7 +909,6 @@ server.registerTool(
|
|
|
906
909
|
AND created_at_epoch < ? ${projectFilter}
|
|
907
910
|
`).get(thirtyDaysAgo, ...baseParams);
|
|
908
911
|
|
|
909
|
-
const noiseRatio = obsTotal.c > 0 ? lowVal.c / obsTotal.c : 0;
|
|
910
912
|
// Low-signal-title population (template / tool-log titles the read-side filter
|
|
911
913
|
// already excludes). The imp=1 "Low-value" metric can't see these, so the
|
|
912
914
|
// gauge under-reports real noise without it. See lib/low-signal-patterns.mjs.
|
|
@@ -915,7 +917,12 @@ server.registerTool(
|
|
|
915
917
|
WHERE NOT ${buildNotLowSignalSql()}
|
|
916
918
|
AND COALESCE(compressed_into, 0) = 0 ${projectFilter}
|
|
917
919
|
`).get(...baseParams);
|
|
918
|
-
|
|
920
|
+
// F7: both noise numerators exclude compressed rows → divide by the LIVE count, not
|
|
921
|
+
// obsTotal (all rows), so a compress-heavy store isn't reported cleaner than it is.
|
|
922
|
+
const liveTotal = db.prepare(
|
|
923
|
+
`SELECT COUNT(*) as c FROM observations WHERE COALESCE(compressed_into, 0) = 0 ${projectFilter}`
|
|
924
|
+
).get(...baseParams);
|
|
925
|
+
const { noiseRatio, lowSignalRatio } = computeNoiseGauge({ liveTotal: liveTotal.c, lowValCount: lowVal.c, lowSignalCount: lowSignalTitle.c });
|
|
919
926
|
const compressedCount = db.prepare(`
|
|
920
927
|
SELECT COUNT(*) as c FROM observations WHERE compressed_into IS NOT NULL ${projectFilter}
|
|
921
928
|
`).get(...baseParams);
|
|
@@ -1153,6 +1160,10 @@ server.registerTool(
|
|
|
1153
1160
|
// lesson-bearing rows only; idempotent no-op once none remain.
|
|
1154
1161
|
const lessonsHealed = recoverBuriedLessons(db, mctx);
|
|
1155
1162
|
if (lessonsHealed > 0) results.push(`Healed ${lessonsHealed} lesson rows buried at importance 0`);
|
|
1163
|
+
// Heal deferred_work rows whose closing obs / source prompt was deleted while FK was
|
|
1164
|
+
// OFF (dangling ref foreign_key_check flags). Applies the ON DELETE SET NULL the FK would.
|
|
1165
|
+
const deferredHealed = sweepDeferredWorkOrphans(db, mctx);
|
|
1166
|
+
if (deferredHealed > 0) results.push(`Healed ${deferredHealed} deferred-work rows with dangling references`);
|
|
1156
1167
|
}
|
|
1157
1168
|
|
|
1158
1169
|
if (ops.includes('decay')) {
|
|
@@ -1622,52 +1633,64 @@ server.registerTool(
|
|
|
1622
1633
|
|
|
1623
1634
|
// ─── Tool: mem_export ────────────────────────────────────────────────────────
|
|
1624
1635
|
|
|
1636
|
+
// In-process test seam (mirrors handleRecentForTest, #8743): threads an injected db
|
|
1637
|
+
// through the SAME body the registered handler runs. NOTE: a `project` arg is still
|
|
1638
|
+
// resolved via resolveProject() against the MODULE db, not the injected one.
|
|
1639
|
+
export async function handleExportForTest(db, args) {
|
|
1640
|
+
return runExport(db, args);
|
|
1641
|
+
}
|
|
1642
|
+
|
|
1643
|
+
async function runExport(db, args) {
|
|
1644
|
+
const wheres = [];
|
|
1645
|
+
const params = [];
|
|
1646
|
+
if (!args.include_compressed) wheres.push('COALESCE(compressed_into, 0) = 0');
|
|
1647
|
+
wheres.push('superseded_at IS NULL');
|
|
1648
|
+
if (args.project) { wheres.push('project = ?'); params.push(_resolveProjectShared(db, args.project)); }
|
|
1649
|
+
if (args.type) { wheres.push('type = ?'); params.push(args.type); }
|
|
1650
|
+
// T3-P1-A: surface invalid dates instead of silently dropping the filter — mirrors
|
|
1651
|
+
// mem_search, which threw. A dropped filter can quietly expand the export blast radius.
|
|
1652
|
+
if (args.date_from) {
|
|
1653
|
+
const epoch = new Date(args.date_from).getTime();
|
|
1654
|
+
if (isNaN(epoch)) throw new Error(`Invalid date_from: "${args.date_from}" (use ISO 8601 or YYYY-MM-DD)`);
|
|
1655
|
+
wheres.push('created_at_epoch >= ?');
|
|
1656
|
+
params.push(epoch);
|
|
1657
|
+
}
|
|
1658
|
+
if (args.date_to) {
|
|
1659
|
+
const d = args.date_to.length === 10 ? args.date_to + 'T23:59:59.999Z' : args.date_to;
|
|
1660
|
+
const epoch = new Date(d).getTime();
|
|
1661
|
+
if (isNaN(epoch)) throw new Error(`Invalid date_to: "${args.date_to}" (use ISO 8601 or YYYY-MM-DD)`);
|
|
1662
|
+
wheres.push('created_at_epoch <= ?');
|
|
1663
|
+
params.push(epoch);
|
|
1664
|
+
}
|
|
1665
|
+
|
|
1666
|
+
const where = wheres.length > 0 ? 'WHERE ' + wheres.join(' AND ') : '';
|
|
1667
|
+
const exportLimit = Math.min(args.limit ?? 200, 1000);
|
|
1668
|
+
// T3-P2-B: probe limit+1 so we can tell "user hit their own limit with more waiting" from
|
|
1669
|
+
// "user got exactly what existed". Trim to exportLimit before rendering.
|
|
1670
|
+
// EXPORT_COLUMNS_SQL: shared with CLI cmdExport — the full round-trippable set restore
|
|
1671
|
+
// reads back (v3.42 HIGH-2: this handler used to carry a narrower 16-col SELECT, silently
|
|
1672
|
+
// dropping text/aliases/citation-signals on the advertised MCP backup→restore flow).
|
|
1673
|
+
const probed = db.prepare(`SELECT ${EXPORT_COLUMNS_SQL} FROM observations ${where} ORDER BY created_at_epoch DESC LIMIT ?`).all(...params, exportLimit + 1);
|
|
1674
|
+
const rows = probed.slice(0, exportLimit);
|
|
1675
|
+
const moreAvailable = probed.length > exportLimit;
|
|
1676
|
+
|
|
1677
|
+
if (rows.length === 0) return { content: [{ type: 'text', text: 'No observations found matching the criteria.' }] };
|
|
1678
|
+
|
|
1679
|
+
const output = args.format === 'jsonl'
|
|
1680
|
+
? rows.map(r => JSON.stringify(r)).join('\n')
|
|
1681
|
+
: JSON.stringify(rows, null, 2);
|
|
1682
|
+
|
|
1683
|
+
const cap = moreAvailable ? `\nNote: Results capped at ${exportLimit}. Use date_from/date_to or increase limit (max 1000) to export more.` : '';
|
|
1684
|
+
return { content: [{ type: 'text', text: `Exported ${rows.length} observations:${cap}\n${output}` }] };
|
|
1685
|
+
}
|
|
1686
|
+
|
|
1625
1687
|
server.registerTool(
|
|
1626
1688
|
'mem_export',
|
|
1627
1689
|
{
|
|
1628
1690
|
description: descriptionOf('mem_export'),
|
|
1629
1691
|
inputSchema: memExportSchema,
|
|
1630
1692
|
},
|
|
1631
|
-
safeHandler(async (args) =>
|
|
1632
|
-
const wheres = [];
|
|
1633
|
-
const params = [];
|
|
1634
|
-
if (!args.include_compressed) wheres.push('COALESCE(compressed_into, 0) = 0');
|
|
1635
|
-
wheres.push('superseded_at IS NULL');
|
|
1636
|
-
if (args.project) { wheres.push('project = ?'); params.push(resolveProject(args.project)); }
|
|
1637
|
-
if (args.type) { wheres.push('type = ?'); params.push(args.type); }
|
|
1638
|
-
// T3-P1-A: surface invalid dates instead of silently dropping the filter — mirrors
|
|
1639
|
-
// mem_search, which threw. A dropped filter can quietly expand the export blast radius.
|
|
1640
|
-
if (args.date_from) {
|
|
1641
|
-
const epoch = new Date(args.date_from).getTime();
|
|
1642
|
-
if (isNaN(epoch)) throw new Error(`Invalid date_from: "${args.date_from}" (use ISO 8601 or YYYY-MM-DD)`);
|
|
1643
|
-
wheres.push('created_at_epoch >= ?');
|
|
1644
|
-
params.push(epoch);
|
|
1645
|
-
}
|
|
1646
|
-
if (args.date_to) {
|
|
1647
|
-
const d = args.date_to.length === 10 ? args.date_to + 'T23:59:59.999Z' : args.date_to;
|
|
1648
|
-
const epoch = new Date(d).getTime();
|
|
1649
|
-
if (isNaN(epoch)) throw new Error(`Invalid date_to: "${args.date_to}" (use ISO 8601 or YYYY-MM-DD)`);
|
|
1650
|
-
wheres.push('created_at_epoch <= ?');
|
|
1651
|
-
params.push(epoch);
|
|
1652
|
-
}
|
|
1653
|
-
|
|
1654
|
-
const where = wheres.length > 0 ? 'WHERE ' + wheres.join(' AND ') : '';
|
|
1655
|
-
const exportLimit = Math.min(args.limit ?? 200, 1000);
|
|
1656
|
-
// T3-P2-B: probe limit+1 so we can tell "user hit their own limit with more waiting" from
|
|
1657
|
-
// "user got exactly what existed". Trim to exportLimit before rendering.
|
|
1658
|
-
const probed = db.prepare(`SELECT id, project, type, title, subtitle, narrative, concepts, facts, lesson_learned, importance, files_modified, branch, access_count, memory_session_id, created_at, created_at_epoch FROM observations ${where} ORDER BY created_at_epoch DESC LIMIT ?`).all(...params, exportLimit + 1);
|
|
1659
|
-
const rows = probed.slice(0, exportLimit);
|
|
1660
|
-
const moreAvailable = probed.length > exportLimit;
|
|
1661
|
-
|
|
1662
|
-
if (rows.length === 0) return { content: [{ type: 'text', text: 'No observations found matching the criteria.' }] };
|
|
1663
|
-
|
|
1664
|
-
const output = args.format === 'jsonl'
|
|
1665
|
-
? rows.map(r => JSON.stringify(r)).join('\n')
|
|
1666
|
-
: JSON.stringify(rows, null, 2);
|
|
1667
|
-
|
|
1668
|
-
const cap = moreAvailable ? `\nNote: Results capped at ${exportLimit}. Use date_from/date_to or increase limit (max 1000) to export more.` : '';
|
|
1669
|
-
return { content: [{ type: 'text', text: `Exported ${rows.length} observations:${cap}\n${output}` }] };
|
|
1670
|
-
})
|
|
1693
|
+
safeHandler(async (args) => runExport(db, args))
|
|
1671
1694
|
);
|
|
1672
1695
|
|
|
1673
1696
|
// ─── Tool: mem_recall ────────────────────────────────────────────────────────
|
package/source-files.mjs
CHANGED
|
@@ -170,6 +170,10 @@ export const SOURCE_FILES = [
|
|
|
170
170
|
// mem-cli.mjs::cmdImportJsonl; listed here so source-files-sync.test.mjs
|
|
171
171
|
// and the npm tarball ship it on every release.
|
|
172
172
|
'lib/import-jsonl.mjs',
|
|
173
|
+
// v3.42 HIGH-2: single source of truth for the export/restore round-trippable column
|
|
174
|
+
// set. Statically imported by server.mjs (mem_export) and mem-cli.mjs (cmdExport) so the
|
|
175
|
+
// two export surfaces can't drift. Must ship or auto-update breaks export on either.
|
|
176
|
+
'lib/export-columns.mjs',
|
|
173
177
|
];
|
|
174
178
|
|
|
175
179
|
/**
|
|
@@ -199,15 +203,39 @@ export const HOOK_SCRIPT_FILES = [
|
|
|
199
203
|
'hook-launcher.mjs',
|
|
200
204
|
];
|
|
201
205
|
|
|
206
|
+
// Executable scripts that are NOT direct-install hook scripts (so they don't belong in
|
|
207
|
+
// HOOK_SCRIPT_FILES, which install.mjs materializes into ~/.claude-mem-lite/scripts/) but
|
|
208
|
+
// ARE run at runtime and MUST be signed:
|
|
209
|
+
// - launch.mjs / launch-preflight.mjs: the plugin MCP server (.mcp.json runs
|
|
210
|
+
// ${CLAUDE_PLUGIN_ROOT}/scripts/launch.mjs; launch.mjs imports launch-preflight.mjs).
|
|
211
|
+
// install.mjs::dedupePluginCacheAndHooks copies BOTH from the tarball into every plugin
|
|
212
|
+
// cache version dir during repair() — an unsigned launcher let a release published
|
|
213
|
+
// without the signing key (reusing the real manifest+sig) swap launch.mjs and gain RCE
|
|
214
|
+
// as the MCP server. v3.42 audit HIGH-1.
|
|
215
|
+
// - setup.sh: run on plugin SessionStart via hooks.json. Signed for defense-in-depth
|
|
216
|
+
// (no tarball→executed-path propagation today, but it ships in files[] and is executed).
|
|
217
|
+
// These ship via package.json files[] directly, not via HOOK_SCRIPT_FILES' copy path, so
|
|
218
|
+
// listing them here changes ONLY what is signed/verified, not what install materializes.
|
|
219
|
+
// Module-internal (spread into RELEASE_SIGNED_FILES below); not exported — no external
|
|
220
|
+
// consumer, and the signing test asserts coverage via the built manifest, not this list.
|
|
221
|
+
const LAUNCHER_SCRIPT_FILES = [
|
|
222
|
+
'launch.mjs',
|
|
223
|
+
'launch-preflight.mjs',
|
|
224
|
+
'setup.sh',
|
|
225
|
+
];
|
|
226
|
+
|
|
202
227
|
// The complete set of files the release signature MUST cover: every runtime .mjs
|
|
203
228
|
// (SOURCE_FILES) PLUS the executable hook scripts (copyReleaseIntoStaging installs these
|
|
204
|
-
// into the live dir and they run on every hook fire)
|
|
205
|
-
// NOT in the signed manifest, so an attacker able to
|
|
206
|
-
// signing key — could swap a hook script (e.g.
|
|
207
|
-
// every SOURCE_FILES hash still matched, and
|
|
208
|
-
// RCE on the next hook fire.
|
|
209
|
-
//
|
|
229
|
+
// into the live dir and they run on every hook fire) PLUS the launcher/setup scripts.
|
|
230
|
+
// HOOK_SCRIPT_FILES were historically NOT in the signed manifest, so an attacker able to
|
|
231
|
+
// PUBLISH a release — but without the signing key — could swap a hook script (e.g.
|
|
232
|
+
// post-tool-use.sh / hook-launcher.mjs) while every SOURCE_FILES hash still matched, and
|
|
233
|
+
// fail-closed verification would still pass → RCE on the next hook fire. The MCP launcher
|
|
234
|
+
// (LAUNCHER_SCRIPT_FILES) was the same gap reopened for the plugin+repair path (v3.42
|
|
235
|
+
// HIGH-1). Keys are ROOT-relative, matching the extracted-tarball layout that
|
|
236
|
+
// verifyReleaseFiles hashes against.
|
|
210
237
|
export const RELEASE_SIGNED_FILES = [
|
|
211
238
|
...SOURCE_FILES,
|
|
212
239
|
...HOOK_SCRIPT_FILES.map(name => `scripts/${name}`),
|
|
240
|
+
...LAUNCHER_SCRIPT_FILES.map(name => `scripts/${name}`),
|
|
213
241
|
];
|
package/tool-schemas.mjs
CHANGED
|
@@ -80,7 +80,7 @@ const coerceMixedIdTokens = z.preprocess(
|
|
|
80
80
|
|
|
81
81
|
export const memSearchSchema = {
|
|
82
82
|
query: z.string().optional().describe('Search query (FTS5 syntax supported)'),
|
|
83
|
-
type: z.enum(['observations', 'sessions', 'prompts']).optional().describe('Limit to one table'),
|
|
83
|
+
type: z.enum(['observations', 'sessions', 'prompts', 'events']).optional().describe('Limit to one source table (default: all). events = the canonical bugfix/feature/decision/lesson history (auto-captured event-typed memories)'),
|
|
84
84
|
obs_type: OBS_TYPE_ENUM.optional().describe('Filter observation type'),
|
|
85
85
|
project: z.string().optional().describe('Filter by project name'),
|
|
86
86
|
date_from: z.string().optional().describe('Start date (ISO 8601 or YYYY-MM-DD)'),
|
package/utils.mjs
CHANGED
|
@@ -9,7 +9,7 @@ import { buildLowSignalRegex } from './lib/low-signal-patterns.mjs';
|
|
|
9
9
|
// ─── Re-exports from extracted modules ──────────────────────────────────────
|
|
10
10
|
// Backward compatibility: all consumers import from utils.mjs
|
|
11
11
|
|
|
12
|
-
export { DECAY_HALF_LIFE_BY_TYPE, DEFAULT_DECAY_HALF_LIFE_MS, OBS_BM25, SESS_BM25, TYPE_DECAY_CASE, TYPE_QUALITY_CASE, OBS_FTS_COLUMNS, notLowSignalTitleClause, noisePenaltyClause } from './scoring-sql.mjs';
|
|
12
|
+
export { DECAY_HALF_LIFE_BY_TYPE, DEFAULT_DECAY_HALF_LIFE_MS, OBS_BM25, SESS_BM25, EVT_BM25, TYPE_DECAY_CASE, TYPE_QUALITY_CASE, OBS_FTS_COLUMNS, notLowSignalTitleClause, noisePenaltyClause } from './scoring-sql.mjs';
|
|
13
13
|
export { cjkBigrams, extractCjkSynonymTokens, extractCjkKeywords, extractCjkLikePatterns, SYNONYM_MAP, expandToken, sanitizeFtsQuery, relaxFtsQueryToOr, FTS_STOP_WORDS, CJK_COMPOUNDS } from './nlp.mjs';
|
|
14
14
|
export { resolveProject, _resetProjectCache } from './project-utils.mjs';
|
|
15
15
|
export { scrubSecrets, SECRET_PATTERNS } from './secret-scrub.mjs';
|
|
@@ -323,9 +323,17 @@ export function parseJsonFromLLM(text) {
|
|
|
323
323
|
// First balanced object — survives unfenced output wrapped in brace-containing prose.
|
|
324
324
|
const balanced = firstBalancedJsonObject(text);
|
|
325
325
|
if (balanced) try { return JSON.parse(balanced); } catch {}
|
|
326
|
-
// Last-resort
|
|
327
|
-
|
|
328
|
-
|
|
326
|
+
// Last-resort span from the first `{` to the last `}` — handles a payload that isn't the
|
|
327
|
+
// FIRST balanced object. Resolved via index scan (O(n)) rather than the greedy
|
|
328
|
+
// /\{[\s\S]*\}/, which backtracks O(n²) across k unclosed opening braces (a synthetic
|
|
329
|
+
// "{{{…" with no close made the regex O(n²); LLM outputs are bounded so it wasn't
|
|
330
|
+
// exploitable — defense in depth). Behaviorally identical: the greedy match anchors on the
|
|
331
|
+
// first `{` that has any `}` after it and, being greedy, ends at the last `}`.
|
|
332
|
+
const firstBrace = text.indexOf('{');
|
|
333
|
+
const lastBrace = text.lastIndexOf('}');
|
|
334
|
+
if (firstBrace !== -1 && lastBrace > firstBrace) {
|
|
335
|
+
try { return JSON.parse(text.slice(firstBrace, lastBrace + 1)); } catch {}
|
|
336
|
+
}
|
|
329
337
|
return null;
|
|
330
338
|
}
|
|
331
339
|
|
|
@@ -456,6 +464,3 @@ export function getCurrentBranch() {
|
|
|
456
464
|
_branchCacheTime = now;
|
|
457
465
|
return _cachedBranch;
|
|
458
466
|
}
|
|
459
|
-
|
|
460
|
-
/** Reset cached branch (for testing or after git checkout) */
|
|
461
|
-
export function _resetBranchCache() { _cachedBranch = undefined; _branchCacheTime = 0; }
|