claude-mem-lite 3.43.0 → 3.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/cli/common.mjs +1 -0
- package/hook-update.mjs +26 -3
- package/hook.mjs +6 -1
- package/install.mjs +16 -2
- package/lib/id-routing.mjs +15 -10
- package/lib/maintain-core.mjs +24 -0
- package/lib/search-core.mjs +85 -6
- package/lib/timeline-core.mjs +12 -0
- package/mem-cli.mjs +68 -20
- package/package.json +1 -1
- package/schema.mjs +77 -1
- package/scoring-sql.mjs +4 -0
- package/search-engine.mjs +32 -2
- package/server.mjs +56 -17
- package/tool-schemas.mjs +9 -9
- package/utils.mjs +12 -7
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"plugins": [
|
|
11
11
|
{
|
|
12
12
|
"name": "claude-mem-lite",
|
|
13
|
-
"version": "3.
|
|
13
|
+
"version": "3.45.0",
|
|
14
14
|
"source": "./",
|
|
15
15
|
"description": "Persistent long-term memory for Claude Code via MCP — captures coding decisions, bugfixes, and context across sessions. Hybrid FTS5 + TF-IDF search with episode batching. Single SQLite DB, no external services. A lighter, lower-cost alternative to claude-mem (episode batching + a smaller model; cost savings are an internal estimate, not a measured benchmark)."
|
|
16
16
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "claude-mem-lite",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.45.0",
|
|
4
4
|
"description": "Persistent long-term memory for Claude Code via MCP — captures coding decisions, bugfixes, and context across sessions. Hybrid FTS5 + TF-IDF search with episode batching. Single SQLite DB, no external services. A lighter, lower-cost alternative to claude-mem (episode batching + a smaller model; cost savings are an internal estimate, not a measured benchmark).",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "sdsrss"
|
package/cli/common.mjs
CHANGED
|
@@ -209,5 +209,6 @@ export function formatProbeHints(probe) {
|
|
|
209
209
|
if (probe.obs.length > 0) hints.push(`#${probe.obs.join(', #')} (obs)`);
|
|
210
210
|
if (probe.session.length > 0) hints.push(`S#${probe.session.join(', S#')} (session)`);
|
|
211
211
|
if (probe.prompt.length > 0) hints.push(`P#${probe.prompt.join(', P#')} (prompt)`);
|
|
212
|
+
if (probe.event?.length > 0) hints.push(`E#${probe.event.join(', E#')} (event)`);
|
|
212
213
|
return hints;
|
|
213
214
|
}
|
package/hook-update.mjs
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// Skips in dev mode (symlinked installs). Silent on network failure.
|
|
4
4
|
|
|
5
5
|
import { execSync, execFileSync } from 'node:child_process';
|
|
6
|
-
import { readFileSync, writeFileSync, copyFileSync, cpSync, readdirSync, existsSync, lstatSync, mkdirSync, rmSync, renameSync, chmodSync } from 'node:fs';
|
|
6
|
+
import { readFileSync, writeFileSync, copyFileSync, cpSync, readdirSync, existsSync, lstatSync, mkdirSync, mkdtempSync, rmSync, renameSync, chmodSync } from 'node:fs';
|
|
7
7
|
import { join, dirname, resolve } from 'node:path';
|
|
8
8
|
import { pathToFileURL } from 'node:url';
|
|
9
9
|
import { tmpdir, homedir } from 'node:os';
|
|
@@ -305,12 +305,22 @@ async function loadReleaseManifest(sourceDir) {
|
|
|
305
305
|
}
|
|
306
306
|
}
|
|
307
307
|
|
|
308
|
+
// Create a private (0700), unpredictably-named staging dir under the system tmpdir.
|
|
309
|
+
// mkdtempSync is atomic and owner-only, closing the predictable-name TOCTOU the old
|
|
310
|
+
// `join(tmpdir(), \`...-${Date.now()}\`)` + mkdirSync(recursive) left open: in a
|
|
311
|
+
// world-writable /tmp a local user could pre-create or symlink that guessable path before
|
|
312
|
+
// we downloaded the tarball / ran validate→install into it. mkdirSync(recursive) succeeds
|
|
313
|
+
// on an existing (attacker-owned) dir; mkdtempSync fails closed unless it creates a fresh
|
|
314
|
+
// one. Mirrors the repair() path (install.mjs) which already uses mkdtempSync. (P3-4)
|
|
315
|
+
export function createUpdateTmpDir() {
|
|
316
|
+
return mkdtempSync(join(tmpdir(), 'claude-mem-lite-update-'));
|
|
317
|
+
}
|
|
318
|
+
|
|
308
319
|
// ── Download & Install ─────────────────────────────────────
|
|
309
320
|
// Direct file copy instead of running old install.mjs (avoids symlink overwrite in dev)
|
|
310
321
|
async function downloadAndInstall(tarballUrl, expectedVersion, assets = []) {
|
|
311
|
-
const tmpDir =
|
|
322
|
+
const tmpDir = createUpdateTmpDir();
|
|
312
323
|
try {
|
|
313
|
-
mkdirSync(tmpDir, { recursive: true });
|
|
314
324
|
|
|
315
325
|
// Download tarball via curl (available on all supported platforms)
|
|
316
326
|
// Validate URL to prevent command injection via crafted tarball URLs
|
|
@@ -389,6 +399,19 @@ export function validateExtractedTarball(sourceDir, expectedVersion, expectedNam
|
|
|
389
399
|
return { ok: true };
|
|
390
400
|
}
|
|
391
401
|
|
|
402
|
+
// Pure downgrade-guard predicate (exported for unit testing). True when `relVersion` is
|
|
403
|
+
// strictly OLDER than the installed `localVersion`. Under signing an attacker cannot forge a
|
|
404
|
+
// release, but CAN replay an older validly-signed one (a since-patched version) by pinning the
|
|
405
|
+
// GitHub "latest" API response — so repair(), which installs whatever fetchLatestRelease()
|
|
406
|
+
// resolves, must refuse to move BACKWARD (parity with checkForUpdate, which only installs when
|
|
407
|
+
// compareVersions(latest,current) > 0). A null/unknown local version (a broken install that
|
|
408
|
+
// genuinely needs repair, or an unreadable package.json) is allowed through — fail toward
|
|
409
|
+
// recoverability, since the signature check downstream still gates authenticity. (P3-3)
|
|
410
|
+
export function isRepairDowngrade(relVersion, localVersion) {
|
|
411
|
+
if (!relVersion || !localVersion) return false;
|
|
412
|
+
return compareVersions(relVersion, localVersion) < 0;
|
|
413
|
+
}
|
|
414
|
+
|
|
392
415
|
// ── Release signature verification (P1 supply-chain hardening) ──────────────
|
|
393
416
|
// Embedded Ed25519 PUBLIC key (SPKI PEM). ACTIVE since v3.20.0 — auto-update now
|
|
394
417
|
// FAILS CLOSED: a release missing valid signature assets is refused (the matching
|
package/hook.mjs
CHANGED
|
@@ -47,7 +47,7 @@ import { handleLLMEpisode, handleLLMSummary, saveObservation, buildImmediateObse
|
|
|
47
47
|
import { scrubRecord } from './lib/scrub-record.mjs';
|
|
48
48
|
import { formatHookError } from './lib/native-binding-hint.mjs';
|
|
49
49
|
import { selectCompressionCandidates, groupByProjectWeek, compressGroup } from './lib/compress-core.mjs';
|
|
50
|
-
import { cleanupBroken, decayAndMarkIdle, boostAccessed, selectFuzzyDedupeIds, hardDeleteCandidateCount, purgeStale, recoverOrphanedChildren, recoverBuriedLessons } from './lib/maintain-core.mjs';
|
|
50
|
+
import { cleanupBroken, decayAndMarkIdle, boostAccessed, selectFuzzyDedupeIds, hardDeleteCandidateCount, purgeStale, recoverOrphanedChildren, recoverBuriedLessons, sweepDeferredWorkOrphans } from './lib/maintain-core.mjs';
|
|
51
51
|
import { snapshotDb } from './lib/db-backup.mjs';
|
|
52
52
|
import {
|
|
53
53
|
extractCitationsFromTranscript,
|
|
@@ -851,6 +851,11 @@ function runSessionStartAutoMaintain(db) {
|
|
|
851
851
|
const lessonsHealed = recoverBuriedLessons(db, mctx);
|
|
852
852
|
if (lessonsHealed > 0) debugLog('DEBUG', 'auto-maintain', `healed ${lessonsHealed} lesson rows buried at importance 0`);
|
|
853
853
|
|
|
854
|
+
// Heal deferred_work rows whose closing obs / source prompt was deleted while FK was OFF
|
|
855
|
+
// (dangling ref foreign_key_check flags). Applies the ON DELETE SET NULL the FK would.
|
|
856
|
+
const deferredHealed = sweepDeferredWorkOrphans(db, mctx);
|
|
857
|
+
if (deferredHealed > 0) debugLog('DEBUG', 'auto-maintain', `healed ${deferredHealed} deferred-work rows with dangling references`);
|
|
858
|
+
|
|
854
859
|
const { decayed, idleMarked } = decayAndMarkIdle(db, mctx);
|
|
855
860
|
if (decayed > 0) debugLog('DEBUG', 'auto-maintain', `decayed ${decayed} stale observations`);
|
|
856
861
|
if (idleMarked > 0) debugLog('DEBUG', 'auto-maintain', `marked ${idleMarked} idle as pending-purge`);
|
package/install.mjs
CHANGED
|
@@ -1996,14 +1996,23 @@ async function repair() {
|
|
|
1996
1996
|
// repo/TLS-MITM compromise achieved RCE, bypassing the Ed25519 signed-release control that
|
|
1997
1997
|
// the manual `update` path enforces. Lazy import so a missing/broken hook-update dependency
|
|
1998
1998
|
// degrades to the manual fallback (fail-closed) rather than to unverified auto-install.
|
|
1999
|
-
let fetchLatestRelease, verifyReleaseAuthenticity;
|
|
1999
|
+
let fetchLatestRelease, verifyReleaseAuthenticity, validateExtractedTarball, isRepairDowngrade, getCurrentVersion;
|
|
2000
2000
|
try {
|
|
2001
|
-
({ fetchLatestRelease, verifyReleaseAuthenticity } = await import('./hook-update.mjs'));
|
|
2001
|
+
({ fetchLatestRelease, verifyReleaseAuthenticity, validateExtractedTarball, isRepairDowngrade, getCurrentVersion } = await import('./hook-update.mjs'));
|
|
2002
2002
|
} catch (e) {
|
|
2003
2003
|
throw new Error(`cannot load the verified-update path (${e.message}) — refusing to auto-install unverified code`, { cause: e });
|
|
2004
2004
|
}
|
|
2005
2005
|
const rel = await fetchLatestRelease();
|
|
2006
2006
|
if (!rel || !rel.tarballUrl) throw new Error('could not resolve the latest release (network / rate-limit)');
|
|
2007
|
+
// Rollback guard: refuse to repair BACKWARD onto an older validly-signed release replayed
|
|
2008
|
+
// as "latest" (the only attack signing leaves open). Skipped when the local version is
|
|
2009
|
+
// unreadable — a broken install still needs repair; the signature check below still gates
|
|
2010
|
+
// authenticity either way.
|
|
2011
|
+
let localVersion = null;
|
|
2012
|
+
try { localVersion = getCurrentVersion(); } catch { /* broken install → allow repair */ }
|
|
2013
|
+
if (isRepairDowngrade(rel.version, localVersion)) {
|
|
2014
|
+
throw new Error(`refusing to repair BACKWARD: resolved release v${rel.version} is older than installed v${localVersion} (possible signed-release rollback)`);
|
|
2015
|
+
}
|
|
2007
2016
|
// URL allow-list mirrors hook-update.downloadAndInstall — only github.com tarball URLs.
|
|
2008
2017
|
if (!/^https:\/\/(?:api\.)?github\.com\/[a-zA-Z0-9./_-]+$/.test(rel.tarballUrl)) {
|
|
2009
2018
|
throw new Error(`refusing suspicious tarball URL: ${rel.tarballUrl}`);
|
|
@@ -2015,6 +2024,11 @@ async function repair() {
|
|
|
2015
2024
|
log('Extracting...');
|
|
2016
2025
|
execFileSync('tar', ['xzf', tarballPath, '-C', stagingDir, '--strip-components=1'],
|
|
2017
2026
|
{ timeout: 30000, stdio: ['ignore', 'pipe', 'inherit'] });
|
|
2027
|
+
// Defense-in-depth on the extracted tarball (name + version === resolved tag + entry
|
|
2028
|
+
// points) BEFORE the signature check — parity with hook-update.downloadAndInstall. Catches
|
|
2029
|
+
// a wrong-version / truncated / squatter artifact whose package.json doesn't match the tag.
|
|
2030
|
+
const tarballValid = validateExtractedTarball(stagingDir, rel.version);
|
|
2031
|
+
if (!tarballValid.ok) throw new Error(`extracted tarball failed validation: ${tarballValid.reason}`);
|
|
2018
2032
|
// Verify the Ed25519 signature BEFORE running the downloaded install.mjs. Fail-closed:
|
|
2019
2033
|
// any tampering / missing-signature / fetch-failure aborts to the manual fallback.
|
|
2020
2034
|
log('Verifying release signature...');
|
package/lib/id-routing.mjs
CHANGED
|
@@ -10,18 +10,19 @@
|
|
|
10
10
|
|
|
11
11
|
/**
|
|
12
12
|
* Parse an ID token as it appears in search output or CLI positional args.
|
|
13
|
-
* Accepts: `123`, `#123`, `P#123` / `p123` (prompt), `S#123` / `s123` (session)
|
|
13
|
+
* Accepts: `123`, `#123`, `P#123` / `p123` (prompt), `S#123` / `s123` (session),
|
|
14
|
+
* `E#123` / `e123` (event — the canonical event-typed store surfaced as `E#N` by mem_search).
|
|
14
15
|
* @param {unknown} raw
|
|
15
|
-
* @returns {{ source: 'obs'|'session'|'prompt'|null, id: number } | null}
|
|
16
|
+
* @returns {{ source: 'obs'|'session'|'prompt'|'event'|null, id: number } | null}
|
|
16
17
|
* source===null means no explicit prefix — caller picks default (typically 'obs').
|
|
17
18
|
*/
|
|
18
19
|
export function parseIdToken(raw) {
|
|
19
|
-
const m = /^([
|
|
20
|
+
const m = /^([EePpSs]?)#?(\d+)$/.exec(String(raw).trim());
|
|
20
21
|
if (!m) return null;
|
|
21
22
|
const p = m[1].toUpperCase();
|
|
22
23
|
const id = parseInt(m[2], 10);
|
|
23
24
|
if (!Number.isFinite(id) || id <= 0) return null;
|
|
24
|
-
const source = p === 'P' ? 'prompt' : p === 'S' ? 'session' : null;
|
|
25
|
+
const source = p === 'P' ? 'prompt' : p === 'S' ? 'session' : p === 'E' ? 'event' : null;
|
|
25
26
|
return { source, id };
|
|
26
27
|
}
|
|
27
28
|
|
|
@@ -35,11 +36,11 @@ export function parseIdToken(raw) {
|
|
|
35
36
|
* per-token prefixes. Un-prefixed tokens fall back to `defaultSource`.
|
|
36
37
|
*
|
|
37
38
|
* @param {Array<string|number>} tokens Mixed input — order preserved within each bucket.
|
|
38
|
-
* @param {{explicit?: 'obs'|'session'|'prompt'|null, defaultSource?: 'obs'|'session'|'prompt'}} opts
|
|
39
|
-
* @returns {{bySrc: {obs:number[], session:number[], prompt:number[]}, invalid: string[]}}
|
|
39
|
+
* @param {{explicit?: 'obs'|'session'|'prompt'|'event'|null, defaultSource?: 'obs'|'session'|'prompt'|'event'}} opts
|
|
40
|
+
* @returns {{bySrc: {obs:number[], session:number[], prompt:number[], event:number[]}, invalid: string[]}}
|
|
40
41
|
*/
|
|
41
42
|
export function bucketIdTokens(tokens, { explicit = null, defaultSource = 'obs' } = {}) {
|
|
42
|
-
const bySrc = { obs: [], session: [], prompt: [] };
|
|
43
|
+
const bySrc = { obs: [], session: [], prompt: [], event: [] };
|
|
43
44
|
const invalid = [];
|
|
44
45
|
for (const raw of tokens) {
|
|
45
46
|
if (typeof raw === 'number' && Number.isFinite(raw) && raw > 0) {
|
|
@@ -60,11 +61,11 @@ export function bucketIdTokens(tokens, { explicit = null, defaultSource = 'obs'
|
|
|
60
61
|
*
|
|
61
62
|
* @param {import('better-sqlite3').Database} db
|
|
62
63
|
* @param {number[]} ids Numeric IDs to probe (non-negative ints).
|
|
63
|
-
* @param {Set<'obs'|'session'|'prompt'>} excludeSrcs Sources to skip.
|
|
64
|
-
* @returns {{obs:number[], session:number[], prompt:number[]}}
|
|
64
|
+
* @param {Set<'obs'|'session'|'prompt'|'event'>} excludeSrcs Sources to skip.
|
|
65
|
+
* @returns {{obs:number[], session:number[], prompt:number[], event:number[]}}
|
|
65
66
|
*/
|
|
66
67
|
export function probeOtherSources(db, ids, excludeSrcs) {
|
|
67
|
-
const result = { obs: [], session: [], prompt: [] };
|
|
68
|
+
const result = { obs: [], session: [], prompt: [], event: [] };
|
|
68
69
|
if (!ids || ids.length === 0) return result;
|
|
69
70
|
const placeholders = ids.map(() => '?').join(',');
|
|
70
71
|
try {
|
|
@@ -80,6 +81,10 @@ export function probeOtherSources(db, ids, excludeSrcs) {
|
|
|
80
81
|
const hits = db.prepare(`SELECT id FROM user_prompts WHERE id IN (${placeholders})`).all(...ids);
|
|
81
82
|
result.prompt = hits.map(r => r.id);
|
|
82
83
|
}
|
|
84
|
+
if (!excludeSrcs.has('event')) {
|
|
85
|
+
const hits = db.prepare(`SELECT id FROM events WHERE id IN (${placeholders})`).all(...ids);
|
|
86
|
+
result.event = hits.map(r => r.id);
|
|
87
|
+
}
|
|
83
88
|
} catch { /* best-effort hint; never block the caller */ }
|
|
84
89
|
return result;
|
|
85
90
|
}
|
package/lib/maintain-core.mjs
CHANGED
|
@@ -140,6 +140,30 @@ export function recoverBuriedLessons(db, { projectFilter = '', baseParams = [] }
|
|
|
140
140
|
`).run(...baseParams).changes;
|
|
141
141
|
}
|
|
142
142
|
|
|
143
|
+
// Heal deferred_work rows whose closing observation / source prompt was hard-deleted while
|
|
144
|
+
// foreign_keys was OFF. The warm-start fast-path deliberately runs with FK disabled (schema.mjs
|
|
145
|
+
// early migrations require cascade off), so the column's `ON DELETE SET NULL` never fired and a
|
|
146
|
+
// dangling closed_by_obs_id / source_prompt_id survives — exactly what `PRAGMA foreign_key_check`
|
|
147
|
+
// flags. This applies the SET NULL the FK would have. Closure state lives in status/
|
|
148
|
+
// closed_at_epoch, NOT the back-ref, so nulling the id does NOT reopen a done item — it only drops
|
|
149
|
+
// a pointer to a row that no longer exists. NON-DESTRUCTIVE + idempotent (a no-op once no dangling
|
|
150
|
+
// refs remain), so safe to run unconditionally alongside recoverOrphanedChildren. (P3-5)
|
|
151
|
+
export function sweepDeferredWorkOrphans(db, { projectFilter = '', baseParams = [] } = {}) {
|
|
152
|
+
const obs = db.prepare(`
|
|
153
|
+
UPDATE deferred_work SET closed_by_obs_id = NULL
|
|
154
|
+
WHERE closed_by_obs_id IS NOT NULL
|
|
155
|
+
AND NOT EXISTS (SELECT 1 FROM observations o WHERE o.id = deferred_work.closed_by_obs_id)
|
|
156
|
+
${projectFilter}
|
|
157
|
+
`).run(...baseParams).changes;
|
|
158
|
+
const prompt = db.prepare(`
|
|
159
|
+
UPDATE deferred_work SET source_prompt_id = NULL
|
|
160
|
+
WHERE source_prompt_id IS NOT NULL
|
|
161
|
+
AND NOT EXISTS (SELECT 1 FROM user_prompts p WHERE p.id = deferred_work.source_prompt_id)
|
|
162
|
+
${projectFilter}
|
|
163
|
+
`).run(...baseParams).changes;
|
|
164
|
+
return obs + prompt;
|
|
165
|
+
}
|
|
166
|
+
|
|
143
167
|
export function cleanupBroken(db, { projectFilter, baseParams, opCap = OP_CAP }) {
|
|
144
168
|
const doomed = db.prepare(`
|
|
145
169
|
SELECT id FROM observations
|
package/lib/search-core.mjs
CHANGED
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
// keys (date=created_at, text=prompt_text, session=content_session_id) so each
|
|
25
25
|
// surface's renderer reads its own field names off a single row shape.
|
|
26
26
|
|
|
27
|
-
import { sanitizeFtsQuery, relaxFtsQueryToOr, SESS_BM25, DEFAULT_DECAY_HALF_LIFE_MS } from '../utils.mjs';
|
|
27
|
+
import { sanitizeFtsQuery, relaxFtsQueryToOr, SESS_BM25, EVT_BM25, DEFAULT_DECAY_HALF_LIFE_MS } from '../utils.mjs';
|
|
28
28
|
import { cjkPrecisionOk, extractCjkLikePatterns } from '../nlp.mjs';
|
|
29
29
|
import { computeTier } from '../tier.mjs';
|
|
30
30
|
import { countSearchTotal, attachBodyTokens } from '../search-engine.mjs';
|
|
@@ -213,6 +213,38 @@ export function searchPromptsFts(db, { query, ftsQuery, project = null, epochFro
|
|
|
213
213
|
.map((r) => ({ ...r, score: 0 }));
|
|
214
214
|
}
|
|
215
215
|
|
|
216
|
+
/**
|
|
217
|
+
* Event FTS search (events table via events_fts) with recency decay + same-project boost —
|
|
218
|
+
* the mirror of searchSessionsFts for the events source. Events are the CANONICAL store for
|
|
219
|
+
* event-typed memories (persistHaikuSummary upgrade-deletes the pre-saved observations row and
|
|
220
|
+
* inserts here), so without this leg the bugfix/feature/decision history is unreachable by
|
|
221
|
+
* mem_search. Excludes superseded events (superseded_at_epoch); events has no low-signal/noise
|
|
222
|
+
* column, so no noise gate (parity with searchEvents in lib/activity.mjs). Returns raw rows
|
|
223
|
+
* { id, event_type, title, body, project, importance, file_paths, created_at_epoch, score }.
|
|
224
|
+
*/
|
|
225
|
+
export function searchEventsFts(db, { ftsQuery, project = null, projectBoost = null, epochFrom = null, epochTo = null, eventType = null, importance = null, perSourceLimit, perSourceOffset = 0 }) {
|
|
226
|
+
const wheres = ['events_fts MATCH ?', 'e.superseded_at_epoch IS NULL'];
|
|
227
|
+
const params = [Date.now(), projectBoost, projectBoost, ftsQuery];
|
|
228
|
+
if (project) { wheres.push('e.project = ?'); params.push(project); }
|
|
229
|
+
if (epochFrom !== null) { wheres.push('e.created_at_epoch >= ?'); params.push(epochFrom); }
|
|
230
|
+
if (epochTo !== null) { wheres.push('e.created_at_epoch <= ?'); params.push(epochTo); }
|
|
231
|
+
// D#76: event_type filter (obs_type maps here) + importance floor (events carry importance).
|
|
232
|
+
if (eventType) { wheres.push('e.event_type = ?'); params.push(eventType); }
|
|
233
|
+
if (importance) { wheres.push('COALESCE(e.importance, 1) >= ?'); params.push(importance); }
|
|
234
|
+
params.push(perSourceLimit, perSourceOffset);
|
|
235
|
+
return db.prepare(`
|
|
236
|
+
SELECT e.id, e.event_type, e.title, e.body, e.project, e.importance, e.file_paths, e.created_at_epoch,
|
|
237
|
+
${EVT_BM25}
|
|
238
|
+
* (1.0 + EXP(-0.693 * MAX(0, ? - e.created_at_epoch) / ${DEFAULT_DECAY_HALF_LIFE_MS}.0))
|
|
239
|
+
* (CASE WHEN ? IS NOT NULL AND e.project = ? THEN 2.0 ELSE 1.0 END) as score
|
|
240
|
+
FROM events_fts
|
|
241
|
+
JOIN events e ON events_fts.rowid = e.id
|
|
242
|
+
WHERE ${wheres.join(' AND ')}
|
|
243
|
+
ORDER BY score
|
|
244
|
+
LIMIT ? OFFSET ?
|
|
245
|
+
`).all(...params);
|
|
246
|
+
}
|
|
247
|
+
|
|
216
248
|
/**
|
|
217
249
|
* Normalize each source's BM25 scores to [-1, 0] before cross-source merge.
|
|
218
250
|
* Prevents observations (BM25 can reach -40) from systematically outranking
|
|
@@ -221,7 +253,7 @@ export function searchPromptsFts(db, { query, ftsQuery, project = null, epochFro
|
|
|
221
253
|
* -1.0. Mutates `results` in place; callers re-sort afterwards.
|
|
222
254
|
*/
|
|
223
255
|
export function normalizeCrossSourceScores(results, sourceKey) {
|
|
224
|
-
for (const src of ['obs', 'session', 'prompt']) {
|
|
256
|
+
for (const src of ['obs', 'session', 'prompt', 'event']) {
|
|
225
257
|
const srcResults = results.filter((r) => r[sourceKey] === src && r.score !== null && r.score !== undefined);
|
|
226
258
|
if (srcResults.length < 2) continue;
|
|
227
259
|
const maxAbs = Math.max(...srcResults.map((r) => Math.abs(r.score)));
|
|
@@ -292,7 +324,7 @@ export function applyTierFilter(db, results, { tier, sourceKey, currentProject }
|
|
|
292
324
|
export function finalizeSearchPage(db, results, {
|
|
293
325
|
isDeep, offset, limit, effectiveSource, ftsQuery, orFallbackFired,
|
|
294
326
|
project = null, obsType = null, importance = null, branch = null,
|
|
295
|
-
epochFrom = null, epochTo = null, includeNoise = false,
|
|
327
|
+
epochFrom = null, epochTo = null, includeNoise = false, obsTypeScoped = false,
|
|
296
328
|
}) {
|
|
297
329
|
const total = isDeep
|
|
298
330
|
? results.length
|
|
@@ -303,6 +335,7 @@ export function finalizeSearchPage(db, results, {
|
|
|
303
335
|
args: { project: project || null, obs_type: obsType || null, importance: importance || null, branch: branch || null },
|
|
304
336
|
project: project || null,
|
|
305
337
|
epochFrom, epochTo, includeNoise,
|
|
338
|
+
obsTypeScoped, // D#76: count obs + type-filtered events only (skip type-less sessions/prompts)
|
|
306
339
|
}), results.length);
|
|
307
340
|
const page = results.slice(offset, offset + limit);
|
|
308
341
|
attachBodyTokens(db, page);
|
|
@@ -342,6 +375,9 @@ export async function coreRunSearchPipeline(ctx, opts) {
|
|
|
342
375
|
limit, offset, project = null, obsType = null, importance = null, branch = null,
|
|
343
376
|
includeNoise = false, epochFrom = null, epochTo = null, sort = 'relevance', tier = null,
|
|
344
377
|
// ── surface policy (strict behavior-preservation; the two surfaces differ) ──
|
|
378
|
+
obsTypeScoped = false, // D#76: obs_type given (no branch/tier) ⇒ scope to obs+events (both carry the type
|
|
379
|
+
// vocabulary), NOT observations-only. Skips the type-less sessions/prompts legs
|
|
380
|
+
// and their counts; the events leg filters by event_type = obsType. Both surfaces.
|
|
345
381
|
obsTypeFallback = false, // A5: list-recent-by-type when 0 matches — MCP true, CLI false (#8217 removed it from CLI)
|
|
346
382
|
crossSourceEpochSortNoFts = false, // A3: epoch-sort the cross-source set when no ftsQuery — MCP true, CLI false
|
|
347
383
|
rerankPolicy = 'mcp', // A4: re-rank/supersede gate + re-sort condition — 'mcp' | 'cli'
|
|
@@ -406,7 +442,11 @@ export async function coreRunSearchPipeline(ctx, opts) {
|
|
|
406
442
|
}
|
|
407
443
|
|
|
408
444
|
// ── Sessions (FTS via shared helper; optional recent-listing when no ftsQuery) ──
|
|
409
|
-
|
|
445
|
+
// D#74: `!isDeep || escalated` — auto-escalation replaces the obs leg with a deep fuse but must
|
|
446
|
+
// NOT silently drop the cross-source complement (it did: escalated → isDeep → every leg below
|
|
447
|
+
// skipped). Explicit deep=true (escalated stays false) still dives obs-only. D#76: obsTypeScoped
|
|
448
|
+
// skips the type-less sessions/prompts legs (they can't honor an obs_type/event_type filter).
|
|
449
|
+
if ((!effectiveSource || effectiveSource === 'sessions') && (!isDeep || escalated) && !obsTypeScoped) {
|
|
410
450
|
const pushSessions = () => {
|
|
411
451
|
if (ftsQuery) {
|
|
412
452
|
const rows = searchSessionsFts(db, { ftsQuery, project, projectBoost: project ? null : currentProject, epochFrom, epochTo, perSourceLimit, perSourceOffset });
|
|
@@ -431,7 +471,7 @@ export async function coreRunSearchPipeline(ctx, opts) {
|
|
|
431
471
|
}
|
|
432
472
|
|
|
433
473
|
// ── Prompts (FTS via shared helper incl. CJK gate; optional recent-listing) ──
|
|
434
|
-
if ((!effectiveSource || effectiveSource === 'prompts') && !isDeep) {
|
|
474
|
+
if ((!effectiveSource || effectiveSource === 'prompts') && (!isDeep || escalated) && !obsTypeScoped) {
|
|
435
475
|
const pushPrompts = () => {
|
|
436
476
|
if (ftsQuery) {
|
|
437
477
|
const rows = searchPromptsFts(db, { query, ftsQuery, project, epochFrom, epochTo, perSourceLimit, perSourceOffset });
|
|
@@ -457,6 +497,45 @@ export async function coreRunSearchPipeline(ctx, opts) {
|
|
|
457
497
|
if (tolerateMissingFts) { try { pushPrompts(); } catch { /* prompt FTS may not exist in older DBs */ } } else pushPrompts();
|
|
458
498
|
}
|
|
459
499
|
|
|
500
|
+
// ── Events (FTS via shared helper; events are the CANONICAL event-typed store) ──
|
|
501
|
+
// Runs on auto-escalation too (D#74) and participates in obsTypeScoped (D#76) — events carry the
|
|
502
|
+
// same type vocabulary as observations, so an obs_type filter maps to e.event_type here.
|
|
503
|
+
if ((!effectiveSource || effectiveSource === 'events') && (!isDeep || escalated)) {
|
|
504
|
+
const toIso = (epoch) => (epoch ? new Date(epoch).toISOString() : null);
|
|
505
|
+
// events has no created_at ISO column (only *_epoch) and no subtitle; body carries the
|
|
506
|
+
// distilled lesson (persistHaikuSummary writes lesson_learned||narrative here), surfaced
|
|
507
|
+
// as lesson_learned (obs parity) AND text (attachBodyTokens / CLI JSON body field).
|
|
508
|
+
const shapeEvent = (r) => ({
|
|
509
|
+
source: 'event', id: r.id, type: r.event_type, title: r.title,
|
|
510
|
+
project: r.project, importance: r.importance, files_modified: r.file_paths,
|
|
511
|
+
created_at_epoch: r.created_at_epoch, created_at: toIso(r.created_at_epoch), date: toIso(r.created_at_epoch),
|
|
512
|
+
lesson_learned: r.body, text: r.body, score: r.score ?? 0, snippet: '',
|
|
513
|
+
});
|
|
514
|
+
const pushEvents = () => {
|
|
515
|
+
if (ftsQuery) {
|
|
516
|
+
// D#76: obsType maps to e.event_type (shared vocab); importance filters events too (they carry it).
|
|
517
|
+
const rows = searchEventsFts(db, { ftsQuery, project, projectBoost: project ? null : currentProject, epochFrom, epochTo, eventType: obsType, importance, perSourceLimit, perSourceOffset });
|
|
518
|
+
for (const r of rows) results.push(shapeEvent(r));
|
|
519
|
+
} else if (recentListingNoFts && effectiveSource === 'events') {
|
|
520
|
+
const params = []; const wheres = ['superseded_at_epoch IS NULL'];
|
|
521
|
+
if (project) { wheres.push('project = ?'); params.push(project); }
|
|
522
|
+
if (epochFrom !== null) { wheres.push('created_at_epoch >= ?'); params.push(epochFrom); }
|
|
523
|
+
if (epochTo !== null) { wheres.push('created_at_epoch <= ?'); params.push(epochTo); }
|
|
524
|
+
if (obsType) { wheres.push('event_type = ?'); params.push(obsType); }
|
|
525
|
+
if (importance) { wheres.push('COALESCE(importance, 1) >= ?'); params.push(importance); }
|
|
526
|
+
params.push(perSourceLimit, perSourceOffset);
|
|
527
|
+
const rows = db.prepare(`
|
|
528
|
+
SELECT id, event_type, title, body, project, importance, file_paths, created_at_epoch
|
|
529
|
+
FROM events WHERE ${wheres.join(' AND ')}
|
|
530
|
+
ORDER BY created_at_epoch DESC
|
|
531
|
+
LIMIT ? OFFSET ?
|
|
532
|
+
`).all(...params);
|
|
533
|
+
for (const r of rows) results.push(shapeEvent(r));
|
|
534
|
+
}
|
|
535
|
+
};
|
|
536
|
+
if (tolerateMissingFts) { try { pushEvents(); } catch { /* events_fts may not exist in older DBs */ } } else pushEvents();
|
|
537
|
+
}
|
|
538
|
+
|
|
460
539
|
// ── Type-list fallback (MCP): obs_type set + 0 matches → list recent of that type ──
|
|
461
540
|
if (obsTypeFallback && results.length === 0 && obsType) {
|
|
462
541
|
const typeWheres = ['COALESCE(compressed_into, 0) = 0', 'superseded_at IS NULL', 'type = ?'];
|
|
@@ -518,7 +597,7 @@ export async function coreRunSearchPipeline(ctx, opts) {
|
|
|
518
597
|
const preFinalizeCount = results.length;
|
|
519
598
|
const { total, page } = finalizeSearchPage(db, results, {
|
|
520
599
|
isDeep, offset, limit, effectiveSource, ftsQuery, orFallbackFired,
|
|
521
|
-
project, obsType, importance, branch, epochFrom, epochTo, includeNoise,
|
|
600
|
+
project, obsType, importance, branch, epochFrom, epochTo, includeNoise, obsTypeScoped,
|
|
522
601
|
});
|
|
523
602
|
|
|
524
603
|
return {
|
package/lib/timeline-core.mjs
CHANGED
|
@@ -53,6 +53,18 @@ export function resolveAnchorToken(db, rawAnchor, { project = null } = {}) {
|
|
|
53
53
|
return { ok: true, anchorId: nearest.id, anchorNote: `(anchored to #${nearest.id}, closest obs to ${srcPrefix}${parsed.id})` };
|
|
54
54
|
}
|
|
55
55
|
|
|
56
|
+
// Events are the canonical event-typed store (E#N in mem_search output). They have no
|
|
57
|
+
// place in the obs-centric before/after window, so — like prompt/session — anchor to the
|
|
58
|
+
// nearest observation by epoch. Without this explicit branch an E#N token would fall through
|
|
59
|
+
// to the bare-int obs lookup below and silently resolve to the COLLIDING observation id.
|
|
60
|
+
if (parsed.source === 'event') {
|
|
61
|
+
const row = db.prepare('SELECT created_at_epoch FROM events WHERE id = ?').get(parsed.id);
|
|
62
|
+
if (!row) return { ok: false, error: { code: 'source-not-found', name: 'Event', prefix: 'E#', id: parsed.id } };
|
|
63
|
+
const nearest = nearestObservation(db, row.created_at_epoch, project);
|
|
64
|
+
if (!nearest) return { ok: false, error: { code: 'no-obs-near', prefix: 'E#', id: parsed.id } };
|
|
65
|
+
return { ok: true, anchorId: nearest.id, anchorNote: `(anchored to #${nearest.id}, closest obs to E#${parsed.id})` };
|
|
66
|
+
}
|
|
67
|
+
|
|
56
68
|
// Bare "#N" or "N" — observation first. Route compressed obs to its live
|
|
57
69
|
// parent so the window (which filters compressed) isn't shown around a dead
|
|
58
70
|
// record; negative sentinels (-1 dropped, -2 pending purge) have no parent.
|
package/mem-cli.mjs
CHANGED
|
@@ -18,7 +18,7 @@ import { selectCompressionCandidates, groupByProjectWeek, compressGroup } from '
|
|
|
18
18
|
import { buildNotLowSignalSql } from './lib/low-signal-patterns.mjs';
|
|
19
19
|
import {
|
|
20
20
|
cleanupBroken, decayAndMarkIdle, boostAccessed, demotePinned, mergeDuplicates,
|
|
21
|
-
recoverOrphanedChildren, recoverBuriedLessons,
|
|
21
|
+
recoverOrphanedChildren, recoverBuriedLessons, sweepDeferredWorkOrphans,
|
|
22
22
|
purgeStale, purgeStalePreview, findDuplicates, maintenanceStats, rebuildVectors, vacuum,
|
|
23
23
|
recoverChildrenOf, hardDeleteCandidateCount,
|
|
24
24
|
OP_CAP, STALE_AGE_MS, PINNED_INJ_THRESHOLD,
|
|
@@ -76,7 +76,7 @@ async function cmdSearch(db, args, { llm } = {}) {
|
|
|
76
76
|
fail(`[mem] Invalid --type "${type}". Valid: ${[...validObsTypes].join(', ')}`);
|
|
77
77
|
return;
|
|
78
78
|
}
|
|
79
|
-
const source = flags.source || null; // observations|sessions|prompts (null = all)
|
|
79
|
+
const source = flags.source || null; // observations|sessions|prompts|events (null = all)
|
|
80
80
|
const project = flags.project ? resolveProject(db, flags.project) : null;
|
|
81
81
|
const bounds = parseDateBounds(flags.from, flags.to, flags.since);
|
|
82
82
|
if (!bounds.ok) {
|
|
@@ -135,8 +135,8 @@ async function cmdSearch(db, args, { llm } = {}) {
|
|
|
135
135
|
process.stderr.write('[mem] Note: --rerank requires --deep (it reranks deep-search candidates); ignored\n');
|
|
136
136
|
}
|
|
137
137
|
|
|
138
|
-
if (source && !['observations', 'sessions', 'prompts'].includes(source)) {
|
|
139
|
-
fail(`[mem] Invalid --source "${source}". Use: observations, sessions, prompts`);
|
|
138
|
+
if (source && !['observations', 'sessions', 'prompts', 'events'].includes(source)) {
|
|
139
|
+
fail(`[mem] Invalid --source "${source}". Use: observations, sessions, prompts, events`);
|
|
140
140
|
return;
|
|
141
141
|
}
|
|
142
142
|
|
|
@@ -176,9 +176,23 @@ async function cmdSearch(db, args, { llm } = {}) {
|
|
|
176
176
|
if (deepMode === 'deep' && source && source !== 'observations') {
|
|
177
177
|
process.stderr.write(`[mem] Note: --deep searches observations only; ignoring --source ${source}\n`);
|
|
178
178
|
}
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
179
|
+
// branch/tier are obs-exclusive columns → force observations. --type (obs_type) maps to both
|
|
180
|
+
// observations.type AND events.event_type, so scope to obs+events and skip the type-less
|
|
181
|
+
// sessions/prompts legs (D#76). --importance rides the obsTypeScoped path (events carry importance).
|
|
182
|
+
let effectiveSource;
|
|
183
|
+
let obsTypeScoped = false;
|
|
184
|
+
if (deepMode === 'deep') {
|
|
185
|
+
effectiveSource = 'observations';
|
|
186
|
+
} else if (source) {
|
|
187
|
+
effectiveSource = source;
|
|
188
|
+
} else if (type && !branch && !tier) {
|
|
189
|
+
effectiveSource = null;
|
|
190
|
+
obsTypeScoped = true;
|
|
191
|
+
} else if (minImportance || branch || tier) {
|
|
192
|
+
effectiveSource = 'observations';
|
|
193
|
+
} else {
|
|
194
|
+
effectiveSource = null;
|
|
195
|
+
}
|
|
182
196
|
|
|
183
197
|
const res = await coreRunSearchPipeline(
|
|
184
198
|
{
|
|
@@ -191,6 +205,7 @@ async function cmdSearch(db, args, { llm } = {}) {
|
|
|
191
205
|
limit, offset, project: project || null, obsType: type, importance: minImportance,
|
|
192
206
|
branch, includeNoise, epochFrom: dateFrom, epochTo: dateTo, sort, tier,
|
|
193
207
|
// ── CLI surface policy ──
|
|
208
|
+
obsTypeScoped, // D#76: obs_type ⇒ obs+events (skip type-less sessions/prompts)
|
|
194
209
|
obsTypeFallback: false, // #8217 removed list-by-type fallback from the CLI
|
|
195
210
|
crossSourceEpochSortNoFts: false, // CLI never reaches cross-source with empty ftsQuery (fails earlier)
|
|
196
211
|
rerankPolicy: 'cli', // re-rank/supersede on any obs; re-sort gated on cross-source
|
|
@@ -244,7 +259,7 @@ async function cmdSearch(db, args, { llm } = {}) {
|
|
|
244
259
|
|
|
245
260
|
// "N of M" total when paged < total (paired-path with server.mjs formatSearchOutput, #8198).
|
|
246
261
|
const showTime = sort === 'time';
|
|
247
|
-
const hasMixed = paged.some(r => r.source === 'session' || r.source === 'prompt');
|
|
262
|
+
const hasMixed = paged.some(r => r.source === 'session' || r.source === 'prompt' || r.source === 'event');
|
|
248
263
|
// Suppressed when --or was explicit — user already asked for OR, no "fallback" there.
|
|
249
264
|
const fallbackHint = orFallbackFired && !useOr ? ' (relaxed AND→OR)' : '';
|
|
250
265
|
|
|
@@ -290,7 +305,7 @@ async function cmdSearch(db, args, { llm } = {}) {
|
|
|
290
305
|
const countLabel = total > paged.length ? `${paged.length} of ${total}` : `${paged.length}`;
|
|
291
306
|
// Pluralize on total — "Found 1 of 44 result" reads wrong; the population (44) drives
|
|
292
307
|
// grammatical number, not the page slice (1).
|
|
293
|
-
out(`[mem] Found ${countLabel} result${total !== 1 ? 's' : ''} for "${query}"${fallbackHint}:${hasMixed ? ' (# observation, S# session, P# prompt)' : ''}`);
|
|
308
|
+
out(`[mem] Found ${countLabel} result${total !== 1 ? 's' : ''} for "${query}"${fallbackHint}:${hasMixed ? ' (# observation, S# session, P# prompt, E# event)' : ''}`);
|
|
294
309
|
// `~Nt` = est. tokens to fetch this row's full body via mem_get (attachBodyTokens, paired with
|
|
295
310
|
// MCP). Conditional so a row that skipped enrichment renders cleanly, not "~undefinedt".
|
|
296
311
|
const tok = r => (r.bodyTokens ? ` ~${r.bodyTokens}t` : '');
|
|
@@ -302,6 +317,12 @@ async function cmdSearch(db, args, { llm } = {}) {
|
|
|
302
317
|
} else if (r.source === 'prompt') {
|
|
303
318
|
const date = fmtDateShort(r.created_at);
|
|
304
319
|
out(`P#${r.id} 💬 ${date}${timeStr} ${truncate(r.prompt_text || '(empty)', 80)}${tok(r)}`);
|
|
320
|
+
} else if (r.source === 'event') {
|
|
321
|
+
const date = fmtDateShort(r.created_at);
|
|
322
|
+
out(`E#${r.id} ${typeIcon(r.type)} ${date}${timeStr} ${truncate(r.title || '(untitled)', 80)}${tok(r)}`);
|
|
323
|
+
if (r.lesson_learned) {
|
|
324
|
+
out(` -> ${truncate(r.lesson_learned, 80)}`);
|
|
325
|
+
}
|
|
305
326
|
} else {
|
|
306
327
|
const date = fmtDateShort(r.created_at);
|
|
307
328
|
const title = truncate(r.title || r.subtitle || '(untitled)', 80);
|
|
@@ -526,12 +547,31 @@ function renderPromptRows(db, ids) {
|
|
|
526
547
|
return { text: parts.join('\n\n'), count: rows.length };
|
|
527
548
|
}
|
|
528
549
|
|
|
550
|
+
function renderEventRows(db, ids) {
|
|
551
|
+
const placeholders = ids.map(() => '?').join(',');
|
|
552
|
+
const rows = db.prepare(`SELECT * FROM events WHERE id IN (${placeholders}) ORDER BY created_at_epoch ASC`).all(...ids);
|
|
553
|
+
if (rows.length === 0) return null;
|
|
554
|
+
const parts = [];
|
|
555
|
+
for (const r of rows) {
|
|
556
|
+
// events store the distilled lesson in `body`; only *_epoch is available for the date.
|
|
557
|
+
const lines = [`E#${r.id} [${r.event_type}] ${r.created_at_epoch ? fmtDateShort(new Date(r.created_at_epoch).toISOString()) : ''}`];
|
|
558
|
+
if (r.title) lines.push(`Title: ${r.title}`);
|
|
559
|
+
if (r.body) lines.push(`Body: ${r.body}`);
|
|
560
|
+
if (r.project) lines.push(`Project: ${r.project}`);
|
|
561
|
+
if (r.importance !== null && r.importance !== undefined) lines.push(`Importance: ${r.importance}`);
|
|
562
|
+
if (r.file_paths) lines.push(`Files: ${r.file_paths}`);
|
|
563
|
+
if (r.git_sha) lines.push(`Git: ${r.git_sha}`);
|
|
564
|
+
parts.push(lines.join('\n'));
|
|
565
|
+
}
|
|
566
|
+
return { text: parts.join('\n\n'), count: rows.length };
|
|
567
|
+
}
|
|
568
|
+
|
|
529
569
|
function cmdGet(db, args) {
|
|
530
570
|
const { positional, flags } = parseArgs(args);
|
|
531
571
|
const idStr = positional.join(',');
|
|
532
572
|
if (!idStr) {
|
|
533
|
-
fail('[mem] Usage: claude-mem-lite get <id1,id2,...> [--source obs|session|prompt] [--fields f1,f2,...]\n' +
|
|
534
|
-
' IDs accept prefix from search output: #123 (obs), P#123 (prompt), S#123 (session).');
|
|
573
|
+
fail('[mem] Usage: claude-mem-lite get <id1,id2,...> [--source obs|session|prompt|event] [--fields f1,f2,...]\n' +
|
|
574
|
+
' IDs accept prefix from search output: #123 (obs), P#123 (prompt), S#123 (session), E#123 (event).');
|
|
535
575
|
return;
|
|
536
576
|
}
|
|
537
577
|
|
|
@@ -539,18 +579,18 @@ function cmdGet(db, args) {
|
|
|
539
579
|
|
|
540
580
|
// Explicit --source overrides any prefix; otherwise each token's prefix routes individually.
|
|
541
581
|
const explicit = flags.source;
|
|
542
|
-
const validSources = new Set(['obs', 'session', 'prompt']);
|
|
582
|
+
const validSources = new Set(['obs', 'session', 'prompt', 'event']);
|
|
543
583
|
if (explicit && !validSources.has(explicit)) {
|
|
544
|
-
fail(`[mem] Invalid --source "${explicit}". Use: obs, session, prompt`);
|
|
584
|
+
fail(`[mem] Invalid --source "${explicit}". Use: obs, session, prompt, event`);
|
|
545
585
|
return;
|
|
546
586
|
}
|
|
547
587
|
|
|
548
|
-
// Shared bucketing with MCP mem_get — single source of truth for P#/S#/# routing (#8050).
|
|
588
|
+
// Shared bucketing with MCP mem_get — single source of truth for P#/S#/E#/# routing (#8050).
|
|
549
589
|
const { bySrc, invalid: unparseable } = bucketIdTokens(tokens, { explicit, defaultSource: 'obs' });
|
|
550
590
|
if (unparseable.length > 0) {
|
|
551
591
|
process.stderr.write(`[mem] Ignoring unparseable ID token(s): ${unparseable.join(', ')}\n`);
|
|
552
592
|
}
|
|
553
|
-
if (bySrc.obs.length + bySrc.session.length + bySrc.prompt.length === 0) {
|
|
593
|
+
if (bySrc.obs.length + bySrc.session.length + bySrc.prompt.length + bySrc.event.length === 0) {
|
|
554
594
|
fail('[mem] No valid IDs provided');
|
|
555
595
|
return;
|
|
556
596
|
}
|
|
@@ -585,11 +625,15 @@ function cmdGet(db, args) {
|
|
|
585
625
|
const s = renderPromptRows(db, bySrc.prompt);
|
|
586
626
|
if (s) { sections.push(s.text); totalFound += s.count; }
|
|
587
627
|
}
|
|
628
|
+
if (bySrc.event.length > 0) {
|
|
629
|
+
const s = renderEventRows(db, bySrc.event);
|
|
630
|
+
if (s) { sections.push(s.text); totalFound += s.count; }
|
|
631
|
+
}
|
|
588
632
|
|
|
589
633
|
if (totalFound === 0) {
|
|
590
634
|
// Probe the OTHER sources so the caller can retry with the right prefix.
|
|
591
635
|
const queried = new Set(Object.entries(bySrc).filter(([, v]) => v.length > 0).map(([k]) => k));
|
|
592
|
-
const allIds = [...bySrc.obs, ...bySrc.session, ...bySrc.prompt];
|
|
636
|
+
const allIds = [...bySrc.obs, ...bySrc.session, ...bySrc.prompt, ...bySrc.event];
|
|
593
637
|
const probe = probeIdSources(db, allIds, queried);
|
|
594
638
|
const hits = formatProbeHints(probe);
|
|
595
639
|
const hint = hits.length > 0 ? ` Try: ${hits.join('; ')}.` : '';
|
|
@@ -1431,10 +1475,10 @@ function cmdDelete(db, args) {
|
|
|
1431
1475
|
// delete operates on observations only. Reject P#/S# explicitly so callers aren't
|
|
1432
1476
|
// surprised by silent NaN filtering when they paste search-output IDs.
|
|
1433
1477
|
const tokens = idStr.split(',').map(s => s.trim()).filter(Boolean);
|
|
1434
|
-
const nonObs = tokens.filter(t => /^[
|
|
1478
|
+
const nonObs = tokens.filter(t => /^[EePpSs]#?\d+$/.test(t));
|
|
1435
1479
|
if (nonObs.length > 0) {
|
|
1436
1480
|
fail(`[mem] delete only works on observations. Rejected: ${nonObs.join(', ')}. ` +
|
|
1437
|
-
`Prompts and
|
|
1481
|
+
`Prompts, sessions, and events are not deletable here — inspect with \`claude-mem-lite get P#N --source prompt\` / \`--source session\` / \`--source event\`.`);
|
|
1438
1482
|
return;
|
|
1439
1483
|
}
|
|
1440
1484
|
const ids = tokens.map(t => {
|
|
@@ -1508,9 +1552,9 @@ function cmdDelete(db, args) {
|
|
|
1508
1552
|
function cmdUpdate(db, args) {
|
|
1509
1553
|
const { positional, flags } = parseArgs(args);
|
|
1510
1554
|
const raw = positional[0];
|
|
1511
|
-
if (raw && /^[
|
|
1555
|
+
if (raw && /^[EePpSs]#?\d+$/.test(String(raw).trim())) {
|
|
1512
1556
|
fail(`[mem] update only works on observations. Rejected: ${raw}. ` +
|
|
1513
|
-
`Prompts and
|
|
1557
|
+
`Prompts, sessions, and events are not editable here.`);
|
|
1514
1558
|
return;
|
|
1515
1559
|
}
|
|
1516
1560
|
// Strict parseIdToken gate (aligned with cmdDelete): a bare parseInt fallback
|
|
@@ -2044,6 +2088,10 @@ function cmdMaintain(db, args) {
|
|
|
2044
2088
|
// lesson-bearing rows only; idempotent no-op once none remain.
|
|
2045
2089
|
const lessonsHealed = recoverBuriedLessons(db, mctx);
|
|
2046
2090
|
if (lessonsHealed > 0) results.push(`Healed ${lessonsHealed} lesson rows buried at importance 0`);
|
|
2091
|
+
// Heal deferred_work rows whose closing obs / source prompt was hard-deleted while FK was
|
|
2092
|
+
// OFF (dangling ref foreign_key_check flags). Applies the ON DELETE SET NULL the FK would.
|
|
2093
|
+
const deferredHealed = sweepDeferredWorkOrphans(db, mctx);
|
|
2094
|
+
if (deferredHealed > 0) results.push(`Healed ${deferredHealed} deferred-work rows with dangling references`);
|
|
2047
2095
|
}
|
|
2048
2096
|
|
|
2049
2097
|
if (ops.includes('decay')) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "claude-mem-lite",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.45.0",
|
|
4
4
|
"description": "Persistent long-term memory for Claude Code via MCP — captures coding decisions, bugfixes, and context across sessions. Hybrid FTS5 + TF-IDF search with episode batching. Single SQLite DB, no external services. A lighter, lower-cost alternative to claude-mem (episode batching + a smaller model; cost savings are an internal estimate, not a measured benchmark).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"packageManager": "npm@10.9.2",
|
package/schema.mjs
CHANGED
|
@@ -104,7 +104,15 @@ export const CODE_DIR = join(homedir(), '.claude-mem-lite');
|
|
|
104
104
|
// previously-uncited obs (see applyCitationDecay). REAL new column, so unlike v38-v40
|
|
105
105
|
// this DOES advance LATEST_MIGRATION_COLUMN (→ observations.last_cited_session_id);
|
|
106
106
|
// existing DBs reach the ALTER because version 40 != 41 falls through the fast-path.
|
|
107
|
-
|
|
107
|
+
// v42 (events_fts self-heal): events_fts was the one FTS table outside ensureFTS's
|
|
108
|
+
// column-aware recreation (its DDL is non-standard — UNINDEXED cols + custom tokenizer +
|
|
109
|
+
// events_fts_* trigger names — so it can't use the generic ensureFTS). Adds a dedicated
|
|
110
|
+
// ensureEventsFTS run in the migration body so a future events column addition self-heals
|
|
111
|
+
// instead of leaving a stale narrow index whose (wider) triggers throw "no column" and
|
|
112
|
+
// silently drop event writes. NO new column, so LATEST_MIGRATION_COLUMN is unchanged — the
|
|
113
|
+
// forced pass alone carries it (same pattern as v35/v36/v38/v39/v40); existing DBs run it
|
|
114
|
+
// because version 41 != 42 falls through the fast-path.
|
|
115
|
+
export const CURRENT_SCHEMA_VERSION = 42;
|
|
108
116
|
|
|
109
117
|
// Sentinel column for the LATEST migration set. The fast-path uses this to
|
|
110
118
|
// self-heal half-migrated DBs — schema_version bumped but column ALTERs rolled
|
|
@@ -555,6 +563,12 @@ export function initSchema(db) {
|
|
|
555
563
|
END;
|
|
556
564
|
`);
|
|
557
565
|
|
|
566
|
+
// v42: column-aware self-heal for events_fts (the one FTS table the generic ensureFTS can't
|
|
567
|
+
// manage — UNINDEXED cols + custom tokenizer + events_fts_* triggers). The CREATE ... IF NOT
|
|
568
|
+
// EXISTS above never widens a drifted (older, narrower) events_fts; this drops+recreates it on
|
|
569
|
+
// column drift and repopulates. No-op on a healthy DB.
|
|
570
|
+
ensureEventsFTS(db);
|
|
571
|
+
|
|
558
572
|
// Observation files junction table for normalized file lookups (replaces LIKE scans on files_modified JSON)
|
|
559
573
|
db.exec(`
|
|
560
574
|
CREATE TABLE IF NOT EXISTS observation_files (
|
|
@@ -1067,3 +1081,65 @@ export function ensureFTS(db, ftsName, tableName, columns) {
|
|
|
1067
1081
|
} catch { /* non-critical — index repopulates lazily on next write */ }
|
|
1068
1082
|
}
|
|
1069
1083
|
}
|
|
1084
|
+
|
|
1085
|
+
// Column-aware self-heal for events_fts — the events table's FTS index. events_fts is NOT
|
|
1086
|
+
// managed by the generic ensureFTS() above because its DDL is non-standard: event_type and
|
|
1087
|
+
// project are UNINDEXED, it uses a custom unicode61 tokenizer with '_-' tokenchars, and its
|
|
1088
|
+
// triggers are named events_fts_* (not events_*). Routing it through ensureFTS would recreate
|
|
1089
|
+
// it WITHOUT the UNINDEXED cols / tokenizer AND install a SECOND, differently-named trigger set
|
|
1090
|
+
// (events_*) that double-writes the index. This dedicated guard mirrors ensureFTS's drift
|
|
1091
|
+
// detection while preserving the exact events_fts DDL (schema.mjs events block) — closing the
|
|
1092
|
+
// F8/P2-4 gap: events_fts was the one FTS table outside self-heal, so a future events column
|
|
1093
|
+
// addition would leave a stale narrow index whose (wider) triggers throw "no column" and
|
|
1094
|
+
// silently drop event writes. Idempotent: a no-op once the column set matches.
|
|
1095
|
+
const EVENTS_FTS_COLUMNS = ['title', 'body', 'event_type', 'project']; // full set (drift check)
|
|
1096
|
+
export function ensureEventsFTS(db) {
|
|
1097
|
+
const ftsRow = db.prepare(`SELECT 1 FROM sqlite_master WHERE type='table' AND name='events_fts'`).get();
|
|
1098
|
+
let recreated = false;
|
|
1099
|
+
if (ftsRow) {
|
|
1100
|
+
let existingCols = [];
|
|
1101
|
+
try { existingCols = db.prepare(`PRAGMA table_info(events_fts)`).all().map(c => c.name); } catch { /* unreadable → recreate */ }
|
|
1102
|
+
const drifted = existingCols.length !== EVENTS_FTS_COLUMNS.length || EVENTS_FTS_COLUMNS.some(c => !existingCols.includes(c));
|
|
1103
|
+
if (drifted) {
|
|
1104
|
+
db.exec(`DROP TRIGGER IF EXISTS events_fts_ai`);
|
|
1105
|
+
db.exec(`DROP TRIGGER IF EXISTS events_fts_ad`);
|
|
1106
|
+
db.exec(`DROP TRIGGER IF EXISTS events_fts_au`);
|
|
1107
|
+
db.exec(`DROP TABLE IF EXISTS events_fts`);
|
|
1108
|
+
recreated = true;
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
if (!ftsRow || recreated) {
|
|
1112
|
+
db.exec(`
|
|
1113
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS events_fts USING fts5(
|
|
1114
|
+
title, body, event_type UNINDEXED, project UNINDEXED,
|
|
1115
|
+
content='events', content_rowid='id',
|
|
1116
|
+
tokenize="unicode61 remove_diacritics 2 tokenchars '_-'"
|
|
1117
|
+
);
|
|
1118
|
+
`);
|
|
1119
|
+
}
|
|
1120
|
+
// Triggers reinstated from the canonical template (INSERT all 4 columns; AFTER UPDATE OF the
|
|
1121
|
+
// two INDEXED columns only, so non-indexed bumps don't thrash the index). IF NOT EXISTS so an
|
|
1122
|
+
// unchanged definition is a no-op — byte-identical to the events block in initSchema.
|
|
1123
|
+
db.exec(`
|
|
1124
|
+
CREATE TRIGGER IF NOT EXISTS events_fts_ai AFTER INSERT ON events BEGIN
|
|
1125
|
+
INSERT INTO events_fts(rowid, title, body, event_type, project)
|
|
1126
|
+
VALUES (new.id, COALESCE(new.title,''), COALESCE(new.body,''), new.event_type, new.project);
|
|
1127
|
+
END;
|
|
1128
|
+
CREATE TRIGGER IF NOT EXISTS events_fts_ad AFTER DELETE ON events BEGIN
|
|
1129
|
+
INSERT INTO events_fts(events_fts, rowid, title, body, event_type, project)
|
|
1130
|
+
VALUES ('delete', old.id, COALESCE(old.title,''), COALESCE(old.body,''), old.event_type, old.project);
|
|
1131
|
+
END;
|
|
1132
|
+
CREATE TRIGGER IF NOT EXISTS events_fts_au AFTER UPDATE OF title, body ON events BEGIN
|
|
1133
|
+
INSERT INTO events_fts(events_fts, rowid, title, body, event_type, project)
|
|
1134
|
+
VALUES ('delete', old.id, COALESCE(old.title,''), COALESCE(old.body,''), old.event_type, old.project);
|
|
1135
|
+
INSERT INTO events_fts(rowid, title, body, event_type, project)
|
|
1136
|
+
VALUES (new.id, COALESCE(new.title,''), COALESCE(new.body,''), new.event_type, new.project);
|
|
1137
|
+
END;
|
|
1138
|
+
`);
|
|
1139
|
+
if (recreated) {
|
|
1140
|
+
try {
|
|
1141
|
+
const cnt = db.prepare(`SELECT COUNT(*) AS c FROM events`).get();
|
|
1142
|
+
if (cnt.c > 0) db.exec(`INSERT INTO events_fts(events_fts) VALUES('rebuild')`);
|
|
1143
|
+
} catch { /* non-critical — index repopulates lazily on next write */ }
|
|
1144
|
+
}
|
|
1145
|
+
}
|
package/scoring-sql.mjs
CHANGED
|
@@ -51,6 +51,10 @@ export const OBS_BM25 = 'bm25(observations_fts, 10, 5, 5, 3, 3, 2, 8, 5)';
|
|
|
51
51
|
/** session_summaries_fts BM25 weights: request=5, investigated=3, learned=3, completed=3, next_steps=2, notes=1, remaining_items=1 */
|
|
52
52
|
export const SESS_BM25 = 'bm25(session_summaries_fts, 5, 3, 3, 3, 2, 1, 1)';
|
|
53
53
|
|
|
54
|
+
/** events_fts BM25 weights: title=5, body=2 (event_type/project are UNINDEXED — weight irrelevant).
|
|
55
|
+
* Title-weighted like OBS/SESS so a title hit outranks a body-only hit. */
|
|
56
|
+
export const EVT_BM25 = 'bm25(events_fts, 5, 2)';
|
|
57
|
+
|
|
54
58
|
/** FTS5 columns for observations (must match BM25 weight order) */
|
|
55
59
|
export const OBS_FTS_COLUMNS = ['title', 'subtitle', 'narrative', 'text', 'facts', 'concepts', 'lesson_learned', 'search_aliases'];
|
|
56
60
|
|
package/search-engine.mjs
CHANGED
|
@@ -167,6 +167,28 @@ export function countPromptFtsMatches(db, { ftsQuery, project = null, epochFrom
|
|
|
167
167
|
} catch { return 0; }
|
|
168
168
|
}
|
|
169
169
|
|
|
170
|
+
export function countEventFtsMatches(db, { ftsQuery, project = null, epochFrom = null, epochTo = null, eventType = null, importance = null }) {
|
|
171
|
+
if (!ftsQuery) return 0;
|
|
172
|
+
try {
|
|
173
|
+
const wheres = ['events_fts MATCH ?', 'e.superseded_at_epoch IS NULL'];
|
|
174
|
+
const params = [ftsQuery];
|
|
175
|
+
if (project) { wheres.push('e.project = ?'); params.push(project); }
|
|
176
|
+
if (epochFrom) { wheres.push('e.created_at_epoch >= ?'); params.push(epochFrom); }
|
|
177
|
+
if (epochTo) { wheres.push('e.created_at_epoch <= ?'); params.push(epochTo); }
|
|
178
|
+
// D#76: keep the count in lockstep with searchEventsFts's event_type/importance filters,
|
|
179
|
+
// else the "N of M" population diverges from the rows actually shown.
|
|
180
|
+
if (eventType) { wheres.push('e.event_type = ?'); params.push(eventType); }
|
|
181
|
+
if (importance) { wheres.push('COALESCE(e.importance, 1) >= ?'); params.push(importance); }
|
|
182
|
+
const row = db.prepare(`
|
|
183
|
+
SELECT COUNT(*) as c
|
|
184
|
+
FROM events_fts
|
|
185
|
+
JOIN events e ON events_fts.rowid = e.id
|
|
186
|
+
WHERE ${wheres.join(' AND ')}
|
|
187
|
+
`).get(...params);
|
|
188
|
+
return row?.c ?? 0;
|
|
189
|
+
} catch { return 0; }
|
|
190
|
+
}
|
|
191
|
+
|
|
170
192
|
/**
|
|
171
193
|
* Sum true match counts across the sources that contribute to a cross-source (or
|
|
172
194
|
* source-restricted) search. `obsFtsQuery` lets callers pass the OR-relaxed query
|
|
@@ -176,17 +198,23 @@ export function countPromptFtsMatches(db, { ftsQuery, project = null, epochFrom
|
|
|
176
198
|
export function countSearchTotal(db, {
|
|
177
199
|
effectiveSource = null, ftsQuery, obsFtsQuery = null,
|
|
178
200
|
args = {}, project = null, epochFrom = null, epochTo = null, includeNoise = false,
|
|
201
|
+
obsTypeScoped = false,
|
|
179
202
|
}) {
|
|
180
203
|
let total = 0;
|
|
181
204
|
if (!effectiveSource || effectiveSource === 'observations') {
|
|
182
205
|
total += countObsFtsMatches(db, { ftsQuery: obsFtsQuery || ftsQuery, args, epochFrom, epochTo, includeNoise });
|
|
183
206
|
}
|
|
184
|
-
|
|
207
|
+
// D#76: obsTypeScoped (obs_type given, no branch/tier) counts obs + type-filtered events only —
|
|
208
|
+
// sessions/prompts have no type column, so they are excluded from both the results and the count.
|
|
209
|
+
if (!obsTypeScoped && (!effectiveSource || effectiveSource === 'sessions')) {
|
|
185
210
|
total += countSessionFtsMatches(db, { ftsQuery, project, epochFrom, epochTo });
|
|
186
211
|
}
|
|
187
|
-
if (!effectiveSource || effectiveSource === 'prompts') {
|
|
212
|
+
if (!obsTypeScoped && (!effectiveSource || effectiveSource === 'prompts')) {
|
|
188
213
|
total += countPromptFtsMatches(db, { ftsQuery, project, epochFrom, epochTo });
|
|
189
214
|
}
|
|
215
|
+
if (!effectiveSource || effectiveSource === 'events') {
|
|
216
|
+
total += countEventFtsMatches(db, { ftsQuery, project, epochFrom, epochTo, eventType: args.obs_type || null, importance: args.importance || null });
|
|
217
|
+
}
|
|
190
218
|
return total;
|
|
191
219
|
}
|
|
192
220
|
|
|
@@ -235,6 +263,8 @@ export function attachBodyTokens(db, results) {
|
|
|
235
263
|
parts = [r.title, r.subtitle, r.lesson_learned, row.narrative, row.facts, row.text];
|
|
236
264
|
} else if (src === 'session') {
|
|
237
265
|
parts = [r.request, r.completed, r.working_on];
|
|
266
|
+
} else if (src === 'event') {
|
|
267
|
+
parts = [r.title, r.lesson_learned]; // events carry title + body(=lesson_learned) on the row
|
|
238
268
|
} else {
|
|
239
269
|
parts = [r.text, r.prompt_text];
|
|
240
270
|
}
|
package/server.mjs
CHANGED
|
@@ -17,7 +17,7 @@ import { resolveAnchorToken, formatAnchorError, resolveQueryAnchor, fetchRecentT
|
|
|
17
17
|
import { buildSearchFtsQuery, parseDateBounds, parseDuration, coreRunSearchPipeline } from './lib/search-core.mjs';
|
|
18
18
|
import {
|
|
19
19
|
cleanupBroken, decayAndMarkIdle, boostAccessed, demotePinned, mergeDuplicates,
|
|
20
|
-
recoverOrphanedChildren, recoverBuriedLessons,
|
|
20
|
+
recoverOrphanedChildren, recoverBuriedLessons, sweepDeferredWorkOrphans,
|
|
21
21
|
purgeStale, purgeStalePreview, findDuplicates, maintenanceStats, rebuildVectors, vacuum,
|
|
22
22
|
recoverChildrenOf, hardDeleteCandidateCount,
|
|
23
23
|
OP_CAP, STALE_AGE_MS,
|
|
@@ -218,7 +218,7 @@ function formatSearchOutput(paginatedResults, args, ftsQuery, totalCount, orFall
|
|
|
218
218
|
const countLabel = totalCount > paginatedResults.length
|
|
219
219
|
? `${paginatedResults.length} of ${totalCount}`
|
|
220
220
|
: `${paginatedResults.length}`;
|
|
221
|
-
const hasMixed = paginatedResults.some(r => r.source === 'session' || r.source === 'prompt');
|
|
221
|
+
const hasMixed = paginatedResults.some(r => r.source === 'session' || r.source === 'prompt' || r.source === 'event');
|
|
222
222
|
// P2-6: empty/omitted query falls through to a "listing recent" path — label it explicitly
|
|
223
223
|
// so callers don't mistake BM25-less results for relevance-ranked ones.
|
|
224
224
|
const qLabel = args.query ? ` for "${args.query}"` : ' (no query — listing recent)';
|
|
@@ -226,7 +226,7 @@ function formatSearchOutput(paginatedResults, args, ftsQuery, totalCount, orFall
|
|
|
226
226
|
// query actually matched only a subset of the terms. Suppressed when the caller
|
|
227
227
|
// explicitly requested OR semantics — there's no "fallback" in that path.
|
|
228
228
|
const fallbackHint = orFallbackFired && !args.or ? ' (relaxed AND→OR)' : '';
|
|
229
|
-
lines.push(`Found ${countLabel} result(s)${qLabel}${fallbackHint}:${hasMixed ? ' (# observation, S# session, P# prompt)' : ''}\n`);
|
|
229
|
+
lines.push(`Found ${countLabel} result(s)${qLabel}${fallbackHint}:${hasMixed ? ' (# observation, S# session, P# prompt, E# event)' : ''}\n`);
|
|
230
230
|
|
|
231
231
|
// `~Nt` = estimated tokens to fetch this row's full body via mem_get (attachBodyTokens).
|
|
232
232
|
// Conditional so a result that skipped enrichment renders cleanly, not "~undefinedt".
|
|
@@ -241,6 +241,8 @@ function formatSearchOutput(paginatedResults, args, ftsQuery, totalCount, orFall
|
|
|
241
241
|
lines.push(`S#${r.id} 📋 ${truncate(r.request || r.completed || '(no summary)')} | ${r.project} | ${fmtDate(r.date)}${tok(r)}`);
|
|
242
242
|
} else if (r.source === 'prompt') {
|
|
243
243
|
lines.push(`P#${r.id} 💬 ${truncate(r.text)} | ${fmtDate(r.date)}${tok(r)}`);
|
|
244
|
+
} else if (r.source === 'event') {
|
|
245
|
+
lines.push(`E#${r.id} ${typeIcon(r.type)} [${r.type}] ${truncate(r.title || '(untitled)')} | ${r.project} | ${fmtDate(r.date)}${tok(r)}`);
|
|
244
246
|
}
|
|
245
247
|
}
|
|
246
248
|
|
|
@@ -287,15 +289,26 @@ async function runSearchPipeline(db, args, { llm, rerankLlm } = {}) {
|
|
|
287
289
|
return { ...formatSearchOutput([], args, ftsQuery, 0), escalated: false, results: [], total: 0, variants: null };
|
|
288
290
|
}
|
|
289
291
|
|
|
290
|
-
//
|
|
291
|
-
// (
|
|
292
|
-
//
|
|
293
|
-
//
|
|
294
|
-
//
|
|
295
|
-
//
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
292
|
+
// Source scoping. deep is observations-only (deepSearch fuses hybrid-obs lists). branch/tier are
|
|
293
|
+
// obs-EXCLUSIVE columns (sessions/prompts/events lack them) → force observations, else those legs
|
|
294
|
+
// return UNFILTERED and leak rows that can't be scoped (the pre-fix cross-source branch/tier leak).
|
|
295
|
+
// obs_type is DIFFERENT: events carry the same type vocabulary (events.event_type), so an obs_type
|
|
296
|
+
// filter maps to obs + events both — scope to those two and skip the type-less sessions/prompts
|
|
297
|
+
// legs (D#76). importance filters obs and events (both carry it), so it rides the obsTypeScoped path.
|
|
298
|
+
let effectiveType;
|
|
299
|
+
let obsTypeScoped = false;
|
|
300
|
+
if (deepMode === 'deep') {
|
|
301
|
+
effectiveType = 'observations';
|
|
302
|
+
} else if (args.type) {
|
|
303
|
+
effectiveType = args.type;
|
|
304
|
+
} else if (args.obs_type && !args.branch && !args.tier) {
|
|
305
|
+
effectiveType = undefined; // cross-source gate open; obsTypeScoped narrows it to obs+events
|
|
306
|
+
obsTypeScoped = true;
|
|
307
|
+
} else if (args.importance || args.branch || args.tier) {
|
|
308
|
+
effectiveType = 'observations';
|
|
309
|
+
} else {
|
|
310
|
+
effectiveType = undefined;
|
|
311
|
+
}
|
|
299
312
|
|
|
300
313
|
const r = await coreRunSearchPipeline(
|
|
301
314
|
{
|
|
@@ -310,6 +323,7 @@ async function runSearchPipeline(db, args, { llm, rerankLlm } = {}) {
|
|
|
310
323
|
includeNoise: args.include_noise === true, epochFrom, epochTo,
|
|
311
324
|
sort: args.sort || 'relevance', tier: args.tier ?? null,
|
|
312
325
|
// ── MCP surface policy ──
|
|
326
|
+
obsTypeScoped, // D#76: obs_type ⇒ obs+events (skip type-less sessions/prompts)
|
|
313
327
|
obsTypeFallback: true, // list-recent-by-type when 0 matches
|
|
314
328
|
crossSourceEpochSortNoFts: true, // epoch-sort cross-source with no ftsQuery
|
|
315
329
|
rerankPolicy: 'mcp', // (ftsQuery||isDeep) gate; re-rank/re-sort on ftsQuery&&!reranked
|
|
@@ -495,9 +509,9 @@ server.registerTool(
|
|
|
495
509
|
const { bySrc, invalid } = bucketIdTokens(args.ids, { explicit: args.source || null, defaultSource: 'obs' });
|
|
496
510
|
if (invalid.length > 0) {
|
|
497
511
|
// Should not happen — schema regex already rejected bad tokens — but guard defensively.
|
|
498
|
-
return { content: [{ type: 'text', text: `Invalid ID token(s): ${invalid.join(', ')}. Expected N, #N, P#N, or
|
|
512
|
+
return { content: [{ type: 'text', text: `Invalid ID token(s): ${invalid.join(', ')}. Expected N, #N, P#N, S#N, or E#N.` }] };
|
|
499
513
|
}
|
|
500
|
-
const totalRequested = bySrc.obs.length + bySrc.session.length + bySrc.prompt.length;
|
|
514
|
+
const totalRequested = bySrc.obs.length + bySrc.session.length + bySrc.prompt.length + bySrc.event.length;
|
|
501
515
|
if (totalRequested === 0) {
|
|
502
516
|
return { content: [{ type: 'text', text: 'No valid IDs provided.' }] };
|
|
503
517
|
}
|
|
@@ -522,7 +536,7 @@ server.registerTool(
|
|
|
522
536
|
|
|
523
537
|
// Per-source fetchers — each returns { rows, foundIds:Set, prefix }.
|
|
524
538
|
const sections = [];
|
|
525
|
-
const foundBySource = { obs: new Set(), session: new Set(), prompt: new Set() };
|
|
539
|
+
const foundBySource = { obs: new Set(), session: new Set(), prompt: new Set(), event: new Set() };
|
|
526
540
|
|
|
527
541
|
if (bySrc.obs.length > 0) {
|
|
528
542
|
const ph = bySrc.obs.map(() => '?').join(',');
|
|
@@ -581,17 +595,37 @@ server.registerTool(
|
|
|
581
595
|
}
|
|
582
596
|
}
|
|
583
597
|
|
|
584
|
-
|
|
598
|
+
if (bySrc.event.length > 0) {
|
|
599
|
+
const ph = bySrc.event.map(() => '?').join(',');
|
|
600
|
+
const rows = db.prepare(`SELECT * FROM events WHERE id IN (${ph}) ORDER BY created_at_epoch ASC`).all(...bySrc.event);
|
|
601
|
+
// events carry the distilled lesson in `body` (persistHaikuSummary writes lesson_learned||narrative).
|
|
602
|
+
// No created_at ISO column (only *_epoch) — render the epoch as an ISO string for a scannable date.
|
|
603
|
+
for (const row of rows) {
|
|
604
|
+
foundBySource.event.add(row.id);
|
|
605
|
+
const lines = [`── E#${row.id} [${row.event_type}] ──`];
|
|
606
|
+
if (row.title) lines.push(`title: ${row.title}`);
|
|
607
|
+
if (row.body) lines.push(`body: ${row.body.length > 500 ? row.body.slice(0, 500) + '…' : row.body}`);
|
|
608
|
+
if (row.project) lines.push(`project: ${row.project}`);
|
|
609
|
+
if (row.importance !== null && row.importance !== undefined) lines.push(`importance: ${row.importance}`);
|
|
610
|
+
if (row.file_paths) lines.push(`file_paths: ${row.file_paths}`);
|
|
611
|
+
if (row.git_sha) lines.push(`git_sha: ${row.git_sha}`);
|
|
612
|
+
if (row.created_at_epoch) lines.push(`created_at: ${new Date(row.created_at_epoch).toISOString()}`);
|
|
613
|
+
sections.push(lines.join('\n'));
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
const totalFound = foundBySource.obs.size + foundBySource.session.size + foundBySource.prompt.size + foundBySource.event.size;
|
|
585
618
|
|
|
586
619
|
if (totalFound === 0) {
|
|
587
620
|
// Probe other sources so callers can retry with the right prefix/source override.
|
|
588
621
|
const queried = new Set(Object.entries(bySrc).filter(([, v]) => v.length > 0).map(([k]) => k));
|
|
589
|
-
const allNumericIds = [...bySrc.obs, ...bySrc.session, ...bySrc.prompt];
|
|
622
|
+
const allNumericIds = [...bySrc.obs, ...bySrc.session, ...bySrc.prompt, ...bySrc.event];
|
|
590
623
|
const probe = probeIdSources(db, allNumericIds, queried);
|
|
591
624
|
const hints = [];
|
|
592
625
|
if (probe.obs.length > 0) hints.push(`#${probe.obs.join(', #')} (obs — use source='obs' or bare #N)`);
|
|
593
626
|
if (probe.session.length > 0) hints.push(`S#${probe.session.join(', S#')} (session — use source='session' or S#N)`);
|
|
594
627
|
if (probe.prompt.length > 0) hints.push(`P#${probe.prompt.join(', P#')} (prompt — use source='prompt' or P#N)`);
|
|
628
|
+
if (probe.event.length > 0) hints.push(`E#${probe.event.join(', E#')} (event — use source='event' or E#N)`);
|
|
595
629
|
const hint = hints.length > 0 ? ` Try: ${hints.join('; ')}.` : '';
|
|
596
630
|
const queriedList = [...queried].join(', ');
|
|
597
631
|
const msg = `No records found in source(s) [${queriedList}] for the given ID(s).${hint}`;
|
|
@@ -605,6 +639,7 @@ server.registerTool(
|
|
|
605
639
|
missingHints.push(...miss(bySrc.obs, foundBySource.obs, '#'));
|
|
606
640
|
missingHints.push(...miss(bySrc.session, foundBySource.session, 'S#'));
|
|
607
641
|
missingHints.push(...miss(bySrc.prompt, foundBySource.prompt, 'P#'));
|
|
642
|
+
missingHints.push(...miss(bySrc.event, foundBySource.event, 'E#'));
|
|
608
643
|
|
|
609
644
|
const parts = [];
|
|
610
645
|
if (fieldsNote) parts.push(fieldsNote);
|
|
@@ -1158,6 +1193,10 @@ server.registerTool(
|
|
|
1158
1193
|
// lesson-bearing rows only; idempotent no-op once none remain.
|
|
1159
1194
|
const lessonsHealed = recoverBuriedLessons(db, mctx);
|
|
1160
1195
|
if (lessonsHealed > 0) results.push(`Healed ${lessonsHealed} lesson rows buried at importance 0`);
|
|
1196
|
+
// Heal deferred_work rows whose closing obs / source prompt was deleted while FK was
|
|
1197
|
+
// OFF (dangling ref foreign_key_check flags). Applies the ON DELETE SET NULL the FK would.
|
|
1198
|
+
const deferredHealed = sweepDeferredWorkOrphans(db, mctx);
|
|
1199
|
+
if (deferredHealed > 0) results.push(`Healed ${deferredHealed} deferred-work rows with dangling references`);
|
|
1161
1200
|
}
|
|
1162
1201
|
|
|
1163
1202
|
if (ops.includes('decay')) {
|
package/tool-schemas.mjs
CHANGED
|
@@ -75,12 +75,12 @@ const coerceMixedIdTokens = z.preprocess(
|
|
|
75
75
|
}
|
|
76
76
|
return v;
|
|
77
77
|
},
|
|
78
|
-
z.array(z.string().regex(/^[
|
|
78
|
+
z.array(z.string().regex(/^[EePpSs]?#?\d+$/, 'Expected N, #N, P#N, S#N, or E#N')).min(1).max(20)
|
|
79
79
|
);
|
|
80
80
|
|
|
81
81
|
export const memSearchSchema = {
|
|
82
82
|
query: z.string().optional().describe('Search query (FTS5 syntax supported)'),
|
|
83
|
-
type: z.enum(['observations', 'sessions', 'prompts']).optional().describe('Limit to one table'),
|
|
83
|
+
type: z.enum(['observations', 'sessions', 'prompts', 'events']).optional().describe('Limit to one source table (default: all). events = the canonical bugfix/feature/decision/lesson history (auto-captured event-typed memories)'),
|
|
84
84
|
obs_type: OBS_TYPE_ENUM.optional().describe('Filter observation type'),
|
|
85
85
|
project: z.string().optional().describe('Filter by project name'),
|
|
86
86
|
date_from: z.string().optional().describe('Start date (ISO 8601 or YYYY-MM-DD)'),
|
|
@@ -120,12 +120,12 @@ const coerceAnchor = z.preprocess(
|
|
|
120
120
|
},
|
|
121
121
|
z.union([
|
|
122
122
|
z.number().int(),
|
|
123
|
-
z.string().regex(/^[
|
|
123
|
+
z.string().regex(/^[EePpSs]?#?\d+$/, 'Expected N, #N, P#N, S#N, or E#N'),
|
|
124
124
|
])
|
|
125
125
|
);
|
|
126
126
|
|
|
127
127
|
export const memTimelineSchema = {
|
|
128
|
-
anchor: coerceAnchor.optional().describe('Anchor as observation ID (int) or prefixed token string: "#123", "P#123" (prompt → nearest obs), "S#123" (session → nearest obs). Takes precedence over query.'),
|
|
128
|
+
anchor: coerceAnchor.optional().describe('Anchor as observation ID (int) or prefixed token string: "#123", "P#123" (prompt → nearest obs), "S#123" (session → nearest obs), "E#123" (event → nearest obs). Takes precedence over query.'),
|
|
129
129
|
query: z.string().optional().describe('FTS5 query to auto-find anchor. Ignored when anchor is also given; use one or the other.'),
|
|
130
130
|
before: coerceInt.pipe(z.number().int().min(0).max(50)).optional().describe('Items before anchor (default 5)'),
|
|
131
131
|
after: coerceInt.pipe(z.number().int().min(0).max(50)).optional().describe('Items after anchor (default 5)'),
|
|
@@ -136,8 +136,8 @@ export const memGetSchema = {
|
|
|
136
136
|
// Accepts mixed tokens so pasted search results work verbatim: [1], [1, "P#2"], "1,P#2,S#3",
|
|
137
137
|
// or the JSON-stringified form ["1","P#2"]. Each token's prefix routes to its source bucket
|
|
138
138
|
// in server.mjs via lib/id-routing.bucketIdTokens. An explicit `source` override still wins.
|
|
139
|
-
ids: coerceMixedIdTokens.describe('Mixed observation/prompt/session IDs — accepts N, #N, P#N, S#N; comma-strings and JSON arrays also coerced'),
|
|
140
|
-
source: z.enum(['obs', 'session', 'prompt']).optional().describe('Force all IDs to this source (overrides per-token prefixes). Omit to let P#/S#/# prefixes route individually.'),
|
|
139
|
+
ids: coerceMixedIdTokens.describe('Mixed observation/prompt/session/event IDs — accepts N, #N, P#N, S#N, E#N; comma-strings and JSON arrays also coerced'),
|
|
140
|
+
source: z.enum(['obs', 'session', 'prompt', 'event']).optional().describe('Force all IDs to this source (overrides per-token prefixes). Omit to let P#/S#/E#/# prefixes route individually.'),
|
|
141
141
|
fields: coerceStringArray.optional().describe('Specific fields to return (default: all; validated against obs schema — session/prompt sources ignore this filter)'),
|
|
142
142
|
};
|
|
143
143
|
|
|
@@ -355,7 +355,8 @@ export const tools = [
|
|
|
355
355
|
{
|
|
356
356
|
name: 'mem_search',
|
|
357
357
|
description:
|
|
358
|
-
'Full-text search across observations, sessions, and
|
|
358
|
+
'Full-text search across observations, sessions, prompts, and events — the canonical\n' +
|
|
359
|
+
'bugfix/feature/decision/lesson history (FTS5, BM25-ranked).\n' +
|
|
359
360
|
'\n' +
|
|
360
361
|
'DO NOT use when:\n' +
|
|
361
362
|
' - The SessionStart context or hook-injected memory already shows #NN entries that answer the question\n' +
|
|
@@ -363,9 +364,8 @@ export const tools = [
|
|
|
363
364
|
' - You want everything about a specific file (use mem_recall — cheaper and file-scoped)\n' +
|
|
364
365
|
'\n' +
|
|
365
366
|
'USE when:\n' +
|
|
366
|
-
' - Investigating
|
|
367
|
+
' - Investigating an error keyword with obs_type="bugfix" (matches observations AND events)\n' +
|
|
367
368
|
' - Looking for prior art on a module/feature before refactoring\n' +
|
|
368
|
-
' - User asks "have we seen this before" or references something not in visible context\n' +
|
|
369
369
|
' - A normal search missed — weak results auto-escalate to deep (set deep=false to opt out)\n' +
|
|
370
370
|
'\n' +
|
|
371
371
|
'Equivalent CLI: ' + CLI_INVOKE + ' search "<query>" [--type bugfix] [--deep]',
|
package/utils.mjs
CHANGED
|
@@ -9,7 +9,7 @@ import { buildLowSignalRegex } from './lib/low-signal-patterns.mjs';
|
|
|
9
9
|
// ─── Re-exports from extracted modules ──────────────────────────────────────
|
|
10
10
|
// Backward compatibility: all consumers import from utils.mjs
|
|
11
11
|
|
|
12
|
-
export { DECAY_HALF_LIFE_BY_TYPE, DEFAULT_DECAY_HALF_LIFE_MS, OBS_BM25, SESS_BM25, TYPE_DECAY_CASE, TYPE_QUALITY_CASE, OBS_FTS_COLUMNS, notLowSignalTitleClause, noisePenaltyClause } from './scoring-sql.mjs';
|
|
12
|
+
export { DECAY_HALF_LIFE_BY_TYPE, DEFAULT_DECAY_HALF_LIFE_MS, OBS_BM25, SESS_BM25, EVT_BM25, TYPE_DECAY_CASE, TYPE_QUALITY_CASE, OBS_FTS_COLUMNS, notLowSignalTitleClause, noisePenaltyClause } from './scoring-sql.mjs';
|
|
13
13
|
export { cjkBigrams, extractCjkSynonymTokens, extractCjkKeywords, extractCjkLikePatterns, SYNONYM_MAP, expandToken, sanitizeFtsQuery, relaxFtsQueryToOr, FTS_STOP_WORDS, CJK_COMPOUNDS } from './nlp.mjs';
|
|
14
14
|
export { resolveProject, _resetProjectCache } from './project-utils.mjs';
|
|
15
15
|
export { scrubSecrets, SECRET_PATTERNS } from './secret-scrub.mjs';
|
|
@@ -323,9 +323,17 @@ export function parseJsonFromLLM(text) {
|
|
|
323
323
|
// First balanced object — survives unfenced output wrapped in brace-containing prose.
|
|
324
324
|
const balanced = firstBalancedJsonObject(text);
|
|
325
325
|
if (balanced) try { return JSON.parse(balanced); } catch {}
|
|
326
|
-
// Last-resort
|
|
327
|
-
|
|
328
|
-
|
|
326
|
+
// Last-resort span from the first `{` to the last `}` — handles a payload that isn't the
|
|
327
|
+
// FIRST balanced object. Resolved via index scan (O(n)) rather than the greedy
|
|
328
|
+
// /\{[\s\S]*\}/, which backtracks O(n²) across k unclosed opening braces (a synthetic
|
|
329
|
+
// "{{{…" with no close made the regex O(n²); LLM outputs are bounded so it wasn't
|
|
330
|
+
// exploitable — defense in depth). Behaviorally identical: the greedy match anchors on the
|
|
331
|
+
// first `{` that has any `}` after it and, being greedy, ends at the last `}`.
|
|
332
|
+
const firstBrace = text.indexOf('{');
|
|
333
|
+
const lastBrace = text.lastIndexOf('}');
|
|
334
|
+
if (firstBrace !== -1 && lastBrace > firstBrace) {
|
|
335
|
+
try { return JSON.parse(text.slice(firstBrace, lastBrace + 1)); } catch {}
|
|
336
|
+
}
|
|
329
337
|
return null;
|
|
330
338
|
}
|
|
331
339
|
|
|
@@ -456,6 +464,3 @@ export function getCurrentBranch() {
|
|
|
456
464
|
_branchCacheTime = now;
|
|
457
465
|
return _cachedBranch;
|
|
458
466
|
}
|
|
459
|
-
|
|
460
|
-
/** Reset cached branch (for testing or after git checkout) */
|
|
461
|
-
export function _resetBranchCache() { _cachedBranch = undefined; _branchCacheTime = 0; }
|