claude-memory-layer 1.0.11 → 1.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +60 -0
- package/README.md +166 -2
- package/bootstrap-kb/decisions/decisions.md +244 -0
- package/bootstrap-kb/glossary/glossary.md +46 -0
- package/bootstrap-kb/modules/.claude-plugin.md +22 -0
- package/bootstrap-kb/modules/agents.md.md +15 -0
- package/bootstrap-kb/modules/claude.md.md +15 -0
- package/bootstrap-kb/modules/context.md.md +15 -0
- package/bootstrap-kb/modules/docs.md +18 -0
- package/bootstrap-kb/modules/handoff.md.md +15 -0
- package/bootstrap-kb/modules/package-lock.json.md +15 -0
- package/bootstrap-kb/modules/package.json.md +15 -0
- package/bootstrap-kb/modules/plan.md.md +15 -0
- package/bootstrap-kb/modules/readme.md.md +15 -0
- package/bootstrap-kb/modules/scripts.md +26 -0
- package/bootstrap-kb/modules/spec.md.md +15 -0
- package/bootstrap-kb/modules/specs.md +20 -0
- package/bootstrap-kb/modules/src.md +51 -0
- package/bootstrap-kb/modules/tests.md +42 -0
- package/bootstrap-kb/modules/tsconfig.json.md +15 -0
- package/bootstrap-kb/modules/vitest.config.ts.md +15 -0
- package/bootstrap-kb/overview/overview.md +40 -0
- package/bootstrap-kb/sources/manifest.json +950 -0
- package/bootstrap-kb/sources/manifest.md +227 -0
- package/bootstrap-kb/timeline/timeline.md +57 -0
- package/d.sh +3 -0
- package/deploy.sh +3 -0
- package/dist/cli/index.js +2389 -286
- package/dist/cli/index.js.map +4 -4
- package/dist/core/index.js +1017 -132
- package/dist/core/index.js.map +4 -4
- package/dist/hooks/post-tool-use.js +1347 -202
- package/dist/hooks/post-tool-use.js.map +4 -4
- package/dist/hooks/session-end.js +1339 -194
- package/dist/hooks/session-end.js.map +4 -4
- package/dist/hooks/session-start.js +1343 -198
- package/dist/hooks/session-start.js.map +4 -4
- package/dist/hooks/stop.js +1351 -206
- package/dist/hooks/stop.js.map +4 -4
- package/dist/hooks/user-prompt-submit.js +1347 -202
- package/dist/hooks/user-prompt-submit.js.map +4 -4
- package/dist/server/api/index.js +1436 -211
- package/dist/server/api/index.js.map +4 -4
- package/dist/server/index.js +1445 -220
- package/dist/server/index.js.map +4 -4
- package/dist/services/memory-service.js +1345 -199
- package/dist/services/memory-service.js.map +4 -4
- package/dist/ui/app.js +69 -2
- package/dist/ui/index.html +8 -0
- package/docs/MCP_MEMORY_SERVICE_COMPARATIVE_REVIEW.md +271 -0
- package/docs/MEMU_ADOPTION.md +40 -0
- package/memory/.claude-plugin/commands/2026-02-25.md +263 -0
- package/memory/_index.md +405 -0
- package/memory/default/uncategorized/2026-02-25.md +4839 -0
- package/memory/specs/20260207-dashboard-upgrade/2026-02-25.md +142 -0
- package/memory/specs/citations-system/2026-02-25.md +1121 -0
- package/memory/specs/endless-mode/2026-02-25.md +1392 -0
- package/memory/specs/entity-edge-model/2026-02-25.md +1263 -0
- package/memory/specs/evidence-aligner-v2/2026-02-25.md +1028 -0
- package/memory/specs/mcp-desktop-integration/2026-02-25.md +1334 -0
- package/memory/specs/post-tool-use-hook/2026-02-25.md +1164 -0
- package/memory/specs/private-tags/2026-02-25.md +1057 -0
- package/memory/specs/progressive-disclosure/2026-02-25.md +1436 -0
- package/memory/specs/task-entity-system/2026-02-25.md +924 -0
- package/memory/specs/vector-outbox-v2/2026-02-25.md +1510 -0
- package/memory/specs/web-viewer-ui/2026-02-25.md +1709 -0
- package/package.json +2 -1
- package/scripts/build.ts +6 -0
- package/src/cli/index.ts +281 -2
- package/src/core/consolidated-store.ts +63 -1
- package/src/core/consolidation-worker.ts +115 -6
- package/src/core/event-store.ts +14 -0
- package/src/core/index.ts +1 -0
- package/src/core/ingest-interceptor.ts +80 -0
- package/src/core/markdown-mirror.ts +70 -0
- package/src/core/md-mirror.ts +92 -0
- package/src/core/mongo-sync-config.ts +165 -0
- package/src/core/mongo-sync-worker.ts +381 -0
- package/src/core/retriever.ts +540 -150
- package/src/core/sqlite-event-store.ts +350 -1
- package/src/core/tag-taxonomy.ts +51 -0
- package/src/core/types.ts +28 -0
- package/src/server/api/health.ts +53 -0
- package/src/server/api/index.ts +3 -1
- package/src/server/api/stats.ts +46 -1
- package/src/services/bootstrap-organizer.ts +443 -0
- package/src/services/codex-session-history-importer.ts +474 -0
- package/src/services/memory-service.ts +373 -68
- package/src/ui/app.js +69 -2
- package/src/ui/index.html +8 -0
- package/tests/bootstrap-organizer.test.ts +111 -0
- package/tests/consolidation-worker.test.ts +75 -0
- package/tests/ingest-interceptor.test.ts +38 -0
- package/tests/markdown-mirror.test.ts +85 -0
- package/tests/md-mirror.test.ts +50 -0
- package/tests/retriever-fallback-chain.test.ts +223 -0
- package/tests/retriever-strategy-scope.test.ts +97 -0
- package/tests/retriever.memu-adoption.test.ts +122 -0
- package/tests/sqlite-event-store-replication.test.ts +92 -0
|
@@ -7,9 +7,9 @@ const __filename = fileURLToPath(import.meta.url);
|
|
|
7
7
|
const __dirname = dirname(__filename);
|
|
8
8
|
|
|
9
9
|
// src/services/memory-service.ts
|
|
10
|
-
import * as
|
|
10
|
+
import * as path3 from "path";
|
|
11
11
|
import * as os from "os";
|
|
12
|
-
import * as
|
|
12
|
+
import * as fs4 from "fs";
|
|
13
13
|
import * as crypto2 from "crypto";
|
|
14
14
|
|
|
15
15
|
// src/core/event-store.ts
|
|
@@ -67,11 +67,11 @@ function toDate(value) {
|
|
|
67
67
|
return new Date(value);
|
|
68
68
|
return new Date(String(value));
|
|
69
69
|
}
|
|
70
|
-
function createDatabase(
|
|
70
|
+
function createDatabase(path5, options) {
|
|
71
71
|
if (options?.readOnly) {
|
|
72
|
-
return new duckdb.Database(
|
|
72
|
+
return new duckdb.Database(path5, { access_mode: "READ_ONLY" });
|
|
73
73
|
}
|
|
74
|
-
return new duckdb.Database(
|
|
74
|
+
return new duckdb.Database(path5);
|
|
75
75
|
}
|
|
76
76
|
function dbRun(db, sql, params = []) {
|
|
77
77
|
return new Promise((resolve2, reject) => {
|
|
@@ -335,6 +335,17 @@ var EventStore = class {
|
|
|
335
335
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
336
336
|
)
|
|
337
337
|
`);
|
|
338
|
+
await dbRun(this.db, `
|
|
339
|
+
CREATE TABLE IF NOT EXISTS consolidated_rules (
|
|
340
|
+
rule_id VARCHAR PRIMARY KEY,
|
|
341
|
+
rule TEXT NOT NULL,
|
|
342
|
+
topics JSON,
|
|
343
|
+
source_memory_ids JSON,
|
|
344
|
+
source_events JSON,
|
|
345
|
+
confidence FLOAT DEFAULT 0.5,
|
|
346
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
347
|
+
)
|
|
348
|
+
`);
|
|
338
349
|
await dbRun(this.db, `
|
|
339
350
|
CREATE TABLE IF NOT EXISTS endless_config (
|
|
340
351
|
key VARCHAR PRIMARY KEY,
|
|
@@ -354,6 +365,7 @@ var EventStore = class {
|
|
|
354
365
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_working_set_expires ON working_set(expires_at)`);
|
|
355
366
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_working_set_relevance ON working_set(relevance_score DESC)`);
|
|
356
367
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_consolidated_confidence ON consolidated_memories(confidence DESC)`);
|
|
368
|
+
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_consolidated_rules_confidence ON consolidated_rules(confidence DESC)`);
|
|
357
369
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_continuity_created ON continuity_log(created_at)`);
|
|
358
370
|
this.initialized = true;
|
|
359
371
|
}
|
|
@@ -743,12 +755,12 @@ import { randomUUID as randomUUID2 } from "crypto";
|
|
|
743
755
|
import Database from "better-sqlite3";
|
|
744
756
|
import * as fs from "fs";
|
|
745
757
|
import * as nodePath from "path";
|
|
746
|
-
function createSQLiteDatabase(
|
|
747
|
-
const dir = nodePath.dirname(
|
|
758
|
+
function createSQLiteDatabase(path5, options) {
|
|
759
|
+
const dir = nodePath.dirname(path5);
|
|
748
760
|
if (!fs.existsSync(dir)) {
|
|
749
761
|
fs.mkdirSync(dir, { recursive: true });
|
|
750
762
|
}
|
|
751
|
-
const db = new Database(
|
|
763
|
+
const db = new Database(path5, {
|
|
752
764
|
readonly: options?.readonly ?? false
|
|
753
765
|
});
|
|
754
766
|
if (!options?.readonly && (options?.walMode ?? true)) {
|
|
@@ -789,6 +801,64 @@ function toSQLiteTimestamp(date) {
|
|
|
789
801
|
return date.toISOString();
|
|
790
802
|
}
|
|
791
803
|
|
|
804
|
+
// src/core/markdown-mirror.ts
|
|
805
|
+
import * as fs2 from "fs/promises";
|
|
806
|
+
import * as path from "path";
|
|
807
|
+
var DEFAULT_NAMESPACE = "default";
|
|
808
|
+
var DEFAULT_CATEGORY = "uncategorized";
|
|
809
|
+
function sanitizeSegment(input, fallback) {
|
|
810
|
+
const raw = String(input ?? "").trim().toLowerCase();
|
|
811
|
+
const safe = raw.normalize("NFKD").replace(/[^a-z0-9_-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
812
|
+
if (!safe || safe === "." || safe === "..")
|
|
813
|
+
return fallback;
|
|
814
|
+
return safe;
|
|
815
|
+
}
|
|
816
|
+
function getCategorySegments(metadata, eventType) {
|
|
817
|
+
const raw = metadata?.categoryPath;
|
|
818
|
+
if (Array.isArray(raw) && raw.length > 0) {
|
|
819
|
+
return raw.map((s) => sanitizeSegment(s, DEFAULT_CATEGORY));
|
|
820
|
+
}
|
|
821
|
+
const single = metadata?.category;
|
|
822
|
+
if (typeof single === "string" && single.trim()) {
|
|
823
|
+
return [sanitizeSegment(single, DEFAULT_CATEGORY)];
|
|
824
|
+
}
|
|
825
|
+
return [sanitizeSegment(eventType, DEFAULT_CATEGORY)];
|
|
826
|
+
}
|
|
827
|
+
function buildMirrorPath(rootDir, event) {
|
|
828
|
+
const metadata = event.metadata;
|
|
829
|
+
const namespace = sanitizeSegment(metadata?.namespace, DEFAULT_NAMESPACE);
|
|
830
|
+
const categories = getCategorySegments(metadata, event.eventType);
|
|
831
|
+
const d = event.timestamp;
|
|
832
|
+
const yyyy = d.getFullYear();
|
|
833
|
+
const mm = String(d.getMonth() + 1).padStart(2, "0");
|
|
834
|
+
const dd = String(d.getDate()).padStart(2, "0");
|
|
835
|
+
return path.join(rootDir, "memory", namespace, ...categories, `${yyyy}-${mm}-${dd}.md`);
|
|
836
|
+
}
|
|
837
|
+
function formatMirrorEntry(event) {
|
|
838
|
+
const category = Array.isArray(event.metadata?.categoryPath) ? event.metadata.categoryPath.join("/") : String(event.metadata?.category ?? event.eventType);
|
|
839
|
+
return [
|
|
840
|
+
"",
|
|
841
|
+
`- ts: ${event.timestamp.toISOString()}`,
|
|
842
|
+
` id: ${event.id}`,
|
|
843
|
+
` type: ${event.eventType}`,
|
|
844
|
+
` session: ${event.sessionId}`,
|
|
845
|
+
` category: ${category}`,
|
|
846
|
+
" content: |",
|
|
847
|
+
...event.content.split("\n").map((line) => ` ${line}`)
|
|
848
|
+
].join("\n") + "\n";
|
|
849
|
+
}
|
|
850
|
+
var MarkdownMirror = class {
|
|
851
|
+
constructor(rootDir) {
|
|
852
|
+
this.rootDir = rootDir;
|
|
853
|
+
}
|
|
854
|
+
async append(event) {
|
|
855
|
+
const outPath = buildMirrorPath(this.rootDir, event);
|
|
856
|
+
await fs2.mkdir(path.dirname(outPath), { recursive: true });
|
|
857
|
+
await fs2.appendFile(outPath, formatMirrorEntry(event), "utf8");
|
|
858
|
+
return outPath;
|
|
859
|
+
}
|
|
860
|
+
};
|
|
861
|
+
|
|
792
862
|
// src/core/sqlite-event-store.ts
|
|
793
863
|
var SQLiteEventStore = class {
|
|
794
864
|
constructor(dbPath, options) {
|
|
@@ -798,10 +868,12 @@ var SQLiteEventStore = class {
|
|
|
798
868
|
readonly: this.readOnly,
|
|
799
869
|
walMode: !this.readOnly
|
|
800
870
|
});
|
|
871
|
+
this.markdownMirror = this.readOnly || !options?.markdownMirrorRoot ? null : new MarkdownMirror(options.markdownMirrorRoot);
|
|
801
872
|
}
|
|
802
873
|
db;
|
|
803
874
|
initialized = false;
|
|
804
875
|
readOnly;
|
|
876
|
+
markdownMirror;
|
|
805
877
|
/**
|
|
806
878
|
* Initialize database schema
|
|
807
879
|
*/
|
|
@@ -1008,6 +1080,17 @@ var SQLiteEventStore = class {
|
|
|
1008
1080
|
created_at TEXT DEFAULT (datetime('now'))
|
|
1009
1081
|
);
|
|
1010
1082
|
|
|
1083
|
+
-- Consolidated Rules table (long-term stable memory)
|
|
1084
|
+
CREATE TABLE IF NOT EXISTS consolidated_rules (
|
|
1085
|
+
rule_id TEXT PRIMARY KEY,
|
|
1086
|
+
rule TEXT NOT NULL,
|
|
1087
|
+
topics TEXT,
|
|
1088
|
+
source_memory_ids TEXT,
|
|
1089
|
+
source_events TEXT,
|
|
1090
|
+
confidence REAL DEFAULT 0.5,
|
|
1091
|
+
created_at TEXT DEFAULT (datetime('now'))
|
|
1092
|
+
);
|
|
1093
|
+
|
|
1011
1094
|
-- Endless Mode Config table
|
|
1012
1095
|
CREATE TABLE IF NOT EXISTS endless_config (
|
|
1013
1096
|
key TEXT PRIMARY KEY,
|
|
@@ -1032,6 +1115,24 @@ var SQLiteEventStore = class {
|
|
|
1032
1115
|
measured_at TEXT
|
|
1033
1116
|
);
|
|
1034
1117
|
|
|
1118
|
+
-- Retrieval trace log (query -> candidates -> selected for context)
|
|
1119
|
+
CREATE TABLE IF NOT EXISTS retrieval_traces (
|
|
1120
|
+
trace_id TEXT PRIMARY KEY,
|
|
1121
|
+
session_id TEXT,
|
|
1122
|
+
project_hash TEXT,
|
|
1123
|
+
query_text TEXT NOT NULL,
|
|
1124
|
+
strategy TEXT,
|
|
1125
|
+
candidate_event_ids TEXT,
|
|
1126
|
+
selected_event_ids TEXT,
|
|
1127
|
+
candidate_details_json TEXT,
|
|
1128
|
+
selected_details_json TEXT,
|
|
1129
|
+
candidate_count INTEGER DEFAULT 0,
|
|
1130
|
+
selected_count INTEGER DEFAULT 0,
|
|
1131
|
+
confidence TEXT,
|
|
1132
|
+
fallback_trace TEXT,
|
|
1133
|
+
created_at TEXT DEFAULT (datetime('now'))
|
|
1134
|
+
);
|
|
1135
|
+
|
|
1035
1136
|
-- Sync position tracking (for SQLite -> DuckDB sync)
|
|
1036
1137
|
CREATE TABLE IF NOT EXISTS sync_positions (
|
|
1037
1138
|
target_name TEXT PRIMARY KEY,
|
|
@@ -1056,10 +1157,14 @@ var SQLiteEventStore = class {
|
|
|
1056
1157
|
CREATE INDEX IF NOT EXISTS idx_working_set_relevance ON working_set(relevance_score);
|
|
1057
1158
|
CREATE INDEX IF NOT EXISTS idx_consolidated_confidence ON consolidated_memories(confidence);
|
|
1058
1159
|
CREATE INDEX IF NOT EXISTS idx_continuity_created ON continuity_log(created_at);
|
|
1160
|
+
CREATE INDEX IF NOT EXISTS idx_consolidated_rules_confidence ON consolidated_rules(confidence);
|
|
1059
1161
|
CREATE INDEX IF NOT EXISTS idx_embedding_outbox_status ON embedding_outbox(status);
|
|
1060
1162
|
CREATE INDEX IF NOT EXISTS idx_helpfulness_event ON memory_helpfulness(event_id);
|
|
1061
1163
|
CREATE INDEX IF NOT EXISTS idx_helpfulness_session ON memory_helpfulness(session_id);
|
|
1062
1164
|
CREATE INDEX IF NOT EXISTS idx_helpfulness_score ON memory_helpfulness(helpfulness_score DESC);
|
|
1165
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_created_at ON retrieval_traces(created_at DESC);
|
|
1166
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_project_hash ON retrieval_traces(project_hash);
|
|
1167
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_session_id ON retrieval_traces(session_id);
|
|
1063
1168
|
|
|
1064
1169
|
-- FTS5 Full-Text Search for fast keyword search
|
|
1065
1170
|
CREATE VIRTUAL TABLE IF NOT EXISTS events_fts USING fts5(
|
|
@@ -1083,6 +1188,14 @@ var SQLiteEventStore = class {
|
|
|
1083
1188
|
INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
|
|
1084
1189
|
END;
|
|
1085
1190
|
`);
|
|
1191
|
+
try {
|
|
1192
|
+
sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN selected_details_json TEXT;`);
|
|
1193
|
+
} catch {
|
|
1194
|
+
}
|
|
1195
|
+
try {
|
|
1196
|
+
sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN candidate_details_json TEXT;`);
|
|
1197
|
+
} catch {
|
|
1198
|
+
}
|
|
1086
1199
|
const tableInfo = sqliteAll(this.db, "PRAGMA table_info(events)", []);
|
|
1087
1200
|
const columnNames = tableInfo.map((col) => col.name);
|
|
1088
1201
|
if (!columnNames.includes("access_count")) {
|
|
@@ -1182,6 +1295,21 @@ var SQLiteEventStore = class {
|
|
|
1182
1295
|
insertLevel.run(id);
|
|
1183
1296
|
});
|
|
1184
1297
|
transaction();
|
|
1298
|
+
if (this.markdownMirror) {
|
|
1299
|
+
const event = {
|
|
1300
|
+
id,
|
|
1301
|
+
eventType: input.eventType,
|
|
1302
|
+
sessionId: input.sessionId,
|
|
1303
|
+
timestamp: input.timestamp,
|
|
1304
|
+
content: input.content,
|
|
1305
|
+
canonicalKey,
|
|
1306
|
+
dedupeKey,
|
|
1307
|
+
metadata
|
|
1308
|
+
};
|
|
1309
|
+
this.markdownMirror.append(event).catch((err) => {
|
|
1310
|
+
console.warn("[SQLiteEventStore] markdown mirror append failed:", err);
|
|
1311
|
+
});
|
|
1312
|
+
}
|
|
1185
1313
|
return { success: true, eventId: id, isDuplicate: false };
|
|
1186
1314
|
} catch (error) {
|
|
1187
1315
|
return {
|
|
@@ -1240,6 +1368,92 @@ var SQLiteEventStore = class {
|
|
|
1240
1368
|
);
|
|
1241
1369
|
return rows.map(this.rowToEvent);
|
|
1242
1370
|
}
|
|
1371
|
+
/**
|
|
1372
|
+
* Get events since a SQLite rowid (for robust incremental replication).
|
|
1373
|
+
* Rowid is monotonic for append-only tables, independent of client timestamps.
|
|
1374
|
+
*/
|
|
1375
|
+
async getEventsSinceRowid(lastRowid, limit = 1e3) {
|
|
1376
|
+
await this.initialize();
|
|
1377
|
+
const rows = sqliteAll(
|
|
1378
|
+
this.db,
|
|
1379
|
+
`SELECT rowid as _rowid, * FROM events WHERE rowid > ? ORDER BY rowid ASC LIMIT ?`,
|
|
1380
|
+
[lastRowid, limit]
|
|
1381
|
+
);
|
|
1382
|
+
return rows.map((row) => ({
|
|
1383
|
+
rowid: row._rowid,
|
|
1384
|
+
event: this.rowToEvent(row)
|
|
1385
|
+
}));
|
|
1386
|
+
}
|
|
1387
|
+
/**
|
|
1388
|
+
* Import events with fixed IDs (used for cross-machine replication).
|
|
1389
|
+
* Idempotent: skips if event id or dedupeKey already exists.
|
|
1390
|
+
*
|
|
1391
|
+
* NOTE: This bypasses the append() id generation to preserve stable IDs.
|
|
1392
|
+
*/
|
|
1393
|
+
async importEvents(events) {
|
|
1394
|
+
if (events.length === 0)
|
|
1395
|
+
return { inserted: 0, skipped: 0 };
|
|
1396
|
+
if (this.readOnly)
|
|
1397
|
+
return { inserted: 0, skipped: events.length };
|
|
1398
|
+
await this.initialize();
|
|
1399
|
+
const getById = this.db.prepare(`SELECT id FROM events WHERE id = ?`);
|
|
1400
|
+
const getByDedupe = this.db.prepare(`SELECT event_id FROM event_dedup WHERE dedupe_key = ?`);
|
|
1401
|
+
const insertEvent = this.db.prepare(`
|
|
1402
|
+
INSERT INTO events (id, event_type, session_id, timestamp, content, canonical_key, dedupe_key, metadata, turn_id)
|
|
1403
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1404
|
+
`);
|
|
1405
|
+
const insertDedup = this.db.prepare(`
|
|
1406
|
+
INSERT INTO event_dedup (dedupe_key, event_id) VALUES (?, ?)
|
|
1407
|
+
`);
|
|
1408
|
+
const insertLevel = this.db.prepare(`
|
|
1409
|
+
INSERT INTO memory_levels (event_id, level) VALUES (?, 'L0')
|
|
1410
|
+
`);
|
|
1411
|
+
let inserted = 0;
|
|
1412
|
+
let skipped = 0;
|
|
1413
|
+
const insertedEvents = [];
|
|
1414
|
+
const tx = this.db.transaction((batch) => {
|
|
1415
|
+
for (const ev of batch) {
|
|
1416
|
+
const existingById = getById.get(ev.id);
|
|
1417
|
+
if (existingById) {
|
|
1418
|
+
skipped++;
|
|
1419
|
+
continue;
|
|
1420
|
+
}
|
|
1421
|
+
const canonicalKey = ev.canonicalKey || makeCanonicalKey(ev.content);
|
|
1422
|
+
const dedupeKey = ev.dedupeKey || makeDedupeKey(ev.content, ev.sessionId);
|
|
1423
|
+
const existingByDedupe = getByDedupe.get(dedupeKey);
|
|
1424
|
+
if (existingByDedupe) {
|
|
1425
|
+
skipped++;
|
|
1426
|
+
continue;
|
|
1427
|
+
}
|
|
1428
|
+
const metadata = ev.metadata || {};
|
|
1429
|
+
const turnId = metadata.turnId;
|
|
1430
|
+
insertEvent.run(
|
|
1431
|
+
ev.id,
|
|
1432
|
+
ev.eventType,
|
|
1433
|
+
ev.sessionId,
|
|
1434
|
+
toSQLiteTimestamp(ev.timestamp),
|
|
1435
|
+
ev.content,
|
|
1436
|
+
canonicalKey,
|
|
1437
|
+
dedupeKey,
|
|
1438
|
+
JSON.stringify(metadata),
|
|
1439
|
+
turnId ?? null
|
|
1440
|
+
);
|
|
1441
|
+
insertDedup.run(dedupeKey, ev.id);
|
|
1442
|
+
insertLevel.run(ev.id);
|
|
1443
|
+
inserted++;
|
|
1444
|
+
insertedEvents.push(ev);
|
|
1445
|
+
}
|
|
1446
|
+
});
|
|
1447
|
+
tx(events);
|
|
1448
|
+
if (this.markdownMirror && insertedEvents.length > 0) {
|
|
1449
|
+
for (const ev of insertedEvents) {
|
|
1450
|
+
this.markdownMirror.append(ev).catch((err) => {
|
|
1451
|
+
console.warn("[SQLiteEventStore] markdown mirror append failed:", err);
|
|
1452
|
+
});
|
|
1453
|
+
}
|
|
1454
|
+
}
|
|
1455
|
+
return { inserted, skipped };
|
|
1456
|
+
}
|
|
1243
1457
|
/**
|
|
1244
1458
|
* Create or update session
|
|
1245
1459
|
*/
|
|
@@ -1402,6 +1616,35 @@ var SQLiteEventStore = class {
|
|
|
1402
1616
|
[error, ...ids]
|
|
1403
1617
|
);
|
|
1404
1618
|
}
|
|
1619
|
+
/**
|
|
1620
|
+
* Get embedding/vector outbox health statistics
|
|
1621
|
+
*/
|
|
1622
|
+
async getOutboxStats() {
|
|
1623
|
+
await this.initialize();
|
|
1624
|
+
const embeddingRows = sqliteAll(
|
|
1625
|
+
this.db,
|
|
1626
|
+
`SELECT status, COUNT(*) as count FROM embedding_outbox GROUP BY status`
|
|
1627
|
+
);
|
|
1628
|
+
const vectorRows = sqliteAll(
|
|
1629
|
+
this.db,
|
|
1630
|
+
`SELECT status, COUNT(*) as count FROM vector_outbox GROUP BY status`
|
|
1631
|
+
);
|
|
1632
|
+
const fromRows = (rows) => {
|
|
1633
|
+
const out = { pending: 0, processing: 0, failed: 0, total: 0 };
|
|
1634
|
+
for (const row of rows) {
|
|
1635
|
+
const key = row.status;
|
|
1636
|
+
if (key === "pending" || key === "processing" || key === "failed") {
|
|
1637
|
+
out[key] += row.count;
|
|
1638
|
+
}
|
|
1639
|
+
out.total += row.count;
|
|
1640
|
+
}
|
|
1641
|
+
return out;
|
|
1642
|
+
};
|
|
1643
|
+
return {
|
|
1644
|
+
embedding: fromRows(embeddingRows),
|
|
1645
|
+
vector: fromRows(vectorRows)
|
|
1646
|
+
};
|
|
1647
|
+
}
|
|
1405
1648
|
/**
|
|
1406
1649
|
* Update memory level
|
|
1407
1650
|
*/
|
|
@@ -1760,6 +2003,79 @@ var SQLiteEventStore = class {
|
|
|
1760
2003
|
getDatabase() {
|
|
1761
2004
|
return this.db;
|
|
1762
2005
|
}
|
|
2006
|
+
async recordRetrievalTrace(input) {
|
|
2007
|
+
await this.initialize();
|
|
2008
|
+
const traceId = randomUUID2();
|
|
2009
|
+
sqliteRun(
|
|
2010
|
+
this.db,
|
|
2011
|
+
`INSERT INTO retrieval_traces (
|
|
2012
|
+
trace_id, session_id, project_hash, query_text, strategy,
|
|
2013
|
+
candidate_event_ids, selected_event_ids, candidate_details_json, selected_details_json,
|
|
2014
|
+
candidate_count, selected_count, confidence, fallback_trace
|
|
2015
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
2016
|
+
[
|
|
2017
|
+
traceId,
|
|
2018
|
+
input.sessionId || null,
|
|
2019
|
+
input.projectHash || null,
|
|
2020
|
+
input.queryText,
|
|
2021
|
+
input.strategy || null,
|
|
2022
|
+
JSON.stringify(input.candidateEventIds || []),
|
|
2023
|
+
JSON.stringify(input.selectedEventIds || []),
|
|
2024
|
+
JSON.stringify(input.candidateDetails || []),
|
|
2025
|
+
JSON.stringify(input.selectedDetails || []),
|
|
2026
|
+
(input.candidateEventIds || []).length,
|
|
2027
|
+
(input.selectedEventIds || []).length,
|
|
2028
|
+
input.confidence || null,
|
|
2029
|
+
JSON.stringify(input.fallbackTrace || [])
|
|
2030
|
+
]
|
|
2031
|
+
);
|
|
2032
|
+
}
|
|
2033
|
+
async getRecentRetrievalTraces(limit = 50) {
|
|
2034
|
+
await this.initialize();
|
|
2035
|
+
const rows = sqliteAll(
|
|
2036
|
+
this.db,
|
|
2037
|
+
`SELECT * FROM retrieval_traces ORDER BY created_at DESC LIMIT ?`,
|
|
2038
|
+
[limit]
|
|
2039
|
+
);
|
|
2040
|
+
return rows.map((row) => ({
|
|
2041
|
+
traceId: row.trace_id,
|
|
2042
|
+
sessionId: row.session_id || void 0,
|
|
2043
|
+
projectHash: row.project_hash || void 0,
|
|
2044
|
+
queryText: row.query_text,
|
|
2045
|
+
strategy: row.strategy || void 0,
|
|
2046
|
+
candidateEventIds: row.candidate_event_ids ? JSON.parse(row.candidate_event_ids) : [],
|
|
2047
|
+
selectedEventIds: row.selected_event_ids ? JSON.parse(row.selected_event_ids) : [],
|
|
2048
|
+
candidateDetails: row.candidate_details_json ? JSON.parse(row.candidate_details_json) : [],
|
|
2049
|
+
selectedDetails: row.selected_details_json ? JSON.parse(row.selected_details_json) : [],
|
|
2050
|
+
candidateCount: Number(row.candidate_count || 0),
|
|
2051
|
+
selectedCount: Number(row.selected_count || 0),
|
|
2052
|
+
confidence: row.confidence || void 0,
|
|
2053
|
+
fallbackTrace: row.fallback_trace ? JSON.parse(row.fallback_trace) : [],
|
|
2054
|
+
createdAt: toDateFromSQLite(row.created_at)
|
|
2055
|
+
}));
|
|
2056
|
+
}
|
|
2057
|
+
async getRetrievalTraceStats() {
|
|
2058
|
+
await this.initialize();
|
|
2059
|
+
const row = sqliteGet(
|
|
2060
|
+
this.db,
|
|
2061
|
+
`SELECT
|
|
2062
|
+
COUNT(*) as total_queries,
|
|
2063
|
+
AVG(candidate_count) as avg_candidate_count,
|
|
2064
|
+
AVG(selected_count) as avg_selected_count,
|
|
2065
|
+
CASE
|
|
2066
|
+
WHEN SUM(candidate_count) > 0 THEN (SUM(selected_count) * 1.0 / SUM(candidate_count))
|
|
2067
|
+
ELSE 0
|
|
2068
|
+
END as selection_rate
|
|
2069
|
+
FROM retrieval_traces`,
|
|
2070
|
+
[]
|
|
2071
|
+
);
|
|
2072
|
+
return {
|
|
2073
|
+
totalQueries: Number(row?.total_queries || 0),
|
|
2074
|
+
avgCandidateCount: Number(row?.avg_candidate_count || 0),
|
|
2075
|
+
avgSelectedCount: Number(row?.avg_selected_count || 0),
|
|
2076
|
+
selectionRate: Number(row?.selection_rate || 0)
|
|
2077
|
+
};
|
|
2078
|
+
}
|
|
1763
2079
|
/**
|
|
1764
2080
|
* Close database connection
|
|
1765
2081
|
*/
|
|
@@ -2621,7 +2937,20 @@ var DEFAULT_OPTIONS = {
|
|
|
2621
2937
|
topK: 5,
|
|
2622
2938
|
minScore: 0.7,
|
|
2623
2939
|
maxTokens: 2e3,
|
|
2624
|
-
includeSessionContext: true
|
|
2940
|
+
includeSessionContext: true,
|
|
2941
|
+
strategy: "auto",
|
|
2942
|
+
rerankWithKeyword: true,
|
|
2943
|
+
decayPolicy: {
|
|
2944
|
+
enabled: true,
|
|
2945
|
+
windowDays: 30,
|
|
2946
|
+
maxPenalty: 0.15
|
|
2947
|
+
},
|
|
2948
|
+
graphHop: {
|
|
2949
|
+
enabled: true,
|
|
2950
|
+
maxHops: 1,
|
|
2951
|
+
hopPenalty: 0.08
|
|
2952
|
+
},
|
|
2953
|
+
projectScopeMode: "global"
|
|
2625
2954
|
};
|
|
2626
2955
|
var Retriever = class {
|
|
2627
2956
|
eventStore;
|
|
@@ -2631,6 +2960,7 @@ var Retriever = class {
|
|
|
2631
2960
|
sharedStore;
|
|
2632
2961
|
sharedVectorStore;
|
|
2633
2962
|
graduation;
|
|
2963
|
+
queryRewriter;
|
|
2634
2964
|
constructor(eventStore, vectorStore, embedder, matcher, sharedOptions) {
|
|
2635
2965
|
this.eventStore = eventStore;
|
|
2636
2966
|
this.vectorStore = vectorStore;
|
|
@@ -2639,47 +2969,105 @@ var Retriever = class {
|
|
|
2639
2969
|
this.sharedStore = sharedOptions?.sharedStore;
|
|
2640
2970
|
this.sharedVectorStore = sharedOptions?.sharedVectorStore;
|
|
2641
2971
|
}
|
|
2642
|
-
/**
|
|
2643
|
-
* Set graduation pipeline for access tracking
|
|
2644
|
-
*/
|
|
2645
2972
|
setGraduationPipeline(graduation) {
|
|
2646
2973
|
this.graduation = graduation;
|
|
2647
2974
|
}
|
|
2648
|
-
/**
|
|
2649
|
-
* Set shared stores after construction
|
|
2650
|
-
*/
|
|
2651
2975
|
setSharedStores(sharedStore, sharedVectorStore) {
|
|
2652
2976
|
this.sharedStore = sharedStore;
|
|
2653
2977
|
this.sharedVectorStore = sharedVectorStore;
|
|
2654
2978
|
}
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2979
|
+
setQueryRewriter(rewriter) {
|
|
2980
|
+
this.queryRewriter = rewriter;
|
|
2981
|
+
}
|
|
2658
2982
|
async retrieve(query, options = {}) {
|
|
2659
2983
|
const opts = { ...DEFAULT_OPTIONS, ...options };
|
|
2660
|
-
const
|
|
2661
|
-
const
|
|
2662
|
-
|
|
2663
|
-
|
|
2984
|
+
const sessionFilter = opts.scope?.sessionId ?? opts.sessionId;
|
|
2985
|
+
const fallbackTrace = [];
|
|
2986
|
+
const fallbackEnabled = (opts.strategy ?? "auto") === "auto";
|
|
2987
|
+
const primaryStrategy = opts.strategy === "auto" ? "fast" : opts.strategy || "fast";
|
|
2988
|
+
let current = await this.runStage(query, {
|
|
2989
|
+
strategy: primaryStrategy,
|
|
2990
|
+
topK: opts.topK,
|
|
2664
2991
|
minScore: opts.minScore,
|
|
2665
|
-
sessionId:
|
|
2992
|
+
sessionId: sessionFilter,
|
|
2993
|
+
scope: opts.scope,
|
|
2994
|
+
rerankWithKeyword: opts.rerankWithKeyword !== false,
|
|
2995
|
+
rerankWeights: opts.rerankWeights,
|
|
2996
|
+
decayPolicy: opts.decayPolicy,
|
|
2997
|
+
intentRewrite: opts.intentRewrite === true,
|
|
2998
|
+
graphHop: opts.graphHop,
|
|
2999
|
+
projectScopeMode: opts.projectScopeMode,
|
|
3000
|
+
projectHash: opts.projectHash,
|
|
3001
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
2666
3002
|
});
|
|
2667
|
-
|
|
2668
|
-
|
|
2669
|
-
|
|
2670
|
-
|
|
2671
|
-
|
|
3003
|
+
fallbackTrace.push(`stage:primary:${primaryStrategy}`);
|
|
3004
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results) && primaryStrategy !== "deep") {
|
|
3005
|
+
current = await this.runStage(query, {
|
|
3006
|
+
strategy: "deep",
|
|
3007
|
+
topK: opts.topK,
|
|
3008
|
+
minScore: opts.minScore,
|
|
3009
|
+
sessionId: sessionFilter,
|
|
3010
|
+
scope: opts.scope,
|
|
3011
|
+
rerankWithKeyword: opts.rerankWithKeyword !== false,
|
|
3012
|
+
rerankWeights: opts.rerankWeights,
|
|
3013
|
+
decayPolicy: opts.decayPolicy,
|
|
3014
|
+
graphHop: opts.graphHop,
|
|
3015
|
+
projectScopeMode: opts.projectScopeMode,
|
|
3016
|
+
projectHash: opts.projectHash,
|
|
3017
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
3018
|
+
});
|
|
3019
|
+
fallbackTrace.push("fallback:deep");
|
|
3020
|
+
}
|
|
3021
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results)) {
|
|
3022
|
+
current = await this.runStage(query, {
|
|
3023
|
+
strategy: "deep",
|
|
3024
|
+
topK: opts.topK,
|
|
3025
|
+
minScore: Math.max(0.5, opts.minScore - 0.15),
|
|
3026
|
+
sessionId: void 0,
|
|
3027
|
+
scope: void 0,
|
|
3028
|
+
rerankWithKeyword: true,
|
|
3029
|
+
rerankWeights: opts.rerankWeights,
|
|
3030
|
+
decayPolicy: opts.decayPolicy,
|
|
3031
|
+
graphHop: opts.graphHop,
|
|
3032
|
+
projectScopeMode: opts.projectScopeMode,
|
|
3033
|
+
projectHash: opts.projectHash,
|
|
3034
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
3035
|
+
});
|
|
3036
|
+
fallbackTrace.push("fallback:scope-expanded");
|
|
3037
|
+
}
|
|
3038
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results)) {
|
|
3039
|
+
const summary = await this.buildSummaryFallback(query, opts.topK);
|
|
3040
|
+
current = {
|
|
3041
|
+
results: summary,
|
|
3042
|
+
candidateResults: summary,
|
|
3043
|
+
matchResult: this.matcher.matchSearchResults(summary, () => 0)
|
|
3044
|
+
};
|
|
3045
|
+
fallbackTrace.push("fallback:summary");
|
|
3046
|
+
}
|
|
3047
|
+
const memories = await this.enrichResults(current.results.slice(0, opts.topK), opts);
|
|
2672
3048
|
const context = this.buildContext(memories, opts.maxTokens);
|
|
2673
3049
|
return {
|
|
2674
3050
|
memories,
|
|
2675
|
-
matchResult,
|
|
3051
|
+
matchResult: current.matchResult,
|
|
2676
3052
|
totalTokens: this.estimateTokens(context),
|
|
2677
|
-
context
|
|
3053
|
+
context,
|
|
3054
|
+
fallbackTrace,
|
|
3055
|
+
selectedDebug: current.results.slice(0, opts.topK).map((r) => ({
|
|
3056
|
+
eventId: r.eventId,
|
|
3057
|
+
score: r.score,
|
|
3058
|
+
semanticScore: r.semanticScore,
|
|
3059
|
+
lexicalScore: r.lexicalScore,
|
|
3060
|
+
recencyScore: r.recencyScore
|
|
3061
|
+
})),
|
|
3062
|
+
candidateDebug: (current.candidateResults || []).slice(0, Math.max(opts.topK * 3, 20)).map((r) => ({
|
|
3063
|
+
eventId: r.eventId,
|
|
3064
|
+
score: r.score,
|
|
3065
|
+
semanticScore: r.semanticScore,
|
|
3066
|
+
lexicalScore: r.lexicalScore,
|
|
3067
|
+
recencyScore: r.recencyScore
|
|
3068
|
+
}))
|
|
2678
3069
|
};
|
|
2679
3070
|
}
|
|
2680
|
-
/**
|
|
2681
|
-
* Retrieve with unified search (project + shared)
|
|
2682
|
-
*/
|
|
2683
3071
|
async retrieveUnified(query, options = {}) {
|
|
2684
3072
|
const projectResult = await this.retrieve(query, options);
|
|
2685
3073
|
if (!options.includeShared || !this.sharedStore || !this.sharedVectorStore) {
|
|
@@ -2687,22 +3075,19 @@ var Retriever = class {
|
|
|
2687
3075
|
}
|
|
2688
3076
|
try {
|
|
2689
3077
|
const queryEmbedding = await this.embedder.embed(query);
|
|
2690
|
-
const sharedVectorResults = await this.sharedVectorStore.search(
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
excludeProjectHash: options.projectHash
|
|
2696
|
-
}
|
|
2697
|
-
);
|
|
3078
|
+
const sharedVectorResults = await this.sharedVectorStore.search(queryEmbedding.vector, {
|
|
3079
|
+
limit: options.topK || 5,
|
|
3080
|
+
minScore: options.minScore || 0.7,
|
|
3081
|
+
excludeProjectHash: options.projectHash
|
|
3082
|
+
});
|
|
2698
3083
|
const sharedMemories = [];
|
|
2699
3084
|
for (const result of sharedVectorResults) {
|
|
2700
3085
|
const entry = await this.sharedStore.get(result.entryId);
|
|
2701
|
-
if (entry)
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
3086
|
+
if (!entry)
|
|
3087
|
+
continue;
|
|
3088
|
+
if (!options.projectHash || entry.sourceProjectHash !== options.projectHash) {
|
|
3089
|
+
sharedMemories.push(entry);
|
|
3090
|
+
await this.sharedStore.recordUsage(entry.entryId);
|
|
2706
3091
|
}
|
|
2707
3092
|
}
|
|
2708
3093
|
const unifiedContext = this.buildUnifiedContext(projectResult, sharedMemories);
|
|
@@ -2717,50 +3102,243 @@ var Retriever = class {
|
|
|
2717
3102
|
return projectResult;
|
|
2718
3103
|
}
|
|
2719
3104
|
}
|
|
2720
|
-
|
|
2721
|
-
|
|
2722
|
-
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
2735
|
-
|
|
2736
|
-
|
|
2737
|
-
|
|
2738
|
-
|
|
2739
|
-
|
|
2740
|
-
|
|
3105
|
+
async runStage(query, input) {
|
|
3106
|
+
let initialResults = await this.searchByStrategy(query, {
|
|
3107
|
+
strategy: input.strategy,
|
|
3108
|
+
topK: input.topK,
|
|
3109
|
+
minScore: input.minScore,
|
|
3110
|
+
sessionId: input.sessionId
|
|
3111
|
+
});
|
|
3112
|
+
if (input.intentRewrite && input.strategy === "deep" && this.queryRewriter) {
|
|
3113
|
+
const rewritten = (await this.queryRewriter(query))?.trim();
|
|
3114
|
+
if (rewritten && rewritten !== query) {
|
|
3115
|
+
const rewrittenResults = await this.searchByStrategy(rewritten, {
|
|
3116
|
+
strategy: "deep",
|
|
3117
|
+
topK: input.topK,
|
|
3118
|
+
minScore: Math.max(0.5, input.minScore - 0.1),
|
|
3119
|
+
sessionId: input.sessionId
|
|
3120
|
+
});
|
|
3121
|
+
initialResults = this.mergeResults(initialResults, rewrittenResults, input.topK * 3);
|
|
3122
|
+
}
|
|
3123
|
+
}
|
|
3124
|
+
const expandedResults = input.graphHop?.enabled === false ? initialResults : await this.expandGraphHops(initialResults, {
|
|
3125
|
+
maxHops: Math.max(1, input.graphHop?.maxHops ?? 1),
|
|
3126
|
+
hopPenalty: Math.max(0, input.graphHop?.hopPenalty ?? 0.08),
|
|
3127
|
+
limit: input.topK * 4
|
|
3128
|
+
});
|
|
3129
|
+
const rerankedResults = input.rerankWithKeyword ? this.rerankByKeywordOverlap(expandedResults, query, input.rerankWeights, input.decayPolicy) : expandedResults;
|
|
3130
|
+
const filtered = await this.applyScopeFilters(rerankedResults, {
|
|
3131
|
+
scope: input.scope,
|
|
3132
|
+
projectScopeMode: input.projectScopeMode,
|
|
3133
|
+
projectHash: input.projectHash,
|
|
3134
|
+
allowedProjectHashes: input.allowedProjectHashes
|
|
3135
|
+
});
|
|
3136
|
+
const top = filtered.slice(0, input.topK);
|
|
3137
|
+
const matchResult = this.matcher.matchSearchResults(top, () => 0);
|
|
3138
|
+
return { results: top, candidateResults: filtered, matchResult };
|
|
3139
|
+
}
|
|
3140
|
+
mergeResults(primary, secondary, limit) {
|
|
3141
|
+
const byId = /* @__PURE__ */ new Map();
|
|
3142
|
+
for (const row of primary)
|
|
3143
|
+
byId.set(row.eventId, row);
|
|
3144
|
+
for (const row of secondary) {
|
|
3145
|
+
const prev = byId.get(row.eventId);
|
|
3146
|
+
if (!prev || row.score > prev.score) {
|
|
3147
|
+
byId.set(row.eventId, row);
|
|
3148
|
+
}
|
|
3149
|
+
}
|
|
3150
|
+
return [...byId.values()].sort((a, b) => b.score - a.score).slice(0, limit);
|
|
3151
|
+
}
|
|
3152
|
+
async expandGraphHops(seeds, opts) {
|
|
3153
|
+
const byId = /* @__PURE__ */ new Map();
|
|
3154
|
+
for (const s of seeds)
|
|
3155
|
+
byId.set(s.eventId, s);
|
|
3156
|
+
let frontier = seeds.map((s) => ({ row: s, hop: 0 }));
|
|
3157
|
+
for (let hop = 1; hop <= opts.maxHops; hop += 1) {
|
|
3158
|
+
const next = [];
|
|
3159
|
+
for (const f of frontier) {
|
|
3160
|
+
const ev = await this.eventStore.getEvent(f.row.eventId);
|
|
3161
|
+
if (!ev)
|
|
3162
|
+
continue;
|
|
3163
|
+
const rel = ev.metadata?.relatedEventIds ?? [];
|
|
3164
|
+
const relatedIds = Array.isArray(rel) ? rel.filter((x) => typeof x === "string") : [];
|
|
3165
|
+
for (const rid of relatedIds) {
|
|
3166
|
+
if (byId.has(rid))
|
|
3167
|
+
continue;
|
|
3168
|
+
const target = await this.eventStore.getEvent(rid);
|
|
3169
|
+
if (!target)
|
|
3170
|
+
continue;
|
|
3171
|
+
const score = Math.max(0, f.row.score - opts.hopPenalty * hop);
|
|
3172
|
+
const row = {
|
|
3173
|
+
id: `hop-${hop}-${rid}`,
|
|
3174
|
+
eventId: target.id,
|
|
3175
|
+
content: target.content,
|
|
3176
|
+
score,
|
|
3177
|
+
sessionId: target.sessionId,
|
|
3178
|
+
eventType: target.eventType,
|
|
3179
|
+
timestamp: target.timestamp.toISOString()
|
|
3180
|
+
};
|
|
3181
|
+
byId.set(row.eventId, row);
|
|
3182
|
+
next.push({ row, hop });
|
|
3183
|
+
if (byId.size >= opts.limit)
|
|
3184
|
+
break;
|
|
2741
3185
|
}
|
|
2742
|
-
|
|
2743
|
-
|
|
2744
|
-
`;
|
|
3186
|
+
if (byId.size >= opts.limit)
|
|
3187
|
+
break;
|
|
2745
3188
|
}
|
|
3189
|
+
frontier = next;
|
|
3190
|
+
if (frontier.length === 0 || byId.size >= opts.limit)
|
|
3191
|
+
break;
|
|
2746
3192
|
}
|
|
2747
|
-
return
|
|
3193
|
+
return [...byId.values()].sort((a, b) => b.score - a.score).slice(0, opts.limit);
|
|
3194
|
+
}
|
|
3195
|
+
shouldFallback(matchResult, results) {
|
|
3196
|
+
if (results.length === 0)
|
|
3197
|
+
return true;
|
|
3198
|
+
if (matchResult.confidence === "none")
|
|
3199
|
+
return true;
|
|
3200
|
+
return false;
|
|
3201
|
+
}
|
|
3202
|
+
async buildSummaryFallback(query, topK) {
|
|
3203
|
+
const recent = await this.eventStore.getRecentEvents(Math.max(topK * 6, 20));
|
|
3204
|
+
const q = this.tokenize(query);
|
|
3205
|
+
const ranked = recent.map((e) => ({ e, overlap: this.keywordOverlap(q, this.tokenize(e.content)) })).filter((r) => r.overlap > 0).sort((a, b) => b.overlap - a.overlap).slice(0, topK).map((row, idx) => ({
|
|
3206
|
+
id: `summary-${row.e.id}`,
|
|
3207
|
+
eventId: row.e.id,
|
|
3208
|
+
content: row.e.content,
|
|
3209
|
+
score: Math.max(0.25, 0.6 - idx * 0.05),
|
|
3210
|
+
sessionId: row.e.sessionId,
|
|
3211
|
+
eventType: row.e.eventType,
|
|
3212
|
+
timestamp: row.e.timestamp.toISOString()
|
|
3213
|
+
}));
|
|
3214
|
+
return ranked;
|
|
3215
|
+
}
|
|
3216
|
+
async searchByStrategy(query, input) {
|
|
3217
|
+
const strategy = input.strategy === "auto" ? "deep" : input.strategy;
|
|
3218
|
+
if (strategy === "fast") {
|
|
3219
|
+
const keyword = await this.searchByKeyword(query, {
|
|
3220
|
+
limit: Math.max(5, input.topK * 3),
|
|
3221
|
+
sessionId: input.sessionId
|
|
3222
|
+
});
|
|
3223
|
+
return keyword;
|
|
3224
|
+
}
|
|
3225
|
+
const queryEmbedding = await this.embedder.embed(query);
|
|
3226
|
+
return this.vectorStore.search(queryEmbedding.vector, {
|
|
3227
|
+
limit: Math.max(5, input.topK * 3),
|
|
3228
|
+
minScore: input.minScore,
|
|
3229
|
+
sessionId: input.sessionId
|
|
3230
|
+
});
|
|
3231
|
+
}
|
|
3232
|
+
async searchByKeyword(query, input) {
|
|
3233
|
+
if (this.eventStore.keywordSearch) {
|
|
3234
|
+
const rows = await this.eventStore.keywordSearch(query, input.limit);
|
|
3235
|
+
const filtered2 = input.sessionId ? rows.filter((r) => r.event.sessionId === input.sessionId) : rows;
|
|
3236
|
+
return filtered2.map((row, idx) => ({
|
|
3237
|
+
id: `kw-${row.event.id}`,
|
|
3238
|
+
eventId: row.event.id,
|
|
3239
|
+
content: row.event.content,
|
|
3240
|
+
score: Math.max(0.4, 1 - idx * 0.04),
|
|
3241
|
+
sessionId: row.event.sessionId,
|
|
3242
|
+
eventType: row.event.eventType,
|
|
3243
|
+
timestamp: row.event.timestamp.toISOString()
|
|
3244
|
+
}));
|
|
3245
|
+
}
|
|
3246
|
+
const recent = await this.eventStore.getRecentEvents(input.limit * 4);
|
|
3247
|
+
const tokens = this.tokenize(query);
|
|
3248
|
+
const filtered = recent.filter((e) => input.sessionId ? e.sessionId === input.sessionId : true).map((e) => ({ e, overlap: this.keywordOverlap(tokens, this.tokenize(e.content)) })).filter((r) => r.overlap > 0).sort((a, b) => b.overlap - a.overlap).slice(0, input.limit);
|
|
3249
|
+
return filtered.map((row, idx) => ({
|
|
3250
|
+
id: `kw-fallback-${row.e.id}`,
|
|
3251
|
+
eventId: row.e.id,
|
|
3252
|
+
content: row.e.content,
|
|
3253
|
+
score: Math.max(0.3, 0.9 - idx * 0.05),
|
|
3254
|
+
sessionId: row.e.sessionId,
|
|
3255
|
+
eventType: row.e.eventType,
|
|
3256
|
+
timestamp: row.e.timestamp.toISOString()
|
|
3257
|
+
}));
|
|
3258
|
+
}
|
|
3259
|
+
rerankByKeywordOverlap(results, query, weights, decayPolicy) {
|
|
3260
|
+
const q = this.tokenize(query);
|
|
3261
|
+
const now = Date.now();
|
|
3262
|
+
const sw = Math.max(0, weights?.semantic ?? 0.7);
|
|
3263
|
+
const lw = Math.max(0, weights?.lexical ?? 0.2);
|
|
3264
|
+
const rw = Math.max(0, weights?.recency ?? 0.1);
|
|
3265
|
+
const total = sw + lw + rw || 1;
|
|
3266
|
+
const decayEnabled = decayPolicy?.enabled !== false;
|
|
3267
|
+
const decayWindow = Math.max(1, decayPolicy?.windowDays ?? 30);
|
|
3268
|
+
const decayMaxPenalty = Math.max(0, decayPolicy?.maxPenalty ?? 0.15);
|
|
3269
|
+
return [...results].map((r) => {
|
|
3270
|
+
const overlap = this.keywordOverlap(q, this.tokenize(r.content));
|
|
3271
|
+
const recencyDays = Math.max(0, (now - new Date(r.timestamp).getTime()) / (1e3 * 60 * 60 * 24));
|
|
3272
|
+
const recency = Math.max(0, 1 - recencyDays / decayWindow);
|
|
3273
|
+
let blended = (r.score * sw + overlap * lw + recency * rw) / total;
|
|
3274
|
+
if (decayEnabled && recencyDays > decayWindow && overlap < 0.5) {
|
|
3275
|
+
const ageFactor = Math.min(1, (recencyDays - decayWindow) / decayWindow);
|
|
3276
|
+
blended -= decayMaxPenalty * ageFactor;
|
|
3277
|
+
}
|
|
3278
|
+
return { ...r, score: Math.max(0, blended), semanticScore: r.score, lexicalScore: overlap, recencyScore: recency };
|
|
3279
|
+
}).sort((a, b) => b.score - a.score);
|
|
3280
|
+
}
|
|
3281
|
+
async applyScopeFilters(results, options) {
|
|
3282
|
+
const scope = options?.scope;
|
|
3283
|
+
const projectScopeMode = options?.projectScopeMode ?? "global";
|
|
3284
|
+
const allowedProjectHashes = new Set(
|
|
3285
|
+
[options?.projectHash, ...options?.allowedProjectHashes || []].filter(
|
|
3286
|
+
(value) => typeof value === "string" && value.length > 0
|
|
3287
|
+
)
|
|
3288
|
+
);
|
|
3289
|
+
if (!scope && projectScopeMode === "global")
|
|
3290
|
+
return results;
|
|
3291
|
+
const normalizedIncludes = (scope?.contentIncludes || []).map((s) => s.toLowerCase());
|
|
3292
|
+
const filtered = [];
|
|
3293
|
+
for (const result of results) {
|
|
3294
|
+
if (scope?.sessionId && result.sessionId !== scope.sessionId)
|
|
3295
|
+
continue;
|
|
3296
|
+
if (scope?.sessionIdPrefix && !result.sessionId.startsWith(scope.sessionIdPrefix))
|
|
3297
|
+
continue;
|
|
3298
|
+
if (scope?.eventTypes && scope.eventTypes.length > 0 && !scope.eventTypes.includes(result.eventType))
|
|
3299
|
+
continue;
|
|
3300
|
+
const event = await this.eventStore.getEvent(result.eventId);
|
|
3301
|
+
if (!event)
|
|
3302
|
+
continue;
|
|
3303
|
+
if (scope?.canonicalKeyPrefix && !event.canonicalKey.startsWith(scope.canonicalKeyPrefix))
|
|
3304
|
+
continue;
|
|
3305
|
+
if (normalizedIncludes.length > 0) {
|
|
3306
|
+
const lc = event.content.toLowerCase();
|
|
3307
|
+
if (!normalizedIncludes.some((needle) => lc.includes(needle)))
|
|
3308
|
+
continue;
|
|
3309
|
+
}
|
|
3310
|
+
if (scope?.metadata && !this.matchesMetadataScope(event.metadata, scope.metadata))
|
|
3311
|
+
continue;
|
|
3312
|
+
const projectHash = this.extractProjectHash(event.metadata);
|
|
3313
|
+
filtered.push({ result, projectHash });
|
|
3314
|
+
}
|
|
3315
|
+
if (projectScopeMode === "global" || allowedProjectHashes.size === 0) {
|
|
3316
|
+
return filtered.map((x) => x.result);
|
|
3317
|
+
}
|
|
3318
|
+
const projectMatched = filtered.filter((x) => x.projectHash && allowedProjectHashes.has(x.projectHash));
|
|
3319
|
+
if (projectScopeMode === "strict") {
|
|
3320
|
+
return projectMatched.map((x) => x.result);
|
|
3321
|
+
}
|
|
3322
|
+
return (projectMatched.length > 0 ? projectMatched : filtered).map((x) => x.result);
|
|
3323
|
+
}
|
|
3324
|
+
extractProjectHash(metadata) {
|
|
3325
|
+
if (!metadata || typeof metadata !== "object")
|
|
3326
|
+
return void 0;
|
|
3327
|
+
const scope = metadata.scope;
|
|
3328
|
+
if (!scope || typeof scope !== "object")
|
|
3329
|
+
return void 0;
|
|
3330
|
+
const project = scope.project;
|
|
3331
|
+
if (!project || typeof project !== "object")
|
|
3332
|
+
return void 0;
|
|
3333
|
+
const hash = project.hash;
|
|
3334
|
+
return typeof hash === "string" && hash.length > 0 ? hash : void 0;
|
|
2748
3335
|
}
|
|
2749
|
-
/**
|
|
2750
|
-
* Retrieve memories from a specific session
|
|
2751
|
-
*/
|
|
2752
3336
|
async retrieveFromSession(sessionId) {
|
|
2753
3337
|
return this.eventStore.getSessionEvents(sessionId);
|
|
2754
3338
|
}
|
|
2755
|
-
/**
|
|
2756
|
-
* Get recent memories across all sessions
|
|
2757
|
-
*/
|
|
2758
3339
|
async retrieveRecent(limit = 100) {
|
|
2759
3340
|
return this.eventStore.getRecentEvents(limit);
|
|
2760
3341
|
}
|
|
2761
|
-
/**
|
|
2762
|
-
* Enrich search results with full event data
|
|
2763
|
-
*/
|
|
2764
3342
|
async enrichResults(results, options) {
|
|
2765
3343
|
const memories = [];
|
|
2766
3344
|
for (const result of results) {
|
|
@@ -2768,27 +3346,16 @@ var Retriever = class {
|
|
|
2768
3346
|
if (!event)
|
|
2769
3347
|
continue;
|
|
2770
3348
|
if (this.graduation) {
|
|
2771
|
-
this.graduation.recordAccess(
|
|
2772
|
-
event.id,
|
|
2773
|
-
options.sessionId || "unknown",
|
|
2774
|
-
result.score
|
|
2775
|
-
);
|
|
3349
|
+
this.graduation.recordAccess(event.id, options.sessionId || "unknown", result.score);
|
|
2776
3350
|
}
|
|
2777
3351
|
let sessionContext;
|
|
2778
3352
|
if (options.includeSessionContext) {
|
|
2779
3353
|
sessionContext = await this.getSessionContext(event.sessionId, event.id);
|
|
2780
3354
|
}
|
|
2781
|
-
memories.push({
|
|
2782
|
-
event,
|
|
2783
|
-
score: result.score,
|
|
2784
|
-
sessionContext
|
|
2785
|
-
});
|
|
3355
|
+
memories.push({ event, score: result.score, sessionContext });
|
|
2786
3356
|
}
|
|
2787
3357
|
return memories;
|
|
2788
3358
|
}
|
|
2789
|
-
/**
|
|
2790
|
-
* Get surrounding context from the same session
|
|
2791
|
-
*/
|
|
2792
3359
|
async getSessionContext(sessionId, eventId) {
|
|
2793
3360
|
const sessionEvents = await this.eventStore.getSessionEvents(sessionId);
|
|
2794
3361
|
const eventIndex = sessionEvents.findIndex((e) => e.id === eventId);
|
|
@@ -2801,55 +3368,86 @@ var Retriever = class {
|
|
|
2801
3368
|
return void 0;
|
|
2802
3369
|
return contextEvents.filter((e) => e.id !== eventId).map((e) => `[${e.eventType}]: ${e.content.slice(0, 200)}...`).join("\n");
|
|
2803
3370
|
}
|
|
2804
|
-
|
|
2805
|
-
|
|
2806
|
-
|
|
3371
|
+
buildUnifiedContext(projectResult, sharedMemories) {
|
|
3372
|
+
let context = projectResult.context;
|
|
3373
|
+
if (sharedMemories.length === 0)
|
|
3374
|
+
return context;
|
|
3375
|
+
context += "\n\n## Cross-Project Knowledge\n\n";
|
|
3376
|
+
for (const memory of sharedMemories.slice(0, 3)) {
|
|
3377
|
+
context += `### ${memory.title}
|
|
3378
|
+
`;
|
|
3379
|
+
if (memory.symptoms.length > 0)
|
|
3380
|
+
context += `**Symptoms:** ${memory.symptoms.join(", ")}
|
|
3381
|
+
`;
|
|
3382
|
+
context += `**Root Cause:** ${memory.rootCause}
|
|
3383
|
+
`;
|
|
3384
|
+
context += `**Solution:** ${memory.solution}
|
|
3385
|
+
`;
|
|
3386
|
+
if (memory.technologies && memory.technologies.length > 0)
|
|
3387
|
+
context += `**Technologies:** ${memory.technologies.join(", ")}
|
|
3388
|
+
`;
|
|
3389
|
+
context += `_Confidence: ${(memory.confidence * 100).toFixed(0)}%_
|
|
3390
|
+
|
|
3391
|
+
`;
|
|
3392
|
+
}
|
|
3393
|
+
return context;
|
|
3394
|
+
}
|
|
2807
3395
|
buildContext(memories, maxTokens) {
|
|
2808
3396
|
const parts = [];
|
|
2809
3397
|
let currentTokens = 0;
|
|
2810
3398
|
for (const memory of memories) {
|
|
2811
3399
|
const memoryText = this.formatMemory(memory);
|
|
2812
3400
|
const memoryTokens = this.estimateTokens(memoryText);
|
|
2813
|
-
if (currentTokens + memoryTokens > maxTokens)
|
|
3401
|
+
if (currentTokens + memoryTokens > maxTokens)
|
|
2814
3402
|
break;
|
|
2815
|
-
}
|
|
2816
3403
|
parts.push(memoryText);
|
|
2817
3404
|
currentTokens += memoryTokens;
|
|
2818
3405
|
}
|
|
2819
|
-
if (parts.length === 0)
|
|
3406
|
+
if (parts.length === 0)
|
|
2820
3407
|
return "";
|
|
2821
|
-
}
|
|
2822
3408
|
return `## Relevant Memories
|
|
2823
3409
|
|
|
2824
3410
|
${parts.join("\n\n---\n\n")}`;
|
|
2825
3411
|
}
|
|
2826
|
-
/**
|
|
2827
|
-
* Format a single memory for context
|
|
2828
|
-
*/
|
|
2829
3412
|
formatMemory(memory) {
|
|
2830
3413
|
const { event, score, sessionContext } = memory;
|
|
2831
3414
|
const date = event.timestamp.toISOString().split("T")[0];
|
|
2832
3415
|
let text = `**${event.eventType}** (${date}, score: ${score.toFixed(2)})
|
|
2833
3416
|
${event.content}`;
|
|
2834
|
-
if (sessionContext)
|
|
3417
|
+
if (sessionContext)
|
|
2835
3418
|
text += `
|
|
2836
3419
|
|
|
2837
3420
|
_Context:_ ${sessionContext}`;
|
|
2838
|
-
}
|
|
2839
3421
|
return text;
|
|
2840
3422
|
}
|
|
2841
|
-
|
|
2842
|
-
|
|
2843
|
-
|
|
3423
|
+
matchesMetadataScope(metadata, expected) {
|
|
3424
|
+
if (!metadata)
|
|
3425
|
+
return false;
|
|
3426
|
+
return Object.entries(expected).every(([path5, value]) => {
|
|
3427
|
+
const actual = path5.split(".").reduce((acc, key) => {
|
|
3428
|
+
if (typeof acc !== "object" || acc === null)
|
|
3429
|
+
return void 0;
|
|
3430
|
+
return acc[key];
|
|
3431
|
+
}, metadata);
|
|
3432
|
+
return actual === value;
|
|
3433
|
+
});
|
|
3434
|
+
}
|
|
3435
|
+
tokenize(text) {
|
|
3436
|
+
return text.toLowerCase().replace(/[^\p{L}\p{N}\s]/gu, " ").split(/\s+/).filter((t) => t.length >= 2).slice(0, 64);
|
|
3437
|
+
}
|
|
3438
|
+
keywordOverlap(a, b) {
|
|
3439
|
+
if (a.length === 0 || b.length === 0)
|
|
3440
|
+
return 0;
|
|
3441
|
+
const bs = new Set(b);
|
|
3442
|
+
let hit = 0;
|
|
3443
|
+
for (const t of a)
|
|
3444
|
+
if (bs.has(t))
|
|
3445
|
+
hit += 1;
|
|
3446
|
+
return hit / a.length;
|
|
3447
|
+
}
|
|
2844
3448
|
estimateTokens(text) {
|
|
2845
3449
|
return Math.ceil(text.length / 4);
|
|
2846
3450
|
}
|
|
2847
|
-
/**
|
|
2848
|
-
* Get event age in days (for recency scoring)
|
|
2849
|
-
*/
|
|
2850
|
-
getEventAgeDays(eventId) {
|
|
2851
|
-
return 0;
|
|
2852
|
-
}
|
|
2853
3451
|
};
|
|
2854
3452
|
function createRetriever(eventStore, vectorStore, embedder, matcher) {
|
|
2855
3453
|
return new Retriever(eventStore, vectorStore, embedder, matcher);
|
|
@@ -4258,6 +4856,59 @@ var ConsolidatedStore = class {
|
|
|
4258
4856
|
[memoryId]
|
|
4259
4857
|
);
|
|
4260
4858
|
}
|
|
4859
|
+
/**
|
|
4860
|
+
* Create a long-term rule promoted from stable summaries
|
|
4861
|
+
*/
|
|
4862
|
+
async createRule(input) {
|
|
4863
|
+
const ruleId = randomUUID6();
|
|
4864
|
+
await dbRun(
|
|
4865
|
+
this.db,
|
|
4866
|
+
`INSERT INTO consolidated_rules
|
|
4867
|
+
(rule_id, rule, topics, source_memory_ids, source_events, confidence, created_at)
|
|
4868
|
+
VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)`,
|
|
4869
|
+
[
|
|
4870
|
+
ruleId,
|
|
4871
|
+
input.rule,
|
|
4872
|
+
JSON.stringify(input.topics),
|
|
4873
|
+
JSON.stringify(input.sourceMemoryIds),
|
|
4874
|
+
JSON.stringify(input.sourceEvents),
|
|
4875
|
+
input.confidence
|
|
4876
|
+
]
|
|
4877
|
+
);
|
|
4878
|
+
return ruleId;
|
|
4879
|
+
}
|
|
4880
|
+
async getRules(options) {
|
|
4881
|
+
const limit = options?.limit || 100;
|
|
4882
|
+
const rows = await dbAll(
|
|
4883
|
+
this.db,
|
|
4884
|
+
`SELECT * FROM consolidated_rules ORDER BY confidence DESC, created_at DESC LIMIT ?`,
|
|
4885
|
+
[limit]
|
|
4886
|
+
);
|
|
4887
|
+
return rows.map((row) => ({
|
|
4888
|
+
ruleId: row.rule_id,
|
|
4889
|
+
rule: row.rule,
|
|
4890
|
+
topics: JSON.parse(row.topics || "[]"),
|
|
4891
|
+
sourceMemoryIds: JSON.parse(row.source_memory_ids || "[]"),
|
|
4892
|
+
sourceEvents: JSON.parse(row.source_events || "[]"),
|
|
4893
|
+
confidence: Number(row.confidence ?? 0.5),
|
|
4894
|
+
createdAt: toDate(row.created_at) || /* @__PURE__ */ new Date()
|
|
4895
|
+
}));
|
|
4896
|
+
}
|
|
4897
|
+
async countRules() {
|
|
4898
|
+
const result = await dbAll(
|
|
4899
|
+
this.db,
|
|
4900
|
+
`SELECT COUNT(*) as count FROM consolidated_rules`
|
|
4901
|
+
);
|
|
4902
|
+
return result[0]?.count || 0;
|
|
4903
|
+
}
|
|
4904
|
+
async hasRuleForSourceMemory(memoryId) {
|
|
4905
|
+
const rows = await dbAll(
|
|
4906
|
+
this.db,
|
|
4907
|
+
`SELECT COUNT(*) as count FROM consolidated_rules WHERE source_memory_ids LIKE ?`,
|
|
4908
|
+
[`%"${memoryId}"%`]
|
|
4909
|
+
);
|
|
4910
|
+
return (rows[0]?.count || 0) > 0;
|
|
4911
|
+
}
|
|
4261
4912
|
/**
|
|
4262
4913
|
* Get count of consolidated memories
|
|
4263
4914
|
*/
|
|
@@ -4407,7 +5058,14 @@ var ConsolidationWorker = class {
|
|
|
4407
5058
|
* Force a consolidation run (manual trigger)
|
|
4408
5059
|
*/
|
|
4409
5060
|
async forceRun() {
|
|
4410
|
-
|
|
5061
|
+
const out = await this.consolidateWithReport();
|
|
5062
|
+
return out.consolidatedCount;
|
|
5063
|
+
}
|
|
5064
|
+
/**
|
|
5065
|
+
* Force a consolidation run and return metrics report
|
|
5066
|
+
*/
|
|
5067
|
+
async forceRunWithReport() {
|
|
5068
|
+
return this.consolidateWithReport();
|
|
4411
5069
|
}
|
|
4412
5070
|
/**
|
|
4413
5071
|
* Schedule the next consolidation check
|
|
@@ -4447,12 +5105,21 @@ var ConsolidationWorker = class {
|
|
|
4447
5105
|
* Perform consolidation
|
|
4448
5106
|
*/
|
|
4449
5107
|
async consolidate() {
|
|
5108
|
+
const out = await this.consolidateWithReport();
|
|
5109
|
+
return out.consolidatedCount;
|
|
5110
|
+
}
|
|
5111
|
+
async consolidateWithReport() {
|
|
4450
5112
|
const workingSet = await this.workingSetStore.get();
|
|
4451
5113
|
if (workingSet.recentEvents.length < 3) {
|
|
4452
|
-
return
|
|
5114
|
+
return {
|
|
5115
|
+
consolidatedCount: 0,
|
|
5116
|
+
promotedRuleCount: 0,
|
|
5117
|
+
report: this.buildCostQualityReport(workingSet.recentEvents, [], 0)
|
|
5118
|
+
};
|
|
4453
5119
|
}
|
|
4454
5120
|
const groups = this.groupByTopic(workingSet.recentEvents);
|
|
4455
5121
|
let consolidatedCount = 0;
|
|
5122
|
+
const createdMemoryIds = [];
|
|
4456
5123
|
for (const group of groups) {
|
|
4457
5124
|
if (group.events.length < 3)
|
|
4458
5125
|
continue;
|
|
@@ -4461,14 +5128,16 @@ var ConsolidationWorker = class {
|
|
|
4461
5128
|
if (alreadyConsolidated)
|
|
4462
5129
|
continue;
|
|
4463
5130
|
const summary = await this.summarize(group);
|
|
4464
|
-
await this.consolidatedStore.create({
|
|
5131
|
+
const memoryId = await this.consolidatedStore.create({
|
|
4465
5132
|
summary,
|
|
4466
5133
|
topics: group.topics,
|
|
4467
5134
|
sourceEvents: eventIds,
|
|
4468
5135
|
confidence: this.calculateConfidence(group)
|
|
4469
5136
|
});
|
|
5137
|
+
createdMemoryIds.push(memoryId);
|
|
4470
5138
|
consolidatedCount++;
|
|
4471
5139
|
}
|
|
5140
|
+
const promotedRuleCount = await this.promoteStableSummariesToRules(createdMemoryIds);
|
|
4472
5141
|
if (consolidatedCount > 0) {
|
|
4473
5142
|
const consolidatedEventIds = groups.filter((g) => g.events.length >= 3).flatMap((g) => g.events.map((e) => e.id));
|
|
4474
5143
|
const oldEventIds = consolidatedEventIds.filter((id) => {
|
|
@@ -4482,7 +5151,61 @@ var ConsolidationWorker = class {
|
|
|
4482
5151
|
await this.workingSetStore.prune(oldEventIds);
|
|
4483
5152
|
}
|
|
4484
5153
|
}
|
|
4485
|
-
|
|
5154
|
+
const report = this.buildCostQualityReport(workingSet.recentEvents, groups, consolidatedCount);
|
|
5155
|
+
return { consolidatedCount, promotedRuleCount, report };
|
|
5156
|
+
}
|
|
5157
|
+
async promoteStableSummariesToRules(memoryIds) {
|
|
5158
|
+
let promoted = 0;
|
|
5159
|
+
for (const memoryId of memoryIds) {
|
|
5160
|
+
const memory = await this.consolidatedStore.get(memoryId);
|
|
5161
|
+
if (!memory)
|
|
5162
|
+
continue;
|
|
5163
|
+
if (memory.confidence < 0.55)
|
|
5164
|
+
continue;
|
|
5165
|
+
if (memory.sourceEvents.length < 4)
|
|
5166
|
+
continue;
|
|
5167
|
+
const exists = await this.consolidatedStore.hasRuleForSourceMemory(memoryId);
|
|
5168
|
+
if (exists)
|
|
5169
|
+
continue;
|
|
5170
|
+
const rule = this.buildRuleFromSummary(memory.summary, memory.topics);
|
|
5171
|
+
if (!rule)
|
|
5172
|
+
continue;
|
|
5173
|
+
await this.consolidatedStore.createRule({
|
|
5174
|
+
rule,
|
|
5175
|
+
topics: memory.topics,
|
|
5176
|
+
sourceMemoryIds: [memory.memoryId],
|
|
5177
|
+
sourceEvents: memory.sourceEvents,
|
|
5178
|
+
confidence: Math.min(1, memory.confidence + 0.08)
|
|
5179
|
+
});
|
|
5180
|
+
promoted++;
|
|
5181
|
+
}
|
|
5182
|
+
return promoted;
|
|
5183
|
+
}
|
|
5184
|
+
buildRuleFromSummary(summary, topics) {
|
|
5185
|
+
const lines = summary.split(/\r?\n/).map((l) => l.trim()).filter(Boolean).filter((l) => !l.toLowerCase().startsWith("topics:"));
|
|
5186
|
+
const bullet = lines.find((l) => l.startsWith("- "))?.replace(/^-\s*/, "");
|
|
5187
|
+
const seed = bullet || lines[0];
|
|
5188
|
+
if (!seed || seed.length < 8)
|
|
5189
|
+
return null;
|
|
5190
|
+
const topicPrefix = topics.length > 0 ? `[${topics.slice(0, 2).join(", ")}] ` : "";
|
|
5191
|
+
return `${topicPrefix}${seed}`;
|
|
5192
|
+
}
|
|
5193
|
+
buildCostQualityReport(events, groups, consolidatedCount) {
|
|
5194
|
+
const beforeTokenEstimate = events.reduce((acc, e) => acc + this.estimateTokens(e.content), 0);
|
|
5195
|
+
const afterSummaries = groups.filter((g) => g.events.length >= 3).slice(0, Math.max(consolidatedCount, 1));
|
|
5196
|
+
const afterTokenEstimate = afterSummaries.length > 0 ? afterSummaries.reduce((acc, g) => acc + this.estimateTokens(this.ruleBasedSummary(g)), 0) : beforeTokenEstimate;
|
|
5197
|
+
const reductionRatio = beforeTokenEstimate > 0 ? Math.max(0, (beforeTokenEstimate - afterTokenEstimate) / beforeTokenEstimate) : 0;
|
|
5198
|
+
const qualityGuardPassed = consolidatedCount === 0 ? true : groups.filter((g) => g.events.length >= 3).every((g) => this.calculateConfidence(g) >= 0.55);
|
|
5199
|
+
return {
|
|
5200
|
+
beforeTokenEstimate,
|
|
5201
|
+
afterTokenEstimate,
|
|
5202
|
+
reductionRatio,
|
|
5203
|
+
qualityGuardPassed,
|
|
5204
|
+
details: `groups=${groups.length}, consolidated=${consolidatedCount}`
|
|
5205
|
+
};
|
|
5206
|
+
}
|
|
5207
|
+
estimateTokens(text) {
|
|
5208
|
+
return Math.ceil((text || "").length / 4);
|
|
4486
5209
|
}
|
|
4487
5210
|
/**
|
|
4488
5211
|
* Check if consolidation should run
|
|
@@ -5042,13 +5765,185 @@ function createGraduationWorker(eventStore, graduation, config) {
|
|
|
5042
5765
|
);
|
|
5043
5766
|
}
|
|
5044
5767
|
|
|
5768
|
+
// src/core/md-mirror.ts
|
|
5769
|
+
import * as fs3 from "node:fs";
|
|
5770
|
+
import * as path2 from "node:path";
|
|
5771
|
+
function sanitizeSegment2(input, fallback) {
|
|
5772
|
+
const v = (input || "").trim().toLowerCase().replace(/[^a-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
5773
|
+
return v || fallback;
|
|
5774
|
+
}
|
|
5775
|
+
function getAtPath(obj, dotted) {
|
|
5776
|
+
if (!obj)
|
|
5777
|
+
return void 0;
|
|
5778
|
+
return dotted.split(".").reduce((acc, key) => {
|
|
5779
|
+
if (!acc || typeof acc !== "object")
|
|
5780
|
+
return void 0;
|
|
5781
|
+
return acc[key];
|
|
5782
|
+
}, obj);
|
|
5783
|
+
}
|
|
5784
|
+
function buildMirrorPath2(rootDir, event) {
|
|
5785
|
+
const meta = event.metadata;
|
|
5786
|
+
const namespaceRaw = getAtPath(meta, "namespace") ?? getAtPath(meta, "scope.namespace") ?? event.eventType;
|
|
5787
|
+
const namespace = sanitizeSegment2(typeof namespaceRaw === "string" ? namespaceRaw : void 0, "general");
|
|
5788
|
+
const categoryRaw = getAtPath(meta, "categoryPath") ?? getAtPath(meta, "scope.categoryPath");
|
|
5789
|
+
const categoryPath = Array.isArray(categoryRaw) && categoryRaw.length > 0 ? categoryRaw.map((x) => sanitizeSegment2(typeof x === "string" ? x : void 0, "uncategorized")) : ["uncategorized"];
|
|
5790
|
+
const d = event.timestamp;
|
|
5791
|
+
const yyyy = d.getFullYear();
|
|
5792
|
+
const mm = String(d.getMonth() + 1).padStart(2, "0");
|
|
5793
|
+
const dd = String(d.getDate()).padStart(2, "0");
|
|
5794
|
+
return path2.join(rootDir, "memory", namespace, ...categoryPath, `${yyyy}-${mm}-${dd}.md`);
|
|
5795
|
+
}
|
|
5796
|
+
var MarkdownMirror2 = class {
|
|
5797
|
+
constructor(rootDir) {
|
|
5798
|
+
this.rootDir = rootDir;
|
|
5799
|
+
}
|
|
5800
|
+
async append(event, eventId) {
|
|
5801
|
+
const out = buildMirrorPath2(this.rootDir, event);
|
|
5802
|
+
fs3.mkdirSync(path2.dirname(out), { recursive: true });
|
|
5803
|
+
const lines = [
|
|
5804
|
+
"",
|
|
5805
|
+
`## ${event.timestamp.toISOString()} | ${eventId ?? "pending-id"}`,
|
|
5806
|
+
`- type: ${event.eventType}`,
|
|
5807
|
+
`- session: ${event.sessionId}`,
|
|
5808
|
+
event.content
|
|
5809
|
+
];
|
|
5810
|
+
await fs3.promises.appendFile(out, lines.join("\n"), "utf8");
|
|
5811
|
+
await this.refreshIndex();
|
|
5812
|
+
}
|
|
5813
|
+
async refreshIndex() {
|
|
5814
|
+
const memoryRoot = path2.join(this.rootDir, "memory");
|
|
5815
|
+
await fs3.promises.mkdir(memoryRoot, { recursive: true });
|
|
5816
|
+
const files = [];
|
|
5817
|
+
await this.walk(memoryRoot, files);
|
|
5818
|
+
const mdFiles = files.filter((f) => f.endsWith(".md")).map((f) => path2.relative(this.rootDir, f)).filter((rel) => rel !== path2.join("memory", "_index.md")).sort();
|
|
5819
|
+
const index = [
|
|
5820
|
+
"# Memory Index",
|
|
5821
|
+
"",
|
|
5822
|
+
"Generated automatically by MarkdownMirror.",
|
|
5823
|
+
"",
|
|
5824
|
+
...mdFiles.map((rel) => `- ${rel}`),
|
|
5825
|
+
""
|
|
5826
|
+
].join("\n");
|
|
5827
|
+
await fs3.promises.writeFile(path2.join(memoryRoot, "_index.md"), index, "utf8");
|
|
5828
|
+
}
|
|
5829
|
+
async walk(dir, out) {
|
|
5830
|
+
const entries = await fs3.promises.readdir(dir, { withFileTypes: true });
|
|
5831
|
+
for (const e of entries) {
|
|
5832
|
+
const full = path2.join(dir, e.name);
|
|
5833
|
+
if (e.isDirectory()) {
|
|
5834
|
+
await this.walk(full, out);
|
|
5835
|
+
} else {
|
|
5836
|
+
out.push(full);
|
|
5837
|
+
}
|
|
5838
|
+
}
|
|
5839
|
+
}
|
|
5840
|
+
};
|
|
5841
|
+
|
|
5842
|
+
// src/core/ingest-interceptor.ts
|
|
5843
|
+
var IngestInterceptorRegistry = class {
|
|
5844
|
+
before = [];
|
|
5845
|
+
after = [];
|
|
5846
|
+
onError = [];
|
|
5847
|
+
registerBefore(interceptor) {
|
|
5848
|
+
this.before.push(interceptor);
|
|
5849
|
+
return () => {
|
|
5850
|
+
this.before = this.before.filter((i) => i !== interceptor);
|
|
5851
|
+
};
|
|
5852
|
+
}
|
|
5853
|
+
registerAfter(interceptor) {
|
|
5854
|
+
this.after.push(interceptor);
|
|
5855
|
+
return () => {
|
|
5856
|
+
this.after = this.after.filter((i) => i !== interceptor);
|
|
5857
|
+
};
|
|
5858
|
+
}
|
|
5859
|
+
registerOnError(interceptor) {
|
|
5860
|
+
this.onError.push(interceptor);
|
|
5861
|
+
return () => {
|
|
5862
|
+
this.onError = this.onError.filter((i) => i !== interceptor);
|
|
5863
|
+
};
|
|
5864
|
+
}
|
|
5865
|
+
async run(stage, context) {
|
|
5866
|
+
const interceptors = stage === "before" ? this.before : stage === "after" ? this.after : this.onError;
|
|
5867
|
+
for (const interceptor of interceptors) {
|
|
5868
|
+
await interceptor({ ...context, stage });
|
|
5869
|
+
}
|
|
5870
|
+
}
|
|
5871
|
+
};
|
|
5872
|
+
function mergeHierarchicalMetadata(base, patch) {
|
|
5873
|
+
if (!base && !patch)
|
|
5874
|
+
return void 0;
|
|
5875
|
+
if (!base)
|
|
5876
|
+
return patch;
|
|
5877
|
+
if (!patch)
|
|
5878
|
+
return base;
|
|
5879
|
+
const result = { ...base };
|
|
5880
|
+
for (const [key, value] of Object.entries(patch)) {
|
|
5881
|
+
const current = result[key];
|
|
5882
|
+
if (typeof current === "object" && current !== null && !Array.isArray(current) && typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
5883
|
+
result[key] = mergeHierarchicalMetadata(
|
|
5884
|
+
current,
|
|
5885
|
+
value
|
|
5886
|
+
);
|
|
5887
|
+
} else {
|
|
5888
|
+
result[key] = value;
|
|
5889
|
+
}
|
|
5890
|
+
}
|
|
5891
|
+
return result;
|
|
5892
|
+
}
|
|
5893
|
+
|
|
5894
|
+
// src/core/tag-taxonomy.ts
|
|
5895
|
+
var TAG_NAMESPACES = {
|
|
5896
|
+
SYSTEM: "sys:",
|
|
5897
|
+
QUALITY: "q:",
|
|
5898
|
+
PROJECT: "proj:",
|
|
5899
|
+
TOPIC: "topic:",
|
|
5900
|
+
TEMPORAL: "t:",
|
|
5901
|
+
USER: "user:",
|
|
5902
|
+
AGENT: "agent:"
|
|
5903
|
+
};
|
|
5904
|
+
var VALID_TAG_NAMESPACES = new Set(Object.values(TAG_NAMESPACES));
|
|
5905
|
+
function parseTag(tag) {
|
|
5906
|
+
const value = (tag || "").trim();
|
|
5907
|
+
const idx = value.indexOf(":");
|
|
5908
|
+
if (idx <= 0)
|
|
5909
|
+
return { value };
|
|
5910
|
+
const namespace = `${value.slice(0, idx)}:`;
|
|
5911
|
+
const tagValue = value.slice(idx + 1);
|
|
5912
|
+
if (!tagValue)
|
|
5913
|
+
return { value };
|
|
5914
|
+
return { namespace, value: tagValue };
|
|
5915
|
+
}
|
|
5916
|
+
function validateTag(tag) {
|
|
5917
|
+
const normalized = (tag || "").trim();
|
|
5918
|
+
if (!normalized)
|
|
5919
|
+
return false;
|
|
5920
|
+
const { namespace } = parseTag(normalized);
|
|
5921
|
+
if (!namespace)
|
|
5922
|
+
return true;
|
|
5923
|
+
return VALID_TAG_NAMESPACES.has(namespace);
|
|
5924
|
+
}
|
|
5925
|
+
function normalizeTags(tags) {
|
|
5926
|
+
if (!Array.isArray(tags))
|
|
5927
|
+
return [];
|
|
5928
|
+
const dedup = /* @__PURE__ */ new Set();
|
|
5929
|
+
for (const item of tags) {
|
|
5930
|
+
if (typeof item !== "string")
|
|
5931
|
+
continue;
|
|
5932
|
+
const normalized = item.trim();
|
|
5933
|
+
if (!validateTag(normalized))
|
|
5934
|
+
continue;
|
|
5935
|
+
dedup.add(normalized);
|
|
5936
|
+
}
|
|
5937
|
+
return [...dedup];
|
|
5938
|
+
}
|
|
5939
|
+
|
|
5045
5940
|
// src/services/memory-service.ts
|
|
5046
5941
|
function normalizePath(projectPath) {
|
|
5047
|
-
const expanded = projectPath.startsWith("~") ?
|
|
5942
|
+
const expanded = projectPath.startsWith("~") ? path3.join(os.homedir(), projectPath.slice(1)) : projectPath;
|
|
5048
5943
|
try {
|
|
5049
|
-
return
|
|
5944
|
+
return fs4.realpathSync(expanded);
|
|
5050
5945
|
} catch {
|
|
5051
|
-
return
|
|
5946
|
+
return path3.resolve(expanded);
|
|
5052
5947
|
}
|
|
5053
5948
|
}
|
|
5054
5949
|
function hashProjectPath(projectPath) {
|
|
@@ -5057,14 +5952,14 @@ function hashProjectPath(projectPath) {
|
|
|
5057
5952
|
}
|
|
5058
5953
|
function getProjectStoragePath(projectPath) {
|
|
5059
5954
|
const hash = hashProjectPath(projectPath);
|
|
5060
|
-
return
|
|
5955
|
+
return path3.join(os.homedir(), ".claude-code", "memory", "projects", hash);
|
|
5061
5956
|
}
|
|
5062
|
-
var REGISTRY_PATH =
|
|
5063
|
-
var SHARED_STORAGE_PATH =
|
|
5957
|
+
var REGISTRY_PATH = path3.join(os.homedir(), ".claude-code", "memory", "session-registry.json");
|
|
5958
|
+
var SHARED_STORAGE_PATH = path3.join(os.homedir(), ".claude-code", "memory", "shared");
|
|
5064
5959
|
function loadSessionRegistry() {
|
|
5065
5960
|
try {
|
|
5066
|
-
if (
|
|
5067
|
-
const data =
|
|
5961
|
+
if (fs4.existsSync(REGISTRY_PATH)) {
|
|
5962
|
+
const data = fs4.readFileSync(REGISTRY_PATH, "utf-8");
|
|
5068
5963
|
return JSON.parse(data);
|
|
5069
5964
|
}
|
|
5070
5965
|
} catch (error) {
|
|
@@ -5090,6 +5985,7 @@ var MemoryService = class {
|
|
|
5090
5985
|
vectorWorker = null;
|
|
5091
5986
|
graduationWorker = null;
|
|
5092
5987
|
initialized = false;
|
|
5988
|
+
ingestInterceptors = new IngestInterceptorRegistry();
|
|
5093
5989
|
// Endless Mode components
|
|
5094
5990
|
workingSetStore = null;
|
|
5095
5991
|
consolidatedStore = null;
|
|
@@ -5103,20 +5999,27 @@ var MemoryService = class {
|
|
|
5103
5999
|
sharedPromoter = null;
|
|
5104
6000
|
sharedStoreConfig = null;
|
|
5105
6001
|
projectHash = null;
|
|
6002
|
+
projectPath = null;
|
|
5106
6003
|
readOnly;
|
|
5107
6004
|
lightweightMode;
|
|
6005
|
+
mdMirror;
|
|
5108
6006
|
constructor(config) {
|
|
5109
6007
|
const storagePath = this.expandPath(config.storagePath);
|
|
5110
6008
|
this.readOnly = config.readOnly ?? false;
|
|
5111
6009
|
this.lightweightMode = config.lightweightMode ?? false;
|
|
5112
|
-
|
|
5113
|
-
|
|
6010
|
+
this.mdMirror = new MarkdownMirror2(process.cwd());
|
|
6011
|
+
if (!this.readOnly && !fs4.existsSync(storagePath)) {
|
|
6012
|
+
fs4.mkdirSync(storagePath, { recursive: true });
|
|
5114
6013
|
}
|
|
5115
6014
|
this.projectHash = config.projectHash || null;
|
|
6015
|
+
this.projectPath = config.projectPath || null;
|
|
5116
6016
|
this.sharedStoreConfig = config.sharedStoreConfig ?? { enabled: true };
|
|
5117
6017
|
this.sqliteStore = new SQLiteEventStore(
|
|
5118
|
-
|
|
5119
|
-
{
|
|
6018
|
+
path3.join(storagePath, "events.sqlite"),
|
|
6019
|
+
{
|
|
6020
|
+
readonly: this.readOnly,
|
|
6021
|
+
markdownMirrorRoot: storagePath
|
|
6022
|
+
}
|
|
5120
6023
|
);
|
|
5121
6024
|
const analyticsEnabled = config.analyticsEnabled ?? this.readOnly;
|
|
5122
6025
|
if (!analyticsEnabled) {
|
|
@@ -5124,7 +6027,7 @@ var MemoryService = class {
|
|
|
5124
6027
|
} else if (this.readOnly) {
|
|
5125
6028
|
try {
|
|
5126
6029
|
this.analyticsStore = new EventStore(
|
|
5127
|
-
|
|
6030
|
+
path3.join(storagePath, "analytics.duckdb"),
|
|
5128
6031
|
{ readOnly: true }
|
|
5129
6032
|
);
|
|
5130
6033
|
} catch {
|
|
@@ -5132,11 +6035,11 @@ var MemoryService = class {
|
|
|
5132
6035
|
}
|
|
5133
6036
|
} else {
|
|
5134
6037
|
this.analyticsStore = new EventStore(
|
|
5135
|
-
|
|
6038
|
+
path3.join(storagePath, "analytics.duckdb"),
|
|
5136
6039
|
{ readOnly: false }
|
|
5137
6040
|
);
|
|
5138
6041
|
}
|
|
5139
|
-
this.vectorStore = new VectorStore(
|
|
6042
|
+
this.vectorStore = new VectorStore(path3.join(storagePath, "vectors"));
|
|
5140
6043
|
this.embedder = config.embeddingModel ? new Embedder(config.embeddingModel) : getDefaultEmbedder();
|
|
5141
6044
|
this.matcher = getDefaultMatcher();
|
|
5142
6045
|
this.retriever = createRetriever(
|
|
@@ -5146,6 +6049,7 @@ var MemoryService = class {
|
|
|
5146
6049
|
this.embedder,
|
|
5147
6050
|
this.matcher
|
|
5148
6051
|
);
|
|
6052
|
+
this.retriever.setQueryRewriter((q) => this.rewriteQueryIntent(q));
|
|
5149
6053
|
this.graduation = createGraduationPipeline(this.sqliteStore);
|
|
5150
6054
|
}
|
|
5151
6055
|
/**
|
|
@@ -5205,16 +6109,16 @@ var MemoryService = class {
|
|
|
5205
6109
|
*/
|
|
5206
6110
|
async initializeSharedStore() {
|
|
5207
6111
|
const sharedPath = this.sharedStoreConfig?.sharedStoragePath ? this.expandPath(this.sharedStoreConfig.sharedStoragePath) : SHARED_STORAGE_PATH;
|
|
5208
|
-
if (!
|
|
5209
|
-
|
|
6112
|
+
if (!fs4.existsSync(sharedPath)) {
|
|
6113
|
+
fs4.mkdirSync(sharedPath, { recursive: true });
|
|
5210
6114
|
}
|
|
5211
6115
|
this.sharedEventStore = createSharedEventStore(
|
|
5212
|
-
|
|
6116
|
+
path3.join(sharedPath, "shared.duckdb")
|
|
5213
6117
|
);
|
|
5214
6118
|
await this.sharedEventStore.initialize();
|
|
5215
6119
|
this.sharedStore = createSharedStore(this.sharedEventStore);
|
|
5216
6120
|
this.sharedVectorStore = createSharedVectorStore(
|
|
5217
|
-
|
|
6121
|
+
path3.join(sharedPath, "vectors")
|
|
5218
6122
|
);
|
|
5219
6123
|
await this.sharedVectorStore.initialize();
|
|
5220
6124
|
this.sharedPromoter = createSharedPromoter(
|
|
@@ -5225,6 +6129,86 @@ var MemoryService = class {
|
|
|
5225
6129
|
);
|
|
5226
6130
|
this.retriever.setSharedStores(this.sharedStore, this.sharedVectorStore);
|
|
5227
6131
|
}
|
|
6132
|
+
registerIngestBefore(interceptor) {
|
|
6133
|
+
return this.ingestInterceptors.registerBefore(interceptor);
|
|
6134
|
+
}
|
|
6135
|
+
registerIngestAfter(interceptor) {
|
|
6136
|
+
return this.ingestInterceptors.registerAfter(interceptor);
|
|
6137
|
+
}
|
|
6138
|
+
registerIngestOnError(interceptor) {
|
|
6139
|
+
return this.ingestInterceptors.registerOnError(interceptor);
|
|
6140
|
+
}
|
|
6141
|
+
async ingestWithInterceptors(operation, input, onSuccess) {
|
|
6142
|
+
const normalizedInput = {
|
|
6143
|
+
...input,
|
|
6144
|
+
metadata: mergeHierarchicalMetadata(
|
|
6145
|
+
{
|
|
6146
|
+
ingest: {
|
|
6147
|
+
operation,
|
|
6148
|
+
pipeline: "default",
|
|
6149
|
+
ts: (/* @__PURE__ */ new Date()).toISOString()
|
|
6150
|
+
},
|
|
6151
|
+
...this.projectHash ? {
|
|
6152
|
+
scope: {
|
|
6153
|
+
project: {
|
|
6154
|
+
hash: this.projectHash,
|
|
6155
|
+
...this.projectPath ? { path: this.projectPath } : {}
|
|
6156
|
+
}
|
|
6157
|
+
},
|
|
6158
|
+
tags: [`proj:${this.projectHash}`]
|
|
6159
|
+
} : {}
|
|
6160
|
+
},
|
|
6161
|
+
input.metadata
|
|
6162
|
+
)
|
|
6163
|
+
};
|
|
6164
|
+
if (this.projectHash && normalizedInput.metadata) {
|
|
6165
|
+
const meta = normalizedInput.metadata;
|
|
6166
|
+
const currentTags = Array.isArray(meta.tags) ? meta.tags.filter((x) => typeof x === "string") : [];
|
|
6167
|
+
const projectTag = `proj:${this.projectHash}`;
|
|
6168
|
+
if (!currentTags.includes(projectTag)) {
|
|
6169
|
+
meta.tags = [...currentTags, projectTag];
|
|
6170
|
+
}
|
|
6171
|
+
}
|
|
6172
|
+
if (normalizedInput.metadata) {
|
|
6173
|
+
const meta = normalizedInput.metadata;
|
|
6174
|
+
const normalizedTags = normalizeTags(meta.tags);
|
|
6175
|
+
if (normalizedTags.length > 0) {
|
|
6176
|
+
meta.tags = normalizedTags;
|
|
6177
|
+
}
|
|
6178
|
+
}
|
|
6179
|
+
await this.ingestInterceptors.run("before", {
|
|
6180
|
+
operation,
|
|
6181
|
+
sessionId: normalizedInput.sessionId,
|
|
6182
|
+
event: normalizedInput
|
|
6183
|
+
});
|
|
6184
|
+
try {
|
|
6185
|
+
const result = await this.sqliteStore.append(normalizedInput);
|
|
6186
|
+
if (result.success && !result.isDuplicate) {
|
|
6187
|
+
if (onSuccess) {
|
|
6188
|
+
await onSuccess(result.eventId);
|
|
6189
|
+
}
|
|
6190
|
+
try {
|
|
6191
|
+
await this.mdMirror.append(normalizedInput, result.eventId);
|
|
6192
|
+
} catch {
|
|
6193
|
+
}
|
|
6194
|
+
}
|
|
6195
|
+
await this.ingestInterceptors.run("after", {
|
|
6196
|
+
operation,
|
|
6197
|
+
sessionId: normalizedInput.sessionId,
|
|
6198
|
+
event: normalizedInput
|
|
6199
|
+
});
|
|
6200
|
+
return result;
|
|
6201
|
+
} catch (error) {
|
|
6202
|
+
const normalizedError = error instanceof Error ? error : new Error(String(error));
|
|
6203
|
+
await this.ingestInterceptors.run("error", {
|
|
6204
|
+
operation,
|
|
6205
|
+
sessionId: normalizedInput.sessionId,
|
|
6206
|
+
event: normalizedInput,
|
|
6207
|
+
error: normalizedError
|
|
6208
|
+
});
|
|
6209
|
+
throw error;
|
|
6210
|
+
}
|
|
6211
|
+
}
|
|
5228
6212
|
/**
|
|
5229
6213
|
* Start a new session
|
|
5230
6214
|
*/
|
|
@@ -5252,50 +6236,57 @@ var MemoryService = class {
|
|
|
5252
6236
|
*/
|
|
5253
6237
|
async storeUserPrompt(sessionId, content, metadata) {
|
|
5254
6238
|
await this.initialize();
|
|
5255
|
-
|
|
5256
|
-
|
|
5257
|
-
|
|
5258
|
-
|
|
5259
|
-
|
|
5260
|
-
|
|
5261
|
-
|
|
5262
|
-
|
|
5263
|
-
|
|
5264
|
-
|
|
5265
|
-
|
|
6239
|
+
return this.ingestWithInterceptors(
|
|
6240
|
+
"user_prompt",
|
|
6241
|
+
{
|
|
6242
|
+
eventType: "user_prompt",
|
|
6243
|
+
sessionId,
|
|
6244
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6245
|
+
content,
|
|
6246
|
+
metadata
|
|
6247
|
+
},
|
|
6248
|
+
async (eventId) => {
|
|
6249
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, content);
|
|
6250
|
+
}
|
|
6251
|
+
);
|
|
5266
6252
|
}
|
|
5267
6253
|
/**
|
|
5268
6254
|
* Store an agent response
|
|
5269
6255
|
*/
|
|
5270
6256
|
async storeAgentResponse(sessionId, content, metadata) {
|
|
5271
6257
|
await this.initialize();
|
|
5272
|
-
|
|
5273
|
-
|
|
5274
|
-
|
|
5275
|
-
|
|
5276
|
-
|
|
5277
|
-
|
|
5278
|
-
|
|
5279
|
-
|
|
5280
|
-
|
|
5281
|
-
|
|
5282
|
-
|
|
6258
|
+
return this.ingestWithInterceptors(
|
|
6259
|
+
"agent_response",
|
|
6260
|
+
{
|
|
6261
|
+
eventType: "agent_response",
|
|
6262
|
+
sessionId,
|
|
6263
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6264
|
+
content,
|
|
6265
|
+
metadata
|
|
6266
|
+
},
|
|
6267
|
+
async (eventId) => {
|
|
6268
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, content);
|
|
6269
|
+
}
|
|
6270
|
+
);
|
|
5283
6271
|
}
|
|
5284
6272
|
/**
|
|
5285
6273
|
* Store a session summary
|
|
5286
6274
|
*/
|
|
5287
|
-
async storeSessionSummary(sessionId, summary) {
|
|
6275
|
+
async storeSessionSummary(sessionId, summary, metadata) {
|
|
5288
6276
|
await this.initialize();
|
|
5289
|
-
|
|
5290
|
-
|
|
5291
|
-
|
|
5292
|
-
|
|
5293
|
-
|
|
5294
|
-
|
|
5295
|
-
|
|
5296
|
-
|
|
5297
|
-
|
|
5298
|
-
|
|
6277
|
+
return this.ingestWithInterceptors(
|
|
6278
|
+
"session_summary",
|
|
6279
|
+
{
|
|
6280
|
+
eventType: "session_summary",
|
|
6281
|
+
sessionId,
|
|
6282
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6283
|
+
content: summary,
|
|
6284
|
+
metadata
|
|
6285
|
+
},
|
|
6286
|
+
async (eventId) => {
|
|
6287
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, summary);
|
|
6288
|
+
}
|
|
6289
|
+
);
|
|
5299
6290
|
}
|
|
5300
6291
|
/**
|
|
5301
6292
|
* Store a tool observation
|
|
@@ -5304,40 +6295,181 @@ var MemoryService = class {
|
|
|
5304
6295
|
await this.initialize();
|
|
5305
6296
|
const content = JSON.stringify(payload);
|
|
5306
6297
|
const turnId = payload.metadata?.turnId;
|
|
5307
|
-
|
|
5308
|
-
|
|
5309
|
-
|
|
5310
|
-
|
|
5311
|
-
|
|
5312
|
-
|
|
5313
|
-
|
|
5314
|
-
|
|
5315
|
-
|
|
6298
|
+
return this.ingestWithInterceptors(
|
|
6299
|
+
"tool_observation",
|
|
6300
|
+
{
|
|
6301
|
+
eventType: "tool_observation",
|
|
6302
|
+
sessionId,
|
|
6303
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6304
|
+
content,
|
|
6305
|
+
metadata: {
|
|
6306
|
+
toolName: payload.toolName,
|
|
6307
|
+
success: payload.success,
|
|
6308
|
+
...turnId ? { turnId } : {}
|
|
6309
|
+
}
|
|
6310
|
+
},
|
|
6311
|
+
async (eventId) => {
|
|
6312
|
+
const embeddingContent = createToolObservationEmbedding(
|
|
6313
|
+
payload.toolName,
|
|
6314
|
+
payload.metadata || {},
|
|
6315
|
+
payload.success
|
|
6316
|
+
);
|
|
6317
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, embeddingContent);
|
|
5316
6318
|
}
|
|
5317
|
-
|
|
5318
|
-
if (result.success && !result.isDuplicate) {
|
|
5319
|
-
const embeddingContent = createToolObservationEmbedding(
|
|
5320
|
-
payload.toolName,
|
|
5321
|
-
payload.metadata || {},
|
|
5322
|
-
payload.success
|
|
5323
|
-
);
|
|
5324
|
-
await this.sqliteStore.enqueueForEmbedding(result.eventId, embeddingContent);
|
|
5325
|
-
}
|
|
5326
|
-
return result;
|
|
6319
|
+
);
|
|
5327
6320
|
}
|
|
5328
6321
|
/**
|
|
5329
6322
|
* Retrieve relevant memories for a query
|
|
5330
6323
|
*/
|
|
5331
6324
|
async retrieveMemories(query, options) {
|
|
5332
6325
|
await this.initialize();
|
|
6326
|
+
const rerankWeights = await this.getRerankWeights(options?.adaptiveRerank === true);
|
|
6327
|
+
let result;
|
|
5333
6328
|
if (options?.includeShared && this.sharedStore) {
|
|
5334
|
-
|
|
6329
|
+
result = await this.retriever.retrieveUnified(query, {
|
|
5335
6330
|
...options,
|
|
6331
|
+
intentRewrite: options?.intentRewrite === true,
|
|
6332
|
+
rerankWeights,
|
|
5336
6333
|
includeShared: true,
|
|
5337
|
-
projectHash: this.projectHash || void 0
|
|
6334
|
+
projectHash: this.projectHash || void 0,
|
|
6335
|
+
projectScopeMode: options?.projectScopeMode ?? (this.projectHash ? "strict" : "global"),
|
|
6336
|
+
allowedProjectHashes: options?.allowedProjectHashes
|
|
6337
|
+
});
|
|
6338
|
+
} else {
|
|
6339
|
+
result = await this.retriever.retrieve(query, {
|
|
6340
|
+
...options,
|
|
6341
|
+
intentRewrite: options?.intentRewrite === true,
|
|
6342
|
+
rerankWeights,
|
|
6343
|
+
projectHash: this.projectHash || void 0,
|
|
6344
|
+
projectScopeMode: options?.projectScopeMode ?? (this.projectHash ? "strict" : "global"),
|
|
6345
|
+
allowedProjectHashes: options?.allowedProjectHashes
|
|
5338
6346
|
});
|
|
5339
6347
|
}
|
|
5340
|
-
|
|
6348
|
+
try {
|
|
6349
|
+
const selectedEventIds = result.memories.map((m) => m.event.id);
|
|
6350
|
+
const selectedDetails = (result.selectedDebug || []).map((d) => ({
|
|
6351
|
+
eventId: d.eventId,
|
|
6352
|
+
score: d.score,
|
|
6353
|
+
semanticScore: d.semanticScore,
|
|
6354
|
+
lexicalScore: d.lexicalScore,
|
|
6355
|
+
recencyScore: d.recencyScore
|
|
6356
|
+
}));
|
|
6357
|
+
const candidateDetails = (result.candidateDebug || []).map((d) => ({
|
|
6358
|
+
eventId: d.eventId,
|
|
6359
|
+
score: d.score,
|
|
6360
|
+
semanticScore: d.semanticScore,
|
|
6361
|
+
lexicalScore: d.lexicalScore,
|
|
6362
|
+
recencyScore: d.recencyScore
|
|
6363
|
+
}));
|
|
6364
|
+
const candidateEventIds = candidateDetails.length > 0 ? candidateDetails.map((d) => d.eventId) : selectedEventIds;
|
|
6365
|
+
await this.sqliteStore.recordRetrievalTrace({
|
|
6366
|
+
sessionId: options?.sessionId,
|
|
6367
|
+
projectHash: this.projectHash || void 0,
|
|
6368
|
+
queryText: query,
|
|
6369
|
+
strategy: options?.strategy || "auto",
|
|
6370
|
+
candidateEventIds,
|
|
6371
|
+
selectedEventIds,
|
|
6372
|
+
candidateDetails,
|
|
6373
|
+
selectedDetails,
|
|
6374
|
+
confidence: result.matchResult.confidence,
|
|
6375
|
+
fallbackTrace: result.fallbackTrace || []
|
|
6376
|
+
});
|
|
6377
|
+
} catch {
|
|
6378
|
+
}
|
|
6379
|
+
return result;
|
|
6380
|
+
}
|
|
6381
|
+
getConfiguredRerankWeights() {
|
|
6382
|
+
const semantic = Number(process.env.MEMORY_RERANK_WEIGHT_SEMANTIC ?? "");
|
|
6383
|
+
const lexical = Number(process.env.MEMORY_RERANK_WEIGHT_LEXICAL ?? "");
|
|
6384
|
+
const recency = Number(process.env.MEMORY_RERANK_WEIGHT_RECENCY ?? "");
|
|
6385
|
+
const allFinite = [semantic, lexical, recency].every((v) => Number.isFinite(v));
|
|
6386
|
+
if (!allFinite)
|
|
6387
|
+
return void 0;
|
|
6388
|
+
const nonNegative = [semantic, lexical, recency].every((v) => v >= 0);
|
|
6389
|
+
const total = semantic + lexical + recency;
|
|
6390
|
+
if (!nonNegative || total <= 0)
|
|
6391
|
+
return void 0;
|
|
6392
|
+
return {
|
|
6393
|
+
semantic: semantic / total,
|
|
6394
|
+
lexical: lexical / total,
|
|
6395
|
+
recency: recency / total
|
|
6396
|
+
};
|
|
6397
|
+
}
|
|
6398
|
+
async getRerankWeights(adaptive) {
|
|
6399
|
+
const configured = this.getConfiguredRerankWeights();
|
|
6400
|
+
if (configured)
|
|
6401
|
+
return configured;
|
|
6402
|
+
if (adaptive)
|
|
6403
|
+
return this.getAdaptiveRerankWeights();
|
|
6404
|
+
return void 0;
|
|
6405
|
+
}
|
|
6406
|
+
async rewriteQueryIntent(query) {
|
|
6407
|
+
if (process.env.MEMORY_INTENT_REWRITE_ENABLED !== "1")
|
|
6408
|
+
return null;
|
|
6409
|
+
const apiUrl = process.env.COMPANY_STOCK_API_URL || process.env.COMPANY_INT_API_URL;
|
|
6410
|
+
if (!apiUrl)
|
|
6411
|
+
return null;
|
|
6412
|
+
const controller = new AbortController();
|
|
6413
|
+
const timeoutMs = Number(process.env.MEMORY_INTENT_REWRITE_TIMEOUT_MS || 5e3);
|
|
6414
|
+
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
|
6415
|
+
try {
|
|
6416
|
+
const prompt = [
|
|
6417
|
+
"Rewrite user query for memory retrieval intent expansion.",
|
|
6418
|
+
"Return plain text only, one line, no markdown.",
|
|
6419
|
+
`Query: ${query}`
|
|
6420
|
+
].join("\n");
|
|
6421
|
+
const res = await fetch(apiUrl, {
|
|
6422
|
+
method: "POST",
|
|
6423
|
+
headers: {
|
|
6424
|
+
"Content-Type": "application/json",
|
|
6425
|
+
Accept: "*/*",
|
|
6426
|
+
Origin: process.env.COMPANY_INT_ORIGIN || "http://company-int.aplusai.ai",
|
|
6427
|
+
Referer: process.env.COMPANY_INT_REFERER || "http://company-int.aplusai.ai/"
|
|
6428
|
+
},
|
|
6429
|
+
body: JSON.stringify({
|
|
6430
|
+
question: prompt,
|
|
6431
|
+
company_name: null,
|
|
6432
|
+
conversation_id: null
|
|
6433
|
+
}),
|
|
6434
|
+
signal: controller.signal
|
|
6435
|
+
});
|
|
6436
|
+
const text = (await res.text()).trim();
|
|
6437
|
+
if (!text)
|
|
6438
|
+
return null;
|
|
6439
|
+
const oneLine = text.replace(/^data:\s*/gm, "").split(/\r?\n/).map((x) => x.trim()).filter(Boolean).join(" ").slice(0, 240);
|
|
6440
|
+
if (!oneLine || oneLine.toLowerCase() === query.toLowerCase())
|
|
6441
|
+
return null;
|
|
6442
|
+
return oneLine;
|
|
6443
|
+
} catch {
|
|
6444
|
+
return null;
|
|
6445
|
+
} finally {
|
|
6446
|
+
clearTimeout(timeout);
|
|
6447
|
+
}
|
|
6448
|
+
}
|
|
6449
|
+
async getAdaptiveRerankWeights() {
|
|
6450
|
+
try {
|
|
6451
|
+
const s = await this.sqliteStore.getHelpfulnessStats();
|
|
6452
|
+
if (s.totalEvaluated < 20)
|
|
6453
|
+
return void 0;
|
|
6454
|
+
let semantic = 0.7;
|
|
6455
|
+
let lexical = 0.2;
|
|
6456
|
+
let recency = 0.1;
|
|
6457
|
+
if (s.avgScore < 0.45) {
|
|
6458
|
+
semantic -= 0.1;
|
|
6459
|
+
lexical += 0.1;
|
|
6460
|
+
} else if (s.avgScore > 0.75) {
|
|
6461
|
+
semantic += 0.05;
|
|
6462
|
+
lexical -= 0.05;
|
|
6463
|
+
}
|
|
6464
|
+
if (s.unhelpful > s.helpful) {
|
|
6465
|
+
recency += 0.05;
|
|
6466
|
+
semantic -= 0.03;
|
|
6467
|
+
lexical -= 0.02;
|
|
6468
|
+
}
|
|
6469
|
+
return { semantic, lexical, recency };
|
|
6470
|
+
} catch {
|
|
6471
|
+
return void 0;
|
|
6472
|
+
}
|
|
5341
6473
|
}
|
|
5342
6474
|
/**
|
|
5343
6475
|
* Fast keyword search using SQLite FTS5
|
|
@@ -5379,6 +6511,18 @@ var MemoryService = class {
|
|
|
5379
6511
|
/**
|
|
5380
6512
|
* Get memory statistics
|
|
5381
6513
|
*/
|
|
6514
|
+
async getOutboxStats() {
|
|
6515
|
+
await this.initialize();
|
|
6516
|
+
return this.sqliteStore.getOutboxStats();
|
|
6517
|
+
}
|
|
6518
|
+
async getRetrievalTraceStats() {
|
|
6519
|
+
await this.initialize();
|
|
6520
|
+
return this.sqliteStore.getRetrievalTraceStats();
|
|
6521
|
+
}
|
|
6522
|
+
async getRecentRetrievalTraces(limit = 50) {
|
|
6523
|
+
await this.initialize();
|
|
6524
|
+
return this.sqliteStore.getRecentRetrievalTraces(limit);
|
|
6525
|
+
}
|
|
5382
6526
|
async getStats() {
|
|
5383
6527
|
await this.initialize();
|
|
5384
6528
|
const recentEvents = await this.sqliteStore.getRecentEvents(1e4);
|
|
@@ -5869,7 +7013,7 @@ var MemoryService = class {
|
|
|
5869
7013
|
*/
|
|
5870
7014
|
expandPath(p) {
|
|
5871
7015
|
if (p.startsWith("~")) {
|
|
5872
|
-
return
|
|
7016
|
+
return path3.join(os.homedir(), p.slice(1));
|
|
5873
7017
|
}
|
|
5874
7018
|
return p;
|
|
5875
7019
|
}
|
|
@@ -5879,10 +7023,11 @@ function getLightweightMemoryService(sessionId) {
|
|
|
5879
7023
|
const projectInfo = getSessionProject(sessionId);
|
|
5880
7024
|
const key = projectInfo ? `lightweight_${projectInfo.projectHash}` : "lightweight_global";
|
|
5881
7025
|
if (!serviceCache.has(key)) {
|
|
5882
|
-
const storagePath = projectInfo ? getProjectStoragePath(projectInfo.projectPath) :
|
|
7026
|
+
const storagePath = projectInfo ? getProjectStoragePath(projectInfo.projectPath) : path3.join(os.homedir(), ".claude-code", "memory");
|
|
5883
7027
|
serviceCache.set(key, new MemoryService({
|
|
5884
7028
|
storagePath,
|
|
5885
7029
|
projectHash: projectInfo?.projectHash,
|
|
7030
|
+
projectPath: projectInfo?.projectPath,
|
|
5886
7031
|
lightweightMode: true,
|
|
5887
7032
|
// Skip embedder/vector/workers
|
|
5888
7033
|
analyticsEnabled: false,
|
|
@@ -6092,20 +7237,20 @@ function truncateOutput(output, options) {
|
|
|
6092
7237
|
}
|
|
6093
7238
|
|
|
6094
7239
|
// src/core/turn-state.ts
|
|
6095
|
-
import * as
|
|
6096
|
-
import * as
|
|
7240
|
+
import * as fs5 from "fs";
|
|
7241
|
+
import * as path4 from "path";
|
|
6097
7242
|
import * as os2 from "os";
|
|
6098
|
-
var TURN_STATE_DIR =
|
|
7243
|
+
var TURN_STATE_DIR = path4.join(os2.homedir(), ".claude-code", "memory");
|
|
6099
7244
|
function getStatePath(sessionId) {
|
|
6100
|
-
return
|
|
7245
|
+
return path4.join(TURN_STATE_DIR, `.turn-state-${sessionId}.json`);
|
|
6101
7246
|
}
|
|
6102
7247
|
function readTurnState(sessionId) {
|
|
6103
7248
|
try {
|
|
6104
7249
|
const filePath = getStatePath(sessionId);
|
|
6105
|
-
if (!
|
|
7250
|
+
if (!fs5.existsSync(filePath)) {
|
|
6106
7251
|
return null;
|
|
6107
7252
|
}
|
|
6108
|
-
const data =
|
|
7253
|
+
const data = fs5.readFileSync(filePath, "utf-8");
|
|
6109
7254
|
const state = JSON.parse(data);
|
|
6110
7255
|
if (state.sessionId !== sessionId) {
|
|
6111
7256
|
return null;
|
|
@@ -6127,8 +7272,8 @@ function readTurnState(sessionId) {
|
|
|
6127
7272
|
function clearTurnState(sessionId) {
|
|
6128
7273
|
try {
|
|
6129
7274
|
const filePath = getStatePath(sessionId);
|
|
6130
|
-
if (
|
|
6131
|
-
|
|
7275
|
+
if (fs5.existsSync(filePath)) {
|
|
7276
|
+
fs5.unlinkSync(filePath);
|
|
6132
7277
|
}
|
|
6133
7278
|
} catch (error) {
|
|
6134
7279
|
if (process.env.CLAUDE_MEMORY_DEBUG) {
|