claude-memory-layer 1.0.11 → 1.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +60 -0
- package/README.md +166 -2
- package/bootstrap-kb/decisions/decisions.md +244 -0
- package/bootstrap-kb/glossary/glossary.md +46 -0
- package/bootstrap-kb/modules/.claude-plugin.md +22 -0
- package/bootstrap-kb/modules/agents.md.md +15 -0
- package/bootstrap-kb/modules/claude.md.md +15 -0
- package/bootstrap-kb/modules/context.md.md +15 -0
- package/bootstrap-kb/modules/docs.md +18 -0
- package/bootstrap-kb/modules/handoff.md.md +15 -0
- package/bootstrap-kb/modules/package-lock.json.md +15 -0
- package/bootstrap-kb/modules/package.json.md +15 -0
- package/bootstrap-kb/modules/plan.md.md +15 -0
- package/bootstrap-kb/modules/readme.md.md +15 -0
- package/bootstrap-kb/modules/scripts.md +26 -0
- package/bootstrap-kb/modules/spec.md.md +15 -0
- package/bootstrap-kb/modules/specs.md +20 -0
- package/bootstrap-kb/modules/src.md +51 -0
- package/bootstrap-kb/modules/tests.md +42 -0
- package/bootstrap-kb/modules/tsconfig.json.md +15 -0
- package/bootstrap-kb/modules/vitest.config.ts.md +15 -0
- package/bootstrap-kb/overview/overview.md +40 -0
- package/bootstrap-kb/sources/manifest.json +950 -0
- package/bootstrap-kb/sources/manifest.md +227 -0
- package/bootstrap-kb/timeline/timeline.md +57 -0
- package/d.sh +3 -0
- package/deploy.sh +3 -0
- package/dist/cli/index.js +2389 -286
- package/dist/cli/index.js.map +4 -4
- package/dist/core/index.js +1017 -132
- package/dist/core/index.js.map +4 -4
- package/dist/hooks/post-tool-use.js +1347 -202
- package/dist/hooks/post-tool-use.js.map +4 -4
- package/dist/hooks/session-end.js +1339 -194
- package/dist/hooks/session-end.js.map +4 -4
- package/dist/hooks/session-start.js +1343 -198
- package/dist/hooks/session-start.js.map +4 -4
- package/dist/hooks/stop.js +1351 -206
- package/dist/hooks/stop.js.map +4 -4
- package/dist/hooks/user-prompt-submit.js +1347 -202
- package/dist/hooks/user-prompt-submit.js.map +4 -4
- package/dist/server/api/index.js +1436 -211
- package/dist/server/api/index.js.map +4 -4
- package/dist/server/index.js +1445 -220
- package/dist/server/index.js.map +4 -4
- package/dist/services/memory-service.js +1345 -199
- package/dist/services/memory-service.js.map +4 -4
- package/dist/ui/app.js +69 -2
- package/dist/ui/index.html +8 -0
- package/docs/MCP_MEMORY_SERVICE_COMPARATIVE_REVIEW.md +271 -0
- package/docs/MEMU_ADOPTION.md +40 -0
- package/memory/.claude-plugin/commands/2026-02-25.md +263 -0
- package/memory/_index.md +405 -0
- package/memory/default/uncategorized/2026-02-25.md +4839 -0
- package/memory/specs/20260207-dashboard-upgrade/2026-02-25.md +142 -0
- package/memory/specs/citations-system/2026-02-25.md +1121 -0
- package/memory/specs/endless-mode/2026-02-25.md +1392 -0
- package/memory/specs/entity-edge-model/2026-02-25.md +1263 -0
- package/memory/specs/evidence-aligner-v2/2026-02-25.md +1028 -0
- package/memory/specs/mcp-desktop-integration/2026-02-25.md +1334 -0
- package/memory/specs/post-tool-use-hook/2026-02-25.md +1164 -0
- package/memory/specs/private-tags/2026-02-25.md +1057 -0
- package/memory/specs/progressive-disclosure/2026-02-25.md +1436 -0
- package/memory/specs/task-entity-system/2026-02-25.md +924 -0
- package/memory/specs/vector-outbox-v2/2026-02-25.md +1510 -0
- package/memory/specs/web-viewer-ui/2026-02-25.md +1709 -0
- package/package.json +2 -1
- package/scripts/build.ts +6 -0
- package/src/cli/index.ts +281 -2
- package/src/core/consolidated-store.ts +63 -1
- package/src/core/consolidation-worker.ts +115 -6
- package/src/core/event-store.ts +14 -0
- package/src/core/index.ts +1 -0
- package/src/core/ingest-interceptor.ts +80 -0
- package/src/core/markdown-mirror.ts +70 -0
- package/src/core/md-mirror.ts +92 -0
- package/src/core/mongo-sync-config.ts +165 -0
- package/src/core/mongo-sync-worker.ts +381 -0
- package/src/core/retriever.ts +540 -150
- package/src/core/sqlite-event-store.ts +350 -1
- package/src/core/tag-taxonomy.ts +51 -0
- package/src/core/types.ts +28 -0
- package/src/server/api/health.ts +53 -0
- package/src/server/api/index.ts +3 -1
- package/src/server/api/stats.ts +46 -1
- package/src/services/bootstrap-organizer.ts +443 -0
- package/src/services/codex-session-history-importer.ts +474 -0
- package/src/services/memory-service.ts +373 -68
- package/src/ui/app.js +69 -2
- package/src/ui/index.html +8 -0
- package/tests/bootstrap-organizer.test.ts +111 -0
- package/tests/consolidation-worker.test.ts +75 -0
- package/tests/ingest-interceptor.test.ts +38 -0
- package/tests/markdown-mirror.test.ts +85 -0
- package/tests/md-mirror.test.ts +50 -0
- package/tests/retriever-fallback-chain.test.ts +223 -0
- package/tests/retriever-strategy-scope.test.ts +97 -0
- package/tests/retriever.memu-adoption.test.ts +122 -0
- package/tests/sqlite-event-store-replication.test.ts +92 -0
|
@@ -7,9 +7,9 @@ const __filename = fileURLToPath(import.meta.url);
|
|
|
7
7
|
const __dirname = dirname(__filename);
|
|
8
8
|
|
|
9
9
|
// src/services/memory-service.ts
|
|
10
|
-
import * as
|
|
10
|
+
import * as path3 from "path";
|
|
11
11
|
import * as os from "os";
|
|
12
|
-
import * as
|
|
12
|
+
import * as fs4 from "fs";
|
|
13
13
|
import * as crypto2 from "crypto";
|
|
14
14
|
|
|
15
15
|
// src/core/event-store.ts
|
|
@@ -67,11 +67,11 @@ function toDate(value) {
|
|
|
67
67
|
return new Date(value);
|
|
68
68
|
return new Date(String(value));
|
|
69
69
|
}
|
|
70
|
-
function createDatabase(
|
|
70
|
+
function createDatabase(path4, options) {
|
|
71
71
|
if (options?.readOnly) {
|
|
72
|
-
return new duckdb.Database(
|
|
72
|
+
return new duckdb.Database(path4, { access_mode: "READ_ONLY" });
|
|
73
73
|
}
|
|
74
|
-
return new duckdb.Database(
|
|
74
|
+
return new duckdb.Database(path4);
|
|
75
75
|
}
|
|
76
76
|
function dbRun(db, sql, params = []) {
|
|
77
77
|
return new Promise((resolve2, reject) => {
|
|
@@ -335,6 +335,17 @@ var EventStore = class {
|
|
|
335
335
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
336
336
|
)
|
|
337
337
|
`);
|
|
338
|
+
await dbRun(this.db, `
|
|
339
|
+
CREATE TABLE IF NOT EXISTS consolidated_rules (
|
|
340
|
+
rule_id VARCHAR PRIMARY KEY,
|
|
341
|
+
rule TEXT NOT NULL,
|
|
342
|
+
topics JSON,
|
|
343
|
+
source_memory_ids JSON,
|
|
344
|
+
source_events JSON,
|
|
345
|
+
confidence FLOAT DEFAULT 0.5,
|
|
346
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
347
|
+
)
|
|
348
|
+
`);
|
|
338
349
|
await dbRun(this.db, `
|
|
339
350
|
CREATE TABLE IF NOT EXISTS endless_config (
|
|
340
351
|
key VARCHAR PRIMARY KEY,
|
|
@@ -354,6 +365,7 @@ var EventStore = class {
|
|
|
354
365
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_working_set_expires ON working_set(expires_at)`);
|
|
355
366
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_working_set_relevance ON working_set(relevance_score DESC)`);
|
|
356
367
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_consolidated_confidence ON consolidated_memories(confidence DESC)`);
|
|
368
|
+
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_consolidated_rules_confidence ON consolidated_rules(confidence DESC)`);
|
|
357
369
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_continuity_created ON continuity_log(created_at)`);
|
|
358
370
|
this.initialized = true;
|
|
359
371
|
}
|
|
@@ -743,12 +755,12 @@ import { randomUUID as randomUUID2 } from "crypto";
|
|
|
743
755
|
import Database from "better-sqlite3";
|
|
744
756
|
import * as fs from "fs";
|
|
745
757
|
import * as nodePath from "path";
|
|
746
|
-
function createSQLiteDatabase(
|
|
747
|
-
const dir = nodePath.dirname(
|
|
758
|
+
function createSQLiteDatabase(path4, options) {
|
|
759
|
+
const dir = nodePath.dirname(path4);
|
|
748
760
|
if (!fs.existsSync(dir)) {
|
|
749
761
|
fs.mkdirSync(dir, { recursive: true });
|
|
750
762
|
}
|
|
751
|
-
const db = new Database(
|
|
763
|
+
const db = new Database(path4, {
|
|
752
764
|
readonly: options?.readonly ?? false
|
|
753
765
|
});
|
|
754
766
|
if (!options?.readonly && (options?.walMode ?? true)) {
|
|
@@ -789,6 +801,64 @@ function toSQLiteTimestamp(date) {
|
|
|
789
801
|
return date.toISOString();
|
|
790
802
|
}
|
|
791
803
|
|
|
804
|
+
// src/core/markdown-mirror.ts
|
|
805
|
+
import * as fs2 from "fs/promises";
|
|
806
|
+
import * as path from "path";
|
|
807
|
+
var DEFAULT_NAMESPACE = "default";
|
|
808
|
+
var DEFAULT_CATEGORY = "uncategorized";
|
|
809
|
+
function sanitizeSegment(input, fallback) {
|
|
810
|
+
const raw = String(input ?? "").trim().toLowerCase();
|
|
811
|
+
const safe = raw.normalize("NFKD").replace(/[^a-z0-9_-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
812
|
+
if (!safe || safe === "." || safe === "..")
|
|
813
|
+
return fallback;
|
|
814
|
+
return safe;
|
|
815
|
+
}
|
|
816
|
+
function getCategorySegments(metadata, eventType) {
|
|
817
|
+
const raw = metadata?.categoryPath;
|
|
818
|
+
if (Array.isArray(raw) && raw.length > 0) {
|
|
819
|
+
return raw.map((s) => sanitizeSegment(s, DEFAULT_CATEGORY));
|
|
820
|
+
}
|
|
821
|
+
const single = metadata?.category;
|
|
822
|
+
if (typeof single === "string" && single.trim()) {
|
|
823
|
+
return [sanitizeSegment(single, DEFAULT_CATEGORY)];
|
|
824
|
+
}
|
|
825
|
+
return [sanitizeSegment(eventType, DEFAULT_CATEGORY)];
|
|
826
|
+
}
|
|
827
|
+
function buildMirrorPath(rootDir, event) {
|
|
828
|
+
const metadata = event.metadata;
|
|
829
|
+
const namespace = sanitizeSegment(metadata?.namespace, DEFAULT_NAMESPACE);
|
|
830
|
+
const categories = getCategorySegments(metadata, event.eventType);
|
|
831
|
+
const d = event.timestamp;
|
|
832
|
+
const yyyy = d.getFullYear();
|
|
833
|
+
const mm = String(d.getMonth() + 1).padStart(2, "0");
|
|
834
|
+
const dd = String(d.getDate()).padStart(2, "0");
|
|
835
|
+
return path.join(rootDir, "memory", namespace, ...categories, `${yyyy}-${mm}-${dd}.md`);
|
|
836
|
+
}
|
|
837
|
+
function formatMirrorEntry(event) {
|
|
838
|
+
const category = Array.isArray(event.metadata?.categoryPath) ? event.metadata.categoryPath.join("/") : String(event.metadata?.category ?? event.eventType);
|
|
839
|
+
return [
|
|
840
|
+
"",
|
|
841
|
+
`- ts: ${event.timestamp.toISOString()}`,
|
|
842
|
+
` id: ${event.id}`,
|
|
843
|
+
` type: ${event.eventType}`,
|
|
844
|
+
` session: ${event.sessionId}`,
|
|
845
|
+
` category: ${category}`,
|
|
846
|
+
" content: |",
|
|
847
|
+
...event.content.split("\n").map((line) => ` ${line}`)
|
|
848
|
+
].join("\n") + "\n";
|
|
849
|
+
}
|
|
850
|
+
var MarkdownMirror = class {
|
|
851
|
+
constructor(rootDir) {
|
|
852
|
+
this.rootDir = rootDir;
|
|
853
|
+
}
|
|
854
|
+
async append(event) {
|
|
855
|
+
const outPath = buildMirrorPath(this.rootDir, event);
|
|
856
|
+
await fs2.mkdir(path.dirname(outPath), { recursive: true });
|
|
857
|
+
await fs2.appendFile(outPath, formatMirrorEntry(event), "utf8");
|
|
858
|
+
return outPath;
|
|
859
|
+
}
|
|
860
|
+
};
|
|
861
|
+
|
|
792
862
|
// src/core/sqlite-event-store.ts
|
|
793
863
|
var SQLiteEventStore = class {
|
|
794
864
|
constructor(dbPath, options) {
|
|
@@ -798,10 +868,12 @@ var SQLiteEventStore = class {
|
|
|
798
868
|
readonly: this.readOnly,
|
|
799
869
|
walMode: !this.readOnly
|
|
800
870
|
});
|
|
871
|
+
this.markdownMirror = this.readOnly || !options?.markdownMirrorRoot ? null : new MarkdownMirror(options.markdownMirrorRoot);
|
|
801
872
|
}
|
|
802
873
|
db;
|
|
803
874
|
initialized = false;
|
|
804
875
|
readOnly;
|
|
876
|
+
markdownMirror;
|
|
805
877
|
/**
|
|
806
878
|
* Initialize database schema
|
|
807
879
|
*/
|
|
@@ -1008,6 +1080,17 @@ var SQLiteEventStore = class {
|
|
|
1008
1080
|
created_at TEXT DEFAULT (datetime('now'))
|
|
1009
1081
|
);
|
|
1010
1082
|
|
|
1083
|
+
-- Consolidated Rules table (long-term stable memory)
|
|
1084
|
+
CREATE TABLE IF NOT EXISTS consolidated_rules (
|
|
1085
|
+
rule_id TEXT PRIMARY KEY,
|
|
1086
|
+
rule TEXT NOT NULL,
|
|
1087
|
+
topics TEXT,
|
|
1088
|
+
source_memory_ids TEXT,
|
|
1089
|
+
source_events TEXT,
|
|
1090
|
+
confidence REAL DEFAULT 0.5,
|
|
1091
|
+
created_at TEXT DEFAULT (datetime('now'))
|
|
1092
|
+
);
|
|
1093
|
+
|
|
1011
1094
|
-- Endless Mode Config table
|
|
1012
1095
|
CREATE TABLE IF NOT EXISTS endless_config (
|
|
1013
1096
|
key TEXT PRIMARY KEY,
|
|
@@ -1032,6 +1115,24 @@ var SQLiteEventStore = class {
|
|
|
1032
1115
|
measured_at TEXT
|
|
1033
1116
|
);
|
|
1034
1117
|
|
|
1118
|
+
-- Retrieval trace log (query -> candidates -> selected for context)
|
|
1119
|
+
CREATE TABLE IF NOT EXISTS retrieval_traces (
|
|
1120
|
+
trace_id TEXT PRIMARY KEY,
|
|
1121
|
+
session_id TEXT,
|
|
1122
|
+
project_hash TEXT,
|
|
1123
|
+
query_text TEXT NOT NULL,
|
|
1124
|
+
strategy TEXT,
|
|
1125
|
+
candidate_event_ids TEXT,
|
|
1126
|
+
selected_event_ids TEXT,
|
|
1127
|
+
candidate_details_json TEXT,
|
|
1128
|
+
selected_details_json TEXT,
|
|
1129
|
+
candidate_count INTEGER DEFAULT 0,
|
|
1130
|
+
selected_count INTEGER DEFAULT 0,
|
|
1131
|
+
confidence TEXT,
|
|
1132
|
+
fallback_trace TEXT,
|
|
1133
|
+
created_at TEXT DEFAULT (datetime('now'))
|
|
1134
|
+
);
|
|
1135
|
+
|
|
1035
1136
|
-- Sync position tracking (for SQLite -> DuckDB sync)
|
|
1036
1137
|
CREATE TABLE IF NOT EXISTS sync_positions (
|
|
1037
1138
|
target_name TEXT PRIMARY KEY,
|
|
@@ -1056,10 +1157,14 @@ var SQLiteEventStore = class {
|
|
|
1056
1157
|
CREATE INDEX IF NOT EXISTS idx_working_set_relevance ON working_set(relevance_score);
|
|
1057
1158
|
CREATE INDEX IF NOT EXISTS idx_consolidated_confidence ON consolidated_memories(confidence);
|
|
1058
1159
|
CREATE INDEX IF NOT EXISTS idx_continuity_created ON continuity_log(created_at);
|
|
1160
|
+
CREATE INDEX IF NOT EXISTS idx_consolidated_rules_confidence ON consolidated_rules(confidence);
|
|
1059
1161
|
CREATE INDEX IF NOT EXISTS idx_embedding_outbox_status ON embedding_outbox(status);
|
|
1060
1162
|
CREATE INDEX IF NOT EXISTS idx_helpfulness_event ON memory_helpfulness(event_id);
|
|
1061
1163
|
CREATE INDEX IF NOT EXISTS idx_helpfulness_session ON memory_helpfulness(session_id);
|
|
1062
1164
|
CREATE INDEX IF NOT EXISTS idx_helpfulness_score ON memory_helpfulness(helpfulness_score DESC);
|
|
1165
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_created_at ON retrieval_traces(created_at DESC);
|
|
1166
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_project_hash ON retrieval_traces(project_hash);
|
|
1167
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_session_id ON retrieval_traces(session_id);
|
|
1063
1168
|
|
|
1064
1169
|
-- FTS5 Full-Text Search for fast keyword search
|
|
1065
1170
|
CREATE VIRTUAL TABLE IF NOT EXISTS events_fts USING fts5(
|
|
@@ -1083,6 +1188,14 @@ var SQLiteEventStore = class {
|
|
|
1083
1188
|
INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
|
|
1084
1189
|
END;
|
|
1085
1190
|
`);
|
|
1191
|
+
try {
|
|
1192
|
+
sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN selected_details_json TEXT;`);
|
|
1193
|
+
} catch {
|
|
1194
|
+
}
|
|
1195
|
+
try {
|
|
1196
|
+
sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN candidate_details_json TEXT;`);
|
|
1197
|
+
} catch {
|
|
1198
|
+
}
|
|
1086
1199
|
const tableInfo = sqliteAll(this.db, "PRAGMA table_info(events)", []);
|
|
1087
1200
|
const columnNames = tableInfo.map((col) => col.name);
|
|
1088
1201
|
if (!columnNames.includes("access_count")) {
|
|
@@ -1182,6 +1295,21 @@ var SQLiteEventStore = class {
|
|
|
1182
1295
|
insertLevel.run(id);
|
|
1183
1296
|
});
|
|
1184
1297
|
transaction();
|
|
1298
|
+
if (this.markdownMirror) {
|
|
1299
|
+
const event = {
|
|
1300
|
+
id,
|
|
1301
|
+
eventType: input.eventType,
|
|
1302
|
+
sessionId: input.sessionId,
|
|
1303
|
+
timestamp: input.timestamp,
|
|
1304
|
+
content: input.content,
|
|
1305
|
+
canonicalKey,
|
|
1306
|
+
dedupeKey,
|
|
1307
|
+
metadata
|
|
1308
|
+
};
|
|
1309
|
+
this.markdownMirror.append(event).catch((err) => {
|
|
1310
|
+
console.warn("[SQLiteEventStore] markdown mirror append failed:", err);
|
|
1311
|
+
});
|
|
1312
|
+
}
|
|
1185
1313
|
return { success: true, eventId: id, isDuplicate: false };
|
|
1186
1314
|
} catch (error) {
|
|
1187
1315
|
return {
|
|
@@ -1240,6 +1368,92 @@ var SQLiteEventStore = class {
|
|
|
1240
1368
|
);
|
|
1241
1369
|
return rows.map(this.rowToEvent);
|
|
1242
1370
|
}
|
|
1371
|
+
/**
|
|
1372
|
+
* Get events since a SQLite rowid (for robust incremental replication).
|
|
1373
|
+
* Rowid is monotonic for append-only tables, independent of client timestamps.
|
|
1374
|
+
*/
|
|
1375
|
+
async getEventsSinceRowid(lastRowid, limit = 1e3) {
|
|
1376
|
+
await this.initialize();
|
|
1377
|
+
const rows = sqliteAll(
|
|
1378
|
+
this.db,
|
|
1379
|
+
`SELECT rowid as _rowid, * FROM events WHERE rowid > ? ORDER BY rowid ASC LIMIT ?`,
|
|
1380
|
+
[lastRowid, limit]
|
|
1381
|
+
);
|
|
1382
|
+
return rows.map((row) => ({
|
|
1383
|
+
rowid: row._rowid,
|
|
1384
|
+
event: this.rowToEvent(row)
|
|
1385
|
+
}));
|
|
1386
|
+
}
|
|
1387
|
+
/**
|
|
1388
|
+
* Import events with fixed IDs (used for cross-machine replication).
|
|
1389
|
+
* Idempotent: skips if event id or dedupeKey already exists.
|
|
1390
|
+
*
|
|
1391
|
+
* NOTE: This bypasses the append() id generation to preserve stable IDs.
|
|
1392
|
+
*/
|
|
1393
|
+
async importEvents(events) {
|
|
1394
|
+
if (events.length === 0)
|
|
1395
|
+
return { inserted: 0, skipped: 0 };
|
|
1396
|
+
if (this.readOnly)
|
|
1397
|
+
return { inserted: 0, skipped: events.length };
|
|
1398
|
+
await this.initialize();
|
|
1399
|
+
const getById = this.db.prepare(`SELECT id FROM events WHERE id = ?`);
|
|
1400
|
+
const getByDedupe = this.db.prepare(`SELECT event_id FROM event_dedup WHERE dedupe_key = ?`);
|
|
1401
|
+
const insertEvent = this.db.prepare(`
|
|
1402
|
+
INSERT INTO events (id, event_type, session_id, timestamp, content, canonical_key, dedupe_key, metadata, turn_id)
|
|
1403
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1404
|
+
`);
|
|
1405
|
+
const insertDedup = this.db.prepare(`
|
|
1406
|
+
INSERT INTO event_dedup (dedupe_key, event_id) VALUES (?, ?)
|
|
1407
|
+
`);
|
|
1408
|
+
const insertLevel = this.db.prepare(`
|
|
1409
|
+
INSERT INTO memory_levels (event_id, level) VALUES (?, 'L0')
|
|
1410
|
+
`);
|
|
1411
|
+
let inserted = 0;
|
|
1412
|
+
let skipped = 0;
|
|
1413
|
+
const insertedEvents = [];
|
|
1414
|
+
const tx = this.db.transaction((batch) => {
|
|
1415
|
+
for (const ev of batch) {
|
|
1416
|
+
const existingById = getById.get(ev.id);
|
|
1417
|
+
if (existingById) {
|
|
1418
|
+
skipped++;
|
|
1419
|
+
continue;
|
|
1420
|
+
}
|
|
1421
|
+
const canonicalKey = ev.canonicalKey || makeCanonicalKey(ev.content);
|
|
1422
|
+
const dedupeKey = ev.dedupeKey || makeDedupeKey(ev.content, ev.sessionId);
|
|
1423
|
+
const existingByDedupe = getByDedupe.get(dedupeKey);
|
|
1424
|
+
if (existingByDedupe) {
|
|
1425
|
+
skipped++;
|
|
1426
|
+
continue;
|
|
1427
|
+
}
|
|
1428
|
+
const metadata = ev.metadata || {};
|
|
1429
|
+
const turnId = metadata.turnId;
|
|
1430
|
+
insertEvent.run(
|
|
1431
|
+
ev.id,
|
|
1432
|
+
ev.eventType,
|
|
1433
|
+
ev.sessionId,
|
|
1434
|
+
toSQLiteTimestamp(ev.timestamp),
|
|
1435
|
+
ev.content,
|
|
1436
|
+
canonicalKey,
|
|
1437
|
+
dedupeKey,
|
|
1438
|
+
JSON.stringify(metadata),
|
|
1439
|
+
turnId ?? null
|
|
1440
|
+
);
|
|
1441
|
+
insertDedup.run(dedupeKey, ev.id);
|
|
1442
|
+
insertLevel.run(ev.id);
|
|
1443
|
+
inserted++;
|
|
1444
|
+
insertedEvents.push(ev);
|
|
1445
|
+
}
|
|
1446
|
+
});
|
|
1447
|
+
tx(events);
|
|
1448
|
+
if (this.markdownMirror && insertedEvents.length > 0) {
|
|
1449
|
+
for (const ev of insertedEvents) {
|
|
1450
|
+
this.markdownMirror.append(ev).catch((err) => {
|
|
1451
|
+
console.warn("[SQLiteEventStore] markdown mirror append failed:", err);
|
|
1452
|
+
});
|
|
1453
|
+
}
|
|
1454
|
+
}
|
|
1455
|
+
return { inserted, skipped };
|
|
1456
|
+
}
|
|
1243
1457
|
/**
|
|
1244
1458
|
* Create or update session
|
|
1245
1459
|
*/
|
|
@@ -1402,6 +1616,35 @@ var SQLiteEventStore = class {
|
|
|
1402
1616
|
[error, ...ids]
|
|
1403
1617
|
);
|
|
1404
1618
|
}
|
|
1619
|
+
/**
|
|
1620
|
+
* Get embedding/vector outbox health statistics
|
|
1621
|
+
*/
|
|
1622
|
+
async getOutboxStats() {
|
|
1623
|
+
await this.initialize();
|
|
1624
|
+
const embeddingRows = sqliteAll(
|
|
1625
|
+
this.db,
|
|
1626
|
+
`SELECT status, COUNT(*) as count FROM embedding_outbox GROUP BY status`
|
|
1627
|
+
);
|
|
1628
|
+
const vectorRows = sqliteAll(
|
|
1629
|
+
this.db,
|
|
1630
|
+
`SELECT status, COUNT(*) as count FROM vector_outbox GROUP BY status`
|
|
1631
|
+
);
|
|
1632
|
+
const fromRows = (rows) => {
|
|
1633
|
+
const out = { pending: 0, processing: 0, failed: 0, total: 0 };
|
|
1634
|
+
for (const row of rows) {
|
|
1635
|
+
const key = row.status;
|
|
1636
|
+
if (key === "pending" || key === "processing" || key === "failed") {
|
|
1637
|
+
out[key] += row.count;
|
|
1638
|
+
}
|
|
1639
|
+
out.total += row.count;
|
|
1640
|
+
}
|
|
1641
|
+
return out;
|
|
1642
|
+
};
|
|
1643
|
+
return {
|
|
1644
|
+
embedding: fromRows(embeddingRows),
|
|
1645
|
+
vector: fromRows(vectorRows)
|
|
1646
|
+
};
|
|
1647
|
+
}
|
|
1405
1648
|
/**
|
|
1406
1649
|
* Update memory level
|
|
1407
1650
|
*/
|
|
@@ -1760,6 +2003,79 @@ var SQLiteEventStore = class {
|
|
|
1760
2003
|
getDatabase() {
|
|
1761
2004
|
return this.db;
|
|
1762
2005
|
}
|
|
2006
|
+
async recordRetrievalTrace(input) {
|
|
2007
|
+
await this.initialize();
|
|
2008
|
+
const traceId = randomUUID2();
|
|
2009
|
+
sqliteRun(
|
|
2010
|
+
this.db,
|
|
2011
|
+
`INSERT INTO retrieval_traces (
|
|
2012
|
+
trace_id, session_id, project_hash, query_text, strategy,
|
|
2013
|
+
candidate_event_ids, selected_event_ids, candidate_details_json, selected_details_json,
|
|
2014
|
+
candidate_count, selected_count, confidence, fallback_trace
|
|
2015
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
2016
|
+
[
|
|
2017
|
+
traceId,
|
|
2018
|
+
input.sessionId || null,
|
|
2019
|
+
input.projectHash || null,
|
|
2020
|
+
input.queryText,
|
|
2021
|
+
input.strategy || null,
|
|
2022
|
+
JSON.stringify(input.candidateEventIds || []),
|
|
2023
|
+
JSON.stringify(input.selectedEventIds || []),
|
|
2024
|
+
JSON.stringify(input.candidateDetails || []),
|
|
2025
|
+
JSON.stringify(input.selectedDetails || []),
|
|
2026
|
+
(input.candidateEventIds || []).length,
|
|
2027
|
+
(input.selectedEventIds || []).length,
|
|
2028
|
+
input.confidence || null,
|
|
2029
|
+
JSON.stringify(input.fallbackTrace || [])
|
|
2030
|
+
]
|
|
2031
|
+
);
|
|
2032
|
+
}
|
|
2033
|
+
async getRecentRetrievalTraces(limit = 50) {
|
|
2034
|
+
await this.initialize();
|
|
2035
|
+
const rows = sqliteAll(
|
|
2036
|
+
this.db,
|
|
2037
|
+
`SELECT * FROM retrieval_traces ORDER BY created_at DESC LIMIT ?`,
|
|
2038
|
+
[limit]
|
|
2039
|
+
);
|
|
2040
|
+
return rows.map((row) => ({
|
|
2041
|
+
traceId: row.trace_id,
|
|
2042
|
+
sessionId: row.session_id || void 0,
|
|
2043
|
+
projectHash: row.project_hash || void 0,
|
|
2044
|
+
queryText: row.query_text,
|
|
2045
|
+
strategy: row.strategy || void 0,
|
|
2046
|
+
candidateEventIds: row.candidate_event_ids ? JSON.parse(row.candidate_event_ids) : [],
|
|
2047
|
+
selectedEventIds: row.selected_event_ids ? JSON.parse(row.selected_event_ids) : [],
|
|
2048
|
+
candidateDetails: row.candidate_details_json ? JSON.parse(row.candidate_details_json) : [],
|
|
2049
|
+
selectedDetails: row.selected_details_json ? JSON.parse(row.selected_details_json) : [],
|
|
2050
|
+
candidateCount: Number(row.candidate_count || 0),
|
|
2051
|
+
selectedCount: Number(row.selected_count || 0),
|
|
2052
|
+
confidence: row.confidence || void 0,
|
|
2053
|
+
fallbackTrace: row.fallback_trace ? JSON.parse(row.fallback_trace) : [],
|
|
2054
|
+
createdAt: toDateFromSQLite(row.created_at)
|
|
2055
|
+
}));
|
|
2056
|
+
}
|
|
2057
|
+
async getRetrievalTraceStats() {
|
|
2058
|
+
await this.initialize();
|
|
2059
|
+
const row = sqliteGet(
|
|
2060
|
+
this.db,
|
|
2061
|
+
`SELECT
|
|
2062
|
+
COUNT(*) as total_queries,
|
|
2063
|
+
AVG(candidate_count) as avg_candidate_count,
|
|
2064
|
+
AVG(selected_count) as avg_selected_count,
|
|
2065
|
+
CASE
|
|
2066
|
+
WHEN SUM(candidate_count) > 0 THEN (SUM(selected_count) * 1.0 / SUM(candidate_count))
|
|
2067
|
+
ELSE 0
|
|
2068
|
+
END as selection_rate
|
|
2069
|
+
FROM retrieval_traces`,
|
|
2070
|
+
[]
|
|
2071
|
+
);
|
|
2072
|
+
return {
|
|
2073
|
+
totalQueries: Number(row?.total_queries || 0),
|
|
2074
|
+
avgCandidateCount: Number(row?.avg_candidate_count || 0),
|
|
2075
|
+
avgSelectedCount: Number(row?.avg_selected_count || 0),
|
|
2076
|
+
selectionRate: Number(row?.selection_rate || 0)
|
|
2077
|
+
};
|
|
2078
|
+
}
|
|
1763
2079
|
/**
|
|
1764
2080
|
* Close database connection
|
|
1765
2081
|
*/
|
|
@@ -2621,7 +2937,20 @@ var DEFAULT_OPTIONS = {
|
|
|
2621
2937
|
topK: 5,
|
|
2622
2938
|
minScore: 0.7,
|
|
2623
2939
|
maxTokens: 2e3,
|
|
2624
|
-
includeSessionContext: true
|
|
2940
|
+
includeSessionContext: true,
|
|
2941
|
+
strategy: "auto",
|
|
2942
|
+
rerankWithKeyword: true,
|
|
2943
|
+
decayPolicy: {
|
|
2944
|
+
enabled: true,
|
|
2945
|
+
windowDays: 30,
|
|
2946
|
+
maxPenalty: 0.15
|
|
2947
|
+
},
|
|
2948
|
+
graphHop: {
|
|
2949
|
+
enabled: true,
|
|
2950
|
+
maxHops: 1,
|
|
2951
|
+
hopPenalty: 0.08
|
|
2952
|
+
},
|
|
2953
|
+
projectScopeMode: "global"
|
|
2625
2954
|
};
|
|
2626
2955
|
var Retriever = class {
|
|
2627
2956
|
eventStore;
|
|
@@ -2631,6 +2960,7 @@ var Retriever = class {
|
|
|
2631
2960
|
sharedStore;
|
|
2632
2961
|
sharedVectorStore;
|
|
2633
2962
|
graduation;
|
|
2963
|
+
queryRewriter;
|
|
2634
2964
|
constructor(eventStore, vectorStore, embedder, matcher, sharedOptions) {
|
|
2635
2965
|
this.eventStore = eventStore;
|
|
2636
2966
|
this.vectorStore = vectorStore;
|
|
@@ -2639,47 +2969,105 @@ var Retriever = class {
|
|
|
2639
2969
|
this.sharedStore = sharedOptions?.sharedStore;
|
|
2640
2970
|
this.sharedVectorStore = sharedOptions?.sharedVectorStore;
|
|
2641
2971
|
}
|
|
2642
|
-
/**
|
|
2643
|
-
* Set graduation pipeline for access tracking
|
|
2644
|
-
*/
|
|
2645
2972
|
setGraduationPipeline(graduation) {
|
|
2646
2973
|
this.graduation = graduation;
|
|
2647
2974
|
}
|
|
2648
|
-
/**
|
|
2649
|
-
* Set shared stores after construction
|
|
2650
|
-
*/
|
|
2651
2975
|
setSharedStores(sharedStore, sharedVectorStore) {
|
|
2652
2976
|
this.sharedStore = sharedStore;
|
|
2653
2977
|
this.sharedVectorStore = sharedVectorStore;
|
|
2654
2978
|
}
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2979
|
+
setQueryRewriter(rewriter) {
|
|
2980
|
+
this.queryRewriter = rewriter;
|
|
2981
|
+
}
|
|
2658
2982
|
async retrieve(query, options = {}) {
|
|
2659
2983
|
const opts = { ...DEFAULT_OPTIONS, ...options };
|
|
2660
|
-
const
|
|
2661
|
-
const
|
|
2662
|
-
|
|
2663
|
-
|
|
2984
|
+
const sessionFilter = opts.scope?.sessionId ?? opts.sessionId;
|
|
2985
|
+
const fallbackTrace = [];
|
|
2986
|
+
const fallbackEnabled = (opts.strategy ?? "auto") === "auto";
|
|
2987
|
+
const primaryStrategy = opts.strategy === "auto" ? "fast" : opts.strategy || "fast";
|
|
2988
|
+
let current = await this.runStage(query, {
|
|
2989
|
+
strategy: primaryStrategy,
|
|
2990
|
+
topK: opts.topK,
|
|
2664
2991
|
minScore: opts.minScore,
|
|
2665
|
-
sessionId:
|
|
2992
|
+
sessionId: sessionFilter,
|
|
2993
|
+
scope: opts.scope,
|
|
2994
|
+
rerankWithKeyword: opts.rerankWithKeyword !== false,
|
|
2995
|
+
rerankWeights: opts.rerankWeights,
|
|
2996
|
+
decayPolicy: opts.decayPolicy,
|
|
2997
|
+
intentRewrite: opts.intentRewrite === true,
|
|
2998
|
+
graphHop: opts.graphHop,
|
|
2999
|
+
projectScopeMode: opts.projectScopeMode,
|
|
3000
|
+
projectHash: opts.projectHash,
|
|
3001
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
2666
3002
|
});
|
|
2667
|
-
|
|
2668
|
-
|
|
2669
|
-
|
|
2670
|
-
|
|
2671
|
-
|
|
3003
|
+
fallbackTrace.push(`stage:primary:${primaryStrategy}`);
|
|
3004
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results) && primaryStrategy !== "deep") {
|
|
3005
|
+
current = await this.runStage(query, {
|
|
3006
|
+
strategy: "deep",
|
|
3007
|
+
topK: opts.topK,
|
|
3008
|
+
minScore: opts.minScore,
|
|
3009
|
+
sessionId: sessionFilter,
|
|
3010
|
+
scope: opts.scope,
|
|
3011
|
+
rerankWithKeyword: opts.rerankWithKeyword !== false,
|
|
3012
|
+
rerankWeights: opts.rerankWeights,
|
|
3013
|
+
decayPolicy: opts.decayPolicy,
|
|
3014
|
+
graphHop: opts.graphHop,
|
|
3015
|
+
projectScopeMode: opts.projectScopeMode,
|
|
3016
|
+
projectHash: opts.projectHash,
|
|
3017
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
3018
|
+
});
|
|
3019
|
+
fallbackTrace.push("fallback:deep");
|
|
3020
|
+
}
|
|
3021
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results)) {
|
|
3022
|
+
current = await this.runStage(query, {
|
|
3023
|
+
strategy: "deep",
|
|
3024
|
+
topK: opts.topK,
|
|
3025
|
+
minScore: Math.max(0.5, opts.minScore - 0.15),
|
|
3026
|
+
sessionId: void 0,
|
|
3027
|
+
scope: void 0,
|
|
3028
|
+
rerankWithKeyword: true,
|
|
3029
|
+
rerankWeights: opts.rerankWeights,
|
|
3030
|
+
decayPolicy: opts.decayPolicy,
|
|
3031
|
+
graphHop: opts.graphHop,
|
|
3032
|
+
projectScopeMode: opts.projectScopeMode,
|
|
3033
|
+
projectHash: opts.projectHash,
|
|
3034
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
3035
|
+
});
|
|
3036
|
+
fallbackTrace.push("fallback:scope-expanded");
|
|
3037
|
+
}
|
|
3038
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results)) {
|
|
3039
|
+
const summary = await this.buildSummaryFallback(query, opts.topK);
|
|
3040
|
+
current = {
|
|
3041
|
+
results: summary,
|
|
3042
|
+
candidateResults: summary,
|
|
3043
|
+
matchResult: this.matcher.matchSearchResults(summary, () => 0)
|
|
3044
|
+
};
|
|
3045
|
+
fallbackTrace.push("fallback:summary");
|
|
3046
|
+
}
|
|
3047
|
+
const memories = await this.enrichResults(current.results.slice(0, opts.topK), opts);
|
|
2672
3048
|
const context = this.buildContext(memories, opts.maxTokens);
|
|
2673
3049
|
return {
|
|
2674
3050
|
memories,
|
|
2675
|
-
matchResult,
|
|
3051
|
+
matchResult: current.matchResult,
|
|
2676
3052
|
totalTokens: this.estimateTokens(context),
|
|
2677
|
-
context
|
|
3053
|
+
context,
|
|
3054
|
+
fallbackTrace,
|
|
3055
|
+
selectedDebug: current.results.slice(0, opts.topK).map((r) => ({
|
|
3056
|
+
eventId: r.eventId,
|
|
3057
|
+
score: r.score,
|
|
3058
|
+
semanticScore: r.semanticScore,
|
|
3059
|
+
lexicalScore: r.lexicalScore,
|
|
3060
|
+
recencyScore: r.recencyScore
|
|
3061
|
+
})),
|
|
3062
|
+
candidateDebug: (current.candidateResults || []).slice(0, Math.max(opts.topK * 3, 20)).map((r) => ({
|
|
3063
|
+
eventId: r.eventId,
|
|
3064
|
+
score: r.score,
|
|
3065
|
+
semanticScore: r.semanticScore,
|
|
3066
|
+
lexicalScore: r.lexicalScore,
|
|
3067
|
+
recencyScore: r.recencyScore
|
|
3068
|
+
}))
|
|
2678
3069
|
};
|
|
2679
3070
|
}
|
|
2680
|
-
/**
|
|
2681
|
-
* Retrieve with unified search (project + shared)
|
|
2682
|
-
*/
|
|
2683
3071
|
async retrieveUnified(query, options = {}) {
|
|
2684
3072
|
const projectResult = await this.retrieve(query, options);
|
|
2685
3073
|
if (!options.includeShared || !this.sharedStore || !this.sharedVectorStore) {
|
|
@@ -2687,22 +3075,19 @@ var Retriever = class {
|
|
|
2687
3075
|
}
|
|
2688
3076
|
try {
|
|
2689
3077
|
const queryEmbedding = await this.embedder.embed(query);
|
|
2690
|
-
const sharedVectorResults = await this.sharedVectorStore.search(
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
excludeProjectHash: options.projectHash
|
|
2696
|
-
}
|
|
2697
|
-
);
|
|
3078
|
+
const sharedVectorResults = await this.sharedVectorStore.search(queryEmbedding.vector, {
|
|
3079
|
+
limit: options.topK || 5,
|
|
3080
|
+
minScore: options.minScore || 0.7,
|
|
3081
|
+
excludeProjectHash: options.projectHash
|
|
3082
|
+
});
|
|
2698
3083
|
const sharedMemories = [];
|
|
2699
3084
|
for (const result of sharedVectorResults) {
|
|
2700
3085
|
const entry = await this.sharedStore.get(result.entryId);
|
|
2701
|
-
if (entry)
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
3086
|
+
if (!entry)
|
|
3087
|
+
continue;
|
|
3088
|
+
if (!options.projectHash || entry.sourceProjectHash !== options.projectHash) {
|
|
3089
|
+
sharedMemories.push(entry);
|
|
3090
|
+
await this.sharedStore.recordUsage(entry.entryId);
|
|
2706
3091
|
}
|
|
2707
3092
|
}
|
|
2708
3093
|
const unifiedContext = this.buildUnifiedContext(projectResult, sharedMemories);
|
|
@@ -2717,50 +3102,243 @@ var Retriever = class {
|
|
|
2717
3102
|
return projectResult;
|
|
2718
3103
|
}
|
|
2719
3104
|
}
|
|
2720
|
-
|
|
2721
|
-
|
|
2722
|
-
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
2735
|
-
|
|
2736
|
-
|
|
2737
|
-
|
|
2738
|
-
|
|
2739
|
-
|
|
2740
|
-
|
|
3105
|
+
async runStage(query, input) {
|
|
3106
|
+
let initialResults = await this.searchByStrategy(query, {
|
|
3107
|
+
strategy: input.strategy,
|
|
3108
|
+
topK: input.topK,
|
|
3109
|
+
minScore: input.minScore,
|
|
3110
|
+
sessionId: input.sessionId
|
|
3111
|
+
});
|
|
3112
|
+
if (input.intentRewrite && input.strategy === "deep" && this.queryRewriter) {
|
|
3113
|
+
const rewritten = (await this.queryRewriter(query))?.trim();
|
|
3114
|
+
if (rewritten && rewritten !== query) {
|
|
3115
|
+
const rewrittenResults = await this.searchByStrategy(rewritten, {
|
|
3116
|
+
strategy: "deep",
|
|
3117
|
+
topK: input.topK,
|
|
3118
|
+
minScore: Math.max(0.5, input.minScore - 0.1),
|
|
3119
|
+
sessionId: input.sessionId
|
|
3120
|
+
});
|
|
3121
|
+
initialResults = this.mergeResults(initialResults, rewrittenResults, input.topK * 3);
|
|
3122
|
+
}
|
|
3123
|
+
}
|
|
3124
|
+
const expandedResults = input.graphHop?.enabled === false ? initialResults : await this.expandGraphHops(initialResults, {
|
|
3125
|
+
maxHops: Math.max(1, input.graphHop?.maxHops ?? 1),
|
|
3126
|
+
hopPenalty: Math.max(0, input.graphHop?.hopPenalty ?? 0.08),
|
|
3127
|
+
limit: input.topK * 4
|
|
3128
|
+
});
|
|
3129
|
+
const rerankedResults = input.rerankWithKeyword ? this.rerankByKeywordOverlap(expandedResults, query, input.rerankWeights, input.decayPolicy) : expandedResults;
|
|
3130
|
+
const filtered = await this.applyScopeFilters(rerankedResults, {
|
|
3131
|
+
scope: input.scope,
|
|
3132
|
+
projectScopeMode: input.projectScopeMode,
|
|
3133
|
+
projectHash: input.projectHash,
|
|
3134
|
+
allowedProjectHashes: input.allowedProjectHashes
|
|
3135
|
+
});
|
|
3136
|
+
const top = filtered.slice(0, input.topK);
|
|
3137
|
+
const matchResult = this.matcher.matchSearchResults(top, () => 0);
|
|
3138
|
+
return { results: top, candidateResults: filtered, matchResult };
|
|
3139
|
+
}
|
|
3140
|
+
mergeResults(primary, secondary, limit) {
|
|
3141
|
+
const byId = /* @__PURE__ */ new Map();
|
|
3142
|
+
for (const row of primary)
|
|
3143
|
+
byId.set(row.eventId, row);
|
|
3144
|
+
for (const row of secondary) {
|
|
3145
|
+
const prev = byId.get(row.eventId);
|
|
3146
|
+
if (!prev || row.score > prev.score) {
|
|
3147
|
+
byId.set(row.eventId, row);
|
|
3148
|
+
}
|
|
3149
|
+
}
|
|
3150
|
+
return [...byId.values()].sort((a, b) => b.score - a.score).slice(0, limit);
|
|
3151
|
+
}
|
|
3152
|
+
async expandGraphHops(seeds, opts) {
|
|
3153
|
+
const byId = /* @__PURE__ */ new Map();
|
|
3154
|
+
for (const s of seeds)
|
|
3155
|
+
byId.set(s.eventId, s);
|
|
3156
|
+
let frontier = seeds.map((s) => ({ row: s, hop: 0 }));
|
|
3157
|
+
for (let hop = 1; hop <= opts.maxHops; hop += 1) {
|
|
3158
|
+
const next = [];
|
|
3159
|
+
for (const f of frontier) {
|
|
3160
|
+
const ev = await this.eventStore.getEvent(f.row.eventId);
|
|
3161
|
+
if (!ev)
|
|
3162
|
+
continue;
|
|
3163
|
+
const rel = ev.metadata?.relatedEventIds ?? [];
|
|
3164
|
+
const relatedIds = Array.isArray(rel) ? rel.filter((x) => typeof x === "string") : [];
|
|
3165
|
+
for (const rid of relatedIds) {
|
|
3166
|
+
if (byId.has(rid))
|
|
3167
|
+
continue;
|
|
3168
|
+
const target = await this.eventStore.getEvent(rid);
|
|
3169
|
+
if (!target)
|
|
3170
|
+
continue;
|
|
3171
|
+
const score = Math.max(0, f.row.score - opts.hopPenalty * hop);
|
|
3172
|
+
const row = {
|
|
3173
|
+
id: `hop-${hop}-${rid}`,
|
|
3174
|
+
eventId: target.id,
|
|
3175
|
+
content: target.content,
|
|
3176
|
+
score,
|
|
3177
|
+
sessionId: target.sessionId,
|
|
3178
|
+
eventType: target.eventType,
|
|
3179
|
+
timestamp: target.timestamp.toISOString()
|
|
3180
|
+
};
|
|
3181
|
+
byId.set(row.eventId, row);
|
|
3182
|
+
next.push({ row, hop });
|
|
3183
|
+
if (byId.size >= opts.limit)
|
|
3184
|
+
break;
|
|
2741
3185
|
}
|
|
2742
|
-
|
|
2743
|
-
|
|
2744
|
-
`;
|
|
3186
|
+
if (byId.size >= opts.limit)
|
|
3187
|
+
break;
|
|
2745
3188
|
}
|
|
3189
|
+
frontier = next;
|
|
3190
|
+
if (frontier.length === 0 || byId.size >= opts.limit)
|
|
3191
|
+
break;
|
|
2746
3192
|
}
|
|
2747
|
-
return
|
|
3193
|
+
return [...byId.values()].sort((a, b) => b.score - a.score).slice(0, opts.limit);
|
|
3194
|
+
}
|
|
3195
|
+
shouldFallback(matchResult, results) {
|
|
3196
|
+
if (results.length === 0)
|
|
3197
|
+
return true;
|
|
3198
|
+
if (matchResult.confidence === "none")
|
|
3199
|
+
return true;
|
|
3200
|
+
return false;
|
|
3201
|
+
}
|
|
3202
|
+
async buildSummaryFallback(query, topK) {
|
|
3203
|
+
const recent = await this.eventStore.getRecentEvents(Math.max(topK * 6, 20));
|
|
3204
|
+
const q = this.tokenize(query);
|
|
3205
|
+
const ranked = recent.map((e) => ({ e, overlap: this.keywordOverlap(q, this.tokenize(e.content)) })).filter((r) => r.overlap > 0).sort((a, b) => b.overlap - a.overlap).slice(0, topK).map((row, idx) => ({
|
|
3206
|
+
id: `summary-${row.e.id}`,
|
|
3207
|
+
eventId: row.e.id,
|
|
3208
|
+
content: row.e.content,
|
|
3209
|
+
score: Math.max(0.25, 0.6 - idx * 0.05),
|
|
3210
|
+
sessionId: row.e.sessionId,
|
|
3211
|
+
eventType: row.e.eventType,
|
|
3212
|
+
timestamp: row.e.timestamp.toISOString()
|
|
3213
|
+
}));
|
|
3214
|
+
return ranked;
|
|
3215
|
+
}
|
|
3216
|
+
async searchByStrategy(query, input) {
|
|
3217
|
+
const strategy = input.strategy === "auto" ? "deep" : input.strategy;
|
|
3218
|
+
if (strategy === "fast") {
|
|
3219
|
+
const keyword = await this.searchByKeyword(query, {
|
|
3220
|
+
limit: Math.max(5, input.topK * 3),
|
|
3221
|
+
sessionId: input.sessionId
|
|
3222
|
+
});
|
|
3223
|
+
return keyword;
|
|
3224
|
+
}
|
|
3225
|
+
const queryEmbedding = await this.embedder.embed(query);
|
|
3226
|
+
return this.vectorStore.search(queryEmbedding.vector, {
|
|
3227
|
+
limit: Math.max(5, input.topK * 3),
|
|
3228
|
+
minScore: input.minScore,
|
|
3229
|
+
sessionId: input.sessionId
|
|
3230
|
+
});
|
|
3231
|
+
}
|
|
3232
|
+
async searchByKeyword(query, input) {
|
|
3233
|
+
if (this.eventStore.keywordSearch) {
|
|
3234
|
+
const rows = await this.eventStore.keywordSearch(query, input.limit);
|
|
3235
|
+
const filtered2 = input.sessionId ? rows.filter((r) => r.event.sessionId === input.sessionId) : rows;
|
|
3236
|
+
return filtered2.map((row, idx) => ({
|
|
3237
|
+
id: `kw-${row.event.id}`,
|
|
3238
|
+
eventId: row.event.id,
|
|
3239
|
+
content: row.event.content,
|
|
3240
|
+
score: Math.max(0.4, 1 - idx * 0.04),
|
|
3241
|
+
sessionId: row.event.sessionId,
|
|
3242
|
+
eventType: row.event.eventType,
|
|
3243
|
+
timestamp: row.event.timestamp.toISOString()
|
|
3244
|
+
}));
|
|
3245
|
+
}
|
|
3246
|
+
const recent = await this.eventStore.getRecentEvents(input.limit * 4);
|
|
3247
|
+
const tokens = this.tokenize(query);
|
|
3248
|
+
const filtered = recent.filter((e) => input.sessionId ? e.sessionId === input.sessionId : true).map((e) => ({ e, overlap: this.keywordOverlap(tokens, this.tokenize(e.content)) })).filter((r) => r.overlap > 0).sort((a, b) => b.overlap - a.overlap).slice(0, input.limit);
|
|
3249
|
+
return filtered.map((row, idx) => ({
|
|
3250
|
+
id: `kw-fallback-${row.e.id}`,
|
|
3251
|
+
eventId: row.e.id,
|
|
3252
|
+
content: row.e.content,
|
|
3253
|
+
score: Math.max(0.3, 0.9 - idx * 0.05),
|
|
3254
|
+
sessionId: row.e.sessionId,
|
|
3255
|
+
eventType: row.e.eventType,
|
|
3256
|
+
timestamp: row.e.timestamp.toISOString()
|
|
3257
|
+
}));
|
|
3258
|
+
}
|
|
3259
|
+
rerankByKeywordOverlap(results, query, weights, decayPolicy) {
|
|
3260
|
+
const q = this.tokenize(query);
|
|
3261
|
+
const now = Date.now();
|
|
3262
|
+
const sw = Math.max(0, weights?.semantic ?? 0.7);
|
|
3263
|
+
const lw = Math.max(0, weights?.lexical ?? 0.2);
|
|
3264
|
+
const rw = Math.max(0, weights?.recency ?? 0.1);
|
|
3265
|
+
const total = sw + lw + rw || 1;
|
|
3266
|
+
const decayEnabled = decayPolicy?.enabled !== false;
|
|
3267
|
+
const decayWindow = Math.max(1, decayPolicy?.windowDays ?? 30);
|
|
3268
|
+
const decayMaxPenalty = Math.max(0, decayPolicy?.maxPenalty ?? 0.15);
|
|
3269
|
+
return [...results].map((r) => {
|
|
3270
|
+
const overlap = this.keywordOverlap(q, this.tokenize(r.content));
|
|
3271
|
+
const recencyDays = Math.max(0, (now - new Date(r.timestamp).getTime()) / (1e3 * 60 * 60 * 24));
|
|
3272
|
+
const recency = Math.max(0, 1 - recencyDays / decayWindow);
|
|
3273
|
+
let blended = (r.score * sw + overlap * lw + recency * rw) / total;
|
|
3274
|
+
if (decayEnabled && recencyDays > decayWindow && overlap < 0.5) {
|
|
3275
|
+
const ageFactor = Math.min(1, (recencyDays - decayWindow) / decayWindow);
|
|
3276
|
+
blended -= decayMaxPenalty * ageFactor;
|
|
3277
|
+
}
|
|
3278
|
+
return { ...r, score: Math.max(0, blended), semanticScore: r.score, lexicalScore: overlap, recencyScore: recency };
|
|
3279
|
+
}).sort((a, b) => b.score - a.score);
|
|
3280
|
+
}
|
|
3281
|
+
async applyScopeFilters(results, options) {
|
|
3282
|
+
const scope = options?.scope;
|
|
3283
|
+
const projectScopeMode = options?.projectScopeMode ?? "global";
|
|
3284
|
+
const allowedProjectHashes = new Set(
|
|
3285
|
+
[options?.projectHash, ...options?.allowedProjectHashes || []].filter(
|
|
3286
|
+
(value) => typeof value === "string" && value.length > 0
|
|
3287
|
+
)
|
|
3288
|
+
);
|
|
3289
|
+
if (!scope && projectScopeMode === "global")
|
|
3290
|
+
return results;
|
|
3291
|
+
const normalizedIncludes = (scope?.contentIncludes || []).map((s) => s.toLowerCase());
|
|
3292
|
+
const filtered = [];
|
|
3293
|
+
for (const result of results) {
|
|
3294
|
+
if (scope?.sessionId && result.sessionId !== scope.sessionId)
|
|
3295
|
+
continue;
|
|
3296
|
+
if (scope?.sessionIdPrefix && !result.sessionId.startsWith(scope.sessionIdPrefix))
|
|
3297
|
+
continue;
|
|
3298
|
+
if (scope?.eventTypes && scope.eventTypes.length > 0 && !scope.eventTypes.includes(result.eventType))
|
|
3299
|
+
continue;
|
|
3300
|
+
const event = await this.eventStore.getEvent(result.eventId);
|
|
3301
|
+
if (!event)
|
|
3302
|
+
continue;
|
|
3303
|
+
if (scope?.canonicalKeyPrefix && !event.canonicalKey.startsWith(scope.canonicalKeyPrefix))
|
|
3304
|
+
continue;
|
|
3305
|
+
if (normalizedIncludes.length > 0) {
|
|
3306
|
+
const lc = event.content.toLowerCase();
|
|
3307
|
+
if (!normalizedIncludes.some((needle) => lc.includes(needle)))
|
|
3308
|
+
continue;
|
|
3309
|
+
}
|
|
3310
|
+
if (scope?.metadata && !this.matchesMetadataScope(event.metadata, scope.metadata))
|
|
3311
|
+
continue;
|
|
3312
|
+
const projectHash = this.extractProjectHash(event.metadata);
|
|
3313
|
+
filtered.push({ result, projectHash });
|
|
3314
|
+
}
|
|
3315
|
+
if (projectScopeMode === "global" || allowedProjectHashes.size === 0) {
|
|
3316
|
+
return filtered.map((x) => x.result);
|
|
3317
|
+
}
|
|
3318
|
+
const projectMatched = filtered.filter((x) => x.projectHash && allowedProjectHashes.has(x.projectHash));
|
|
3319
|
+
if (projectScopeMode === "strict") {
|
|
3320
|
+
return projectMatched.map((x) => x.result);
|
|
3321
|
+
}
|
|
3322
|
+
return (projectMatched.length > 0 ? projectMatched : filtered).map((x) => x.result);
|
|
3323
|
+
}
|
|
3324
|
+
extractProjectHash(metadata) {
|
|
3325
|
+
if (!metadata || typeof metadata !== "object")
|
|
3326
|
+
return void 0;
|
|
3327
|
+
const scope = metadata.scope;
|
|
3328
|
+
if (!scope || typeof scope !== "object")
|
|
3329
|
+
return void 0;
|
|
3330
|
+
const project = scope.project;
|
|
3331
|
+
if (!project || typeof project !== "object")
|
|
3332
|
+
return void 0;
|
|
3333
|
+
const hash = project.hash;
|
|
3334
|
+
return typeof hash === "string" && hash.length > 0 ? hash : void 0;
|
|
2748
3335
|
}
|
|
2749
|
-
/**
|
|
2750
|
-
* Retrieve memories from a specific session
|
|
2751
|
-
*/
|
|
2752
3336
|
async retrieveFromSession(sessionId) {
|
|
2753
3337
|
return this.eventStore.getSessionEvents(sessionId);
|
|
2754
3338
|
}
|
|
2755
|
-
/**
|
|
2756
|
-
* Get recent memories across all sessions
|
|
2757
|
-
*/
|
|
2758
3339
|
async retrieveRecent(limit = 100) {
|
|
2759
3340
|
return this.eventStore.getRecentEvents(limit);
|
|
2760
3341
|
}
|
|
2761
|
-
/**
|
|
2762
|
-
* Enrich search results with full event data
|
|
2763
|
-
*/
|
|
2764
3342
|
async enrichResults(results, options) {
|
|
2765
3343
|
const memories = [];
|
|
2766
3344
|
for (const result of results) {
|
|
@@ -2768,27 +3346,16 @@ var Retriever = class {
|
|
|
2768
3346
|
if (!event)
|
|
2769
3347
|
continue;
|
|
2770
3348
|
if (this.graduation) {
|
|
2771
|
-
this.graduation.recordAccess(
|
|
2772
|
-
event.id,
|
|
2773
|
-
options.sessionId || "unknown",
|
|
2774
|
-
result.score
|
|
2775
|
-
);
|
|
3349
|
+
this.graduation.recordAccess(event.id, options.sessionId || "unknown", result.score);
|
|
2776
3350
|
}
|
|
2777
3351
|
let sessionContext;
|
|
2778
3352
|
if (options.includeSessionContext) {
|
|
2779
3353
|
sessionContext = await this.getSessionContext(event.sessionId, event.id);
|
|
2780
3354
|
}
|
|
2781
|
-
memories.push({
|
|
2782
|
-
event,
|
|
2783
|
-
score: result.score,
|
|
2784
|
-
sessionContext
|
|
2785
|
-
});
|
|
3355
|
+
memories.push({ event, score: result.score, sessionContext });
|
|
2786
3356
|
}
|
|
2787
3357
|
return memories;
|
|
2788
3358
|
}
|
|
2789
|
-
/**
|
|
2790
|
-
* Get surrounding context from the same session
|
|
2791
|
-
*/
|
|
2792
3359
|
async getSessionContext(sessionId, eventId) {
|
|
2793
3360
|
const sessionEvents = await this.eventStore.getSessionEvents(sessionId);
|
|
2794
3361
|
const eventIndex = sessionEvents.findIndex((e) => e.id === eventId);
|
|
@@ -2801,55 +3368,86 @@ var Retriever = class {
|
|
|
2801
3368
|
return void 0;
|
|
2802
3369
|
return contextEvents.filter((e) => e.id !== eventId).map((e) => `[${e.eventType}]: ${e.content.slice(0, 200)}...`).join("\n");
|
|
2803
3370
|
}
|
|
2804
|
-
|
|
2805
|
-
|
|
2806
|
-
|
|
3371
|
+
buildUnifiedContext(projectResult, sharedMemories) {
|
|
3372
|
+
let context = projectResult.context;
|
|
3373
|
+
if (sharedMemories.length === 0)
|
|
3374
|
+
return context;
|
|
3375
|
+
context += "\n\n## Cross-Project Knowledge\n\n";
|
|
3376
|
+
for (const memory of sharedMemories.slice(0, 3)) {
|
|
3377
|
+
context += `### ${memory.title}
|
|
3378
|
+
`;
|
|
3379
|
+
if (memory.symptoms.length > 0)
|
|
3380
|
+
context += `**Symptoms:** ${memory.symptoms.join(", ")}
|
|
3381
|
+
`;
|
|
3382
|
+
context += `**Root Cause:** ${memory.rootCause}
|
|
3383
|
+
`;
|
|
3384
|
+
context += `**Solution:** ${memory.solution}
|
|
3385
|
+
`;
|
|
3386
|
+
if (memory.technologies && memory.technologies.length > 0)
|
|
3387
|
+
context += `**Technologies:** ${memory.technologies.join(", ")}
|
|
3388
|
+
`;
|
|
3389
|
+
context += `_Confidence: ${(memory.confidence * 100).toFixed(0)}%_
|
|
3390
|
+
|
|
3391
|
+
`;
|
|
3392
|
+
}
|
|
3393
|
+
return context;
|
|
3394
|
+
}
|
|
2807
3395
|
buildContext(memories, maxTokens) {
|
|
2808
3396
|
const parts = [];
|
|
2809
3397
|
let currentTokens = 0;
|
|
2810
3398
|
for (const memory of memories) {
|
|
2811
3399
|
const memoryText = this.formatMemory(memory);
|
|
2812
3400
|
const memoryTokens = this.estimateTokens(memoryText);
|
|
2813
|
-
if (currentTokens + memoryTokens > maxTokens)
|
|
3401
|
+
if (currentTokens + memoryTokens > maxTokens)
|
|
2814
3402
|
break;
|
|
2815
|
-
}
|
|
2816
3403
|
parts.push(memoryText);
|
|
2817
3404
|
currentTokens += memoryTokens;
|
|
2818
3405
|
}
|
|
2819
|
-
if (parts.length === 0)
|
|
3406
|
+
if (parts.length === 0)
|
|
2820
3407
|
return "";
|
|
2821
|
-
}
|
|
2822
3408
|
return `## Relevant Memories
|
|
2823
3409
|
|
|
2824
3410
|
${parts.join("\n\n---\n\n")}`;
|
|
2825
3411
|
}
|
|
2826
|
-
/**
|
|
2827
|
-
* Format a single memory for context
|
|
2828
|
-
*/
|
|
2829
3412
|
formatMemory(memory) {
|
|
2830
3413
|
const { event, score, sessionContext } = memory;
|
|
2831
3414
|
const date = event.timestamp.toISOString().split("T")[0];
|
|
2832
3415
|
let text = `**${event.eventType}** (${date}, score: ${score.toFixed(2)})
|
|
2833
3416
|
${event.content}`;
|
|
2834
|
-
if (sessionContext)
|
|
3417
|
+
if (sessionContext)
|
|
2835
3418
|
text += `
|
|
2836
3419
|
|
|
2837
3420
|
_Context:_ ${sessionContext}`;
|
|
2838
|
-
}
|
|
2839
3421
|
return text;
|
|
2840
3422
|
}
|
|
2841
|
-
|
|
2842
|
-
|
|
2843
|
-
|
|
3423
|
+
matchesMetadataScope(metadata, expected) {
|
|
3424
|
+
if (!metadata)
|
|
3425
|
+
return false;
|
|
3426
|
+
return Object.entries(expected).every(([path4, value]) => {
|
|
3427
|
+
const actual = path4.split(".").reduce((acc, key) => {
|
|
3428
|
+
if (typeof acc !== "object" || acc === null)
|
|
3429
|
+
return void 0;
|
|
3430
|
+
return acc[key];
|
|
3431
|
+
}, metadata);
|
|
3432
|
+
return actual === value;
|
|
3433
|
+
});
|
|
3434
|
+
}
|
|
3435
|
+
tokenize(text) {
|
|
3436
|
+
return text.toLowerCase().replace(/[^\p{L}\p{N}\s]/gu, " ").split(/\s+/).filter((t) => t.length >= 2).slice(0, 64);
|
|
3437
|
+
}
|
|
3438
|
+
keywordOverlap(a, b) {
|
|
3439
|
+
if (a.length === 0 || b.length === 0)
|
|
3440
|
+
return 0;
|
|
3441
|
+
const bs = new Set(b);
|
|
3442
|
+
let hit = 0;
|
|
3443
|
+
for (const t of a)
|
|
3444
|
+
if (bs.has(t))
|
|
3445
|
+
hit += 1;
|
|
3446
|
+
return hit / a.length;
|
|
3447
|
+
}
|
|
2844
3448
|
estimateTokens(text) {
|
|
2845
3449
|
return Math.ceil(text.length / 4);
|
|
2846
3450
|
}
|
|
2847
|
-
/**
|
|
2848
|
-
* Get event age in days (for recency scoring)
|
|
2849
|
-
*/
|
|
2850
|
-
getEventAgeDays(eventId) {
|
|
2851
|
-
return 0;
|
|
2852
|
-
}
|
|
2853
3451
|
};
|
|
2854
3452
|
function createRetriever(eventStore, vectorStore, embedder, matcher) {
|
|
2855
3453
|
return new Retriever(eventStore, vectorStore, embedder, matcher);
|
|
@@ -4139,6 +4737,59 @@ var ConsolidatedStore = class {
|
|
|
4139
4737
|
[memoryId]
|
|
4140
4738
|
);
|
|
4141
4739
|
}
|
|
4740
|
+
/**
|
|
4741
|
+
* Create a long-term rule promoted from stable summaries
|
|
4742
|
+
*/
|
|
4743
|
+
async createRule(input) {
|
|
4744
|
+
const ruleId = randomUUID6();
|
|
4745
|
+
await dbRun(
|
|
4746
|
+
this.db,
|
|
4747
|
+
`INSERT INTO consolidated_rules
|
|
4748
|
+
(rule_id, rule, topics, source_memory_ids, source_events, confidence, created_at)
|
|
4749
|
+
VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)`,
|
|
4750
|
+
[
|
|
4751
|
+
ruleId,
|
|
4752
|
+
input.rule,
|
|
4753
|
+
JSON.stringify(input.topics),
|
|
4754
|
+
JSON.stringify(input.sourceMemoryIds),
|
|
4755
|
+
JSON.stringify(input.sourceEvents),
|
|
4756
|
+
input.confidence
|
|
4757
|
+
]
|
|
4758
|
+
);
|
|
4759
|
+
return ruleId;
|
|
4760
|
+
}
|
|
4761
|
+
async getRules(options) {
|
|
4762
|
+
const limit = options?.limit || 100;
|
|
4763
|
+
const rows = await dbAll(
|
|
4764
|
+
this.db,
|
|
4765
|
+
`SELECT * FROM consolidated_rules ORDER BY confidence DESC, created_at DESC LIMIT ?`,
|
|
4766
|
+
[limit]
|
|
4767
|
+
);
|
|
4768
|
+
return rows.map((row) => ({
|
|
4769
|
+
ruleId: row.rule_id,
|
|
4770
|
+
rule: row.rule,
|
|
4771
|
+
topics: JSON.parse(row.topics || "[]"),
|
|
4772
|
+
sourceMemoryIds: JSON.parse(row.source_memory_ids || "[]"),
|
|
4773
|
+
sourceEvents: JSON.parse(row.source_events || "[]"),
|
|
4774
|
+
confidence: Number(row.confidence ?? 0.5),
|
|
4775
|
+
createdAt: toDate(row.created_at) || /* @__PURE__ */ new Date()
|
|
4776
|
+
}));
|
|
4777
|
+
}
|
|
4778
|
+
async countRules() {
|
|
4779
|
+
const result = await dbAll(
|
|
4780
|
+
this.db,
|
|
4781
|
+
`SELECT COUNT(*) as count FROM consolidated_rules`
|
|
4782
|
+
);
|
|
4783
|
+
return result[0]?.count || 0;
|
|
4784
|
+
}
|
|
4785
|
+
async hasRuleForSourceMemory(memoryId) {
|
|
4786
|
+
const rows = await dbAll(
|
|
4787
|
+
this.db,
|
|
4788
|
+
`SELECT COUNT(*) as count FROM consolidated_rules WHERE source_memory_ids LIKE ?`,
|
|
4789
|
+
[`%"${memoryId}"%`]
|
|
4790
|
+
);
|
|
4791
|
+
return (rows[0]?.count || 0) > 0;
|
|
4792
|
+
}
|
|
4142
4793
|
/**
|
|
4143
4794
|
* Get count of consolidated memories
|
|
4144
4795
|
*/
|
|
@@ -4288,7 +4939,14 @@ var ConsolidationWorker = class {
|
|
|
4288
4939
|
* Force a consolidation run (manual trigger)
|
|
4289
4940
|
*/
|
|
4290
4941
|
async forceRun() {
|
|
4291
|
-
|
|
4942
|
+
const out = await this.consolidateWithReport();
|
|
4943
|
+
return out.consolidatedCount;
|
|
4944
|
+
}
|
|
4945
|
+
/**
|
|
4946
|
+
* Force a consolidation run and return metrics report
|
|
4947
|
+
*/
|
|
4948
|
+
async forceRunWithReport() {
|
|
4949
|
+
return this.consolidateWithReport();
|
|
4292
4950
|
}
|
|
4293
4951
|
/**
|
|
4294
4952
|
* Schedule the next consolidation check
|
|
@@ -4328,12 +4986,21 @@ var ConsolidationWorker = class {
|
|
|
4328
4986
|
* Perform consolidation
|
|
4329
4987
|
*/
|
|
4330
4988
|
async consolidate() {
|
|
4989
|
+
const out = await this.consolidateWithReport();
|
|
4990
|
+
return out.consolidatedCount;
|
|
4991
|
+
}
|
|
4992
|
+
async consolidateWithReport() {
|
|
4331
4993
|
const workingSet = await this.workingSetStore.get();
|
|
4332
4994
|
if (workingSet.recentEvents.length < 3) {
|
|
4333
|
-
return
|
|
4995
|
+
return {
|
|
4996
|
+
consolidatedCount: 0,
|
|
4997
|
+
promotedRuleCount: 0,
|
|
4998
|
+
report: this.buildCostQualityReport(workingSet.recentEvents, [], 0)
|
|
4999
|
+
};
|
|
4334
5000
|
}
|
|
4335
5001
|
const groups = this.groupByTopic(workingSet.recentEvents);
|
|
4336
5002
|
let consolidatedCount = 0;
|
|
5003
|
+
const createdMemoryIds = [];
|
|
4337
5004
|
for (const group of groups) {
|
|
4338
5005
|
if (group.events.length < 3)
|
|
4339
5006
|
continue;
|
|
@@ -4342,14 +5009,16 @@ var ConsolidationWorker = class {
|
|
|
4342
5009
|
if (alreadyConsolidated)
|
|
4343
5010
|
continue;
|
|
4344
5011
|
const summary = await this.summarize(group);
|
|
4345
|
-
await this.consolidatedStore.create({
|
|
5012
|
+
const memoryId = await this.consolidatedStore.create({
|
|
4346
5013
|
summary,
|
|
4347
5014
|
topics: group.topics,
|
|
4348
5015
|
sourceEvents: eventIds,
|
|
4349
5016
|
confidence: this.calculateConfidence(group)
|
|
4350
5017
|
});
|
|
5018
|
+
createdMemoryIds.push(memoryId);
|
|
4351
5019
|
consolidatedCount++;
|
|
4352
5020
|
}
|
|
5021
|
+
const promotedRuleCount = await this.promoteStableSummariesToRules(createdMemoryIds);
|
|
4353
5022
|
if (consolidatedCount > 0) {
|
|
4354
5023
|
const consolidatedEventIds = groups.filter((g) => g.events.length >= 3).flatMap((g) => g.events.map((e) => e.id));
|
|
4355
5024
|
const oldEventIds = consolidatedEventIds.filter((id) => {
|
|
@@ -4363,7 +5032,61 @@ var ConsolidationWorker = class {
|
|
|
4363
5032
|
await this.workingSetStore.prune(oldEventIds);
|
|
4364
5033
|
}
|
|
4365
5034
|
}
|
|
4366
|
-
|
|
5035
|
+
const report = this.buildCostQualityReport(workingSet.recentEvents, groups, consolidatedCount);
|
|
5036
|
+
return { consolidatedCount, promotedRuleCount, report };
|
|
5037
|
+
}
|
|
5038
|
+
async promoteStableSummariesToRules(memoryIds) {
|
|
5039
|
+
let promoted = 0;
|
|
5040
|
+
for (const memoryId of memoryIds) {
|
|
5041
|
+
const memory = await this.consolidatedStore.get(memoryId);
|
|
5042
|
+
if (!memory)
|
|
5043
|
+
continue;
|
|
5044
|
+
if (memory.confidence < 0.55)
|
|
5045
|
+
continue;
|
|
5046
|
+
if (memory.sourceEvents.length < 4)
|
|
5047
|
+
continue;
|
|
5048
|
+
const exists = await this.consolidatedStore.hasRuleForSourceMemory(memoryId);
|
|
5049
|
+
if (exists)
|
|
5050
|
+
continue;
|
|
5051
|
+
const rule = this.buildRuleFromSummary(memory.summary, memory.topics);
|
|
5052
|
+
if (!rule)
|
|
5053
|
+
continue;
|
|
5054
|
+
await this.consolidatedStore.createRule({
|
|
5055
|
+
rule,
|
|
5056
|
+
topics: memory.topics,
|
|
5057
|
+
sourceMemoryIds: [memory.memoryId],
|
|
5058
|
+
sourceEvents: memory.sourceEvents,
|
|
5059
|
+
confidence: Math.min(1, memory.confidence + 0.08)
|
|
5060
|
+
});
|
|
5061
|
+
promoted++;
|
|
5062
|
+
}
|
|
5063
|
+
return promoted;
|
|
5064
|
+
}
|
|
5065
|
+
buildRuleFromSummary(summary, topics) {
|
|
5066
|
+
const lines = summary.split(/\r?\n/).map((l) => l.trim()).filter(Boolean).filter((l) => !l.toLowerCase().startsWith("topics:"));
|
|
5067
|
+
const bullet = lines.find((l) => l.startsWith("- "))?.replace(/^-\s*/, "");
|
|
5068
|
+
const seed = bullet || lines[0];
|
|
5069
|
+
if (!seed || seed.length < 8)
|
|
5070
|
+
return null;
|
|
5071
|
+
const topicPrefix = topics.length > 0 ? `[${topics.slice(0, 2).join(", ")}] ` : "";
|
|
5072
|
+
return `${topicPrefix}${seed}`;
|
|
5073
|
+
}
|
|
5074
|
+
buildCostQualityReport(events, groups, consolidatedCount) {
|
|
5075
|
+
const beforeTokenEstimate = events.reduce((acc, e) => acc + this.estimateTokens(e.content), 0);
|
|
5076
|
+
const afterSummaries = groups.filter((g) => g.events.length >= 3).slice(0, Math.max(consolidatedCount, 1));
|
|
5077
|
+
const afterTokenEstimate = afterSummaries.length > 0 ? afterSummaries.reduce((acc, g) => acc + this.estimateTokens(this.ruleBasedSummary(g)), 0) : beforeTokenEstimate;
|
|
5078
|
+
const reductionRatio = beforeTokenEstimate > 0 ? Math.max(0, (beforeTokenEstimate - afterTokenEstimate) / beforeTokenEstimate) : 0;
|
|
5079
|
+
const qualityGuardPassed = consolidatedCount === 0 ? true : groups.filter((g) => g.events.length >= 3).every((g) => this.calculateConfidence(g) >= 0.55);
|
|
5080
|
+
return {
|
|
5081
|
+
beforeTokenEstimate,
|
|
5082
|
+
afterTokenEstimate,
|
|
5083
|
+
reductionRatio,
|
|
5084
|
+
qualityGuardPassed,
|
|
5085
|
+
details: `groups=${groups.length}, consolidated=${consolidatedCount}`
|
|
5086
|
+
};
|
|
5087
|
+
}
|
|
5088
|
+
estimateTokens(text) {
|
|
5089
|
+
return Math.ceil((text || "").length / 4);
|
|
4367
5090
|
}
|
|
4368
5091
|
/**
|
|
4369
5092
|
* Check if consolidation should run
|
|
@@ -4923,13 +5646,185 @@ function createGraduationWorker(eventStore, graduation, config) {
|
|
|
4923
5646
|
);
|
|
4924
5647
|
}
|
|
4925
5648
|
|
|
5649
|
+
// src/core/md-mirror.ts
|
|
5650
|
+
import * as fs3 from "node:fs";
|
|
5651
|
+
import * as path2 from "node:path";
|
|
5652
|
+
function sanitizeSegment2(input, fallback) {
|
|
5653
|
+
const v = (input || "").trim().toLowerCase().replace(/[^a-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
5654
|
+
return v || fallback;
|
|
5655
|
+
}
|
|
5656
|
+
function getAtPath(obj, dotted) {
|
|
5657
|
+
if (!obj)
|
|
5658
|
+
return void 0;
|
|
5659
|
+
return dotted.split(".").reduce((acc, key) => {
|
|
5660
|
+
if (!acc || typeof acc !== "object")
|
|
5661
|
+
return void 0;
|
|
5662
|
+
return acc[key];
|
|
5663
|
+
}, obj);
|
|
5664
|
+
}
|
|
5665
|
+
function buildMirrorPath2(rootDir, event) {
|
|
5666
|
+
const meta = event.metadata;
|
|
5667
|
+
const namespaceRaw = getAtPath(meta, "namespace") ?? getAtPath(meta, "scope.namespace") ?? event.eventType;
|
|
5668
|
+
const namespace = sanitizeSegment2(typeof namespaceRaw === "string" ? namespaceRaw : void 0, "general");
|
|
5669
|
+
const categoryRaw = getAtPath(meta, "categoryPath") ?? getAtPath(meta, "scope.categoryPath");
|
|
5670
|
+
const categoryPath = Array.isArray(categoryRaw) && categoryRaw.length > 0 ? categoryRaw.map((x) => sanitizeSegment2(typeof x === "string" ? x : void 0, "uncategorized")) : ["uncategorized"];
|
|
5671
|
+
const d = event.timestamp;
|
|
5672
|
+
const yyyy = d.getFullYear();
|
|
5673
|
+
const mm = String(d.getMonth() + 1).padStart(2, "0");
|
|
5674
|
+
const dd = String(d.getDate()).padStart(2, "0");
|
|
5675
|
+
return path2.join(rootDir, "memory", namespace, ...categoryPath, `${yyyy}-${mm}-${dd}.md`);
|
|
5676
|
+
}
|
|
5677
|
+
var MarkdownMirror2 = class {
|
|
5678
|
+
constructor(rootDir) {
|
|
5679
|
+
this.rootDir = rootDir;
|
|
5680
|
+
}
|
|
5681
|
+
async append(event, eventId) {
|
|
5682
|
+
const out = buildMirrorPath2(this.rootDir, event);
|
|
5683
|
+
fs3.mkdirSync(path2.dirname(out), { recursive: true });
|
|
5684
|
+
const lines = [
|
|
5685
|
+
"",
|
|
5686
|
+
`## ${event.timestamp.toISOString()} | ${eventId ?? "pending-id"}`,
|
|
5687
|
+
`- type: ${event.eventType}`,
|
|
5688
|
+
`- session: ${event.sessionId}`,
|
|
5689
|
+
event.content
|
|
5690
|
+
];
|
|
5691
|
+
await fs3.promises.appendFile(out, lines.join("\n"), "utf8");
|
|
5692
|
+
await this.refreshIndex();
|
|
5693
|
+
}
|
|
5694
|
+
async refreshIndex() {
|
|
5695
|
+
const memoryRoot = path2.join(this.rootDir, "memory");
|
|
5696
|
+
await fs3.promises.mkdir(memoryRoot, { recursive: true });
|
|
5697
|
+
const files = [];
|
|
5698
|
+
await this.walk(memoryRoot, files);
|
|
5699
|
+
const mdFiles = files.filter((f) => f.endsWith(".md")).map((f) => path2.relative(this.rootDir, f)).filter((rel) => rel !== path2.join("memory", "_index.md")).sort();
|
|
5700
|
+
const index = [
|
|
5701
|
+
"# Memory Index",
|
|
5702
|
+
"",
|
|
5703
|
+
"Generated automatically by MarkdownMirror.",
|
|
5704
|
+
"",
|
|
5705
|
+
...mdFiles.map((rel) => `- ${rel}`),
|
|
5706
|
+
""
|
|
5707
|
+
].join("\n");
|
|
5708
|
+
await fs3.promises.writeFile(path2.join(memoryRoot, "_index.md"), index, "utf8");
|
|
5709
|
+
}
|
|
5710
|
+
async walk(dir, out) {
|
|
5711
|
+
const entries = await fs3.promises.readdir(dir, { withFileTypes: true });
|
|
5712
|
+
for (const e of entries) {
|
|
5713
|
+
const full = path2.join(dir, e.name);
|
|
5714
|
+
if (e.isDirectory()) {
|
|
5715
|
+
await this.walk(full, out);
|
|
5716
|
+
} else {
|
|
5717
|
+
out.push(full);
|
|
5718
|
+
}
|
|
5719
|
+
}
|
|
5720
|
+
}
|
|
5721
|
+
};
|
|
5722
|
+
|
|
5723
|
+
// src/core/ingest-interceptor.ts
|
|
5724
|
+
var IngestInterceptorRegistry = class {
|
|
5725
|
+
before = [];
|
|
5726
|
+
after = [];
|
|
5727
|
+
onError = [];
|
|
5728
|
+
registerBefore(interceptor) {
|
|
5729
|
+
this.before.push(interceptor);
|
|
5730
|
+
return () => {
|
|
5731
|
+
this.before = this.before.filter((i) => i !== interceptor);
|
|
5732
|
+
};
|
|
5733
|
+
}
|
|
5734
|
+
registerAfter(interceptor) {
|
|
5735
|
+
this.after.push(interceptor);
|
|
5736
|
+
return () => {
|
|
5737
|
+
this.after = this.after.filter((i) => i !== interceptor);
|
|
5738
|
+
};
|
|
5739
|
+
}
|
|
5740
|
+
registerOnError(interceptor) {
|
|
5741
|
+
this.onError.push(interceptor);
|
|
5742
|
+
return () => {
|
|
5743
|
+
this.onError = this.onError.filter((i) => i !== interceptor);
|
|
5744
|
+
};
|
|
5745
|
+
}
|
|
5746
|
+
async run(stage, context) {
|
|
5747
|
+
const interceptors = stage === "before" ? this.before : stage === "after" ? this.after : this.onError;
|
|
5748
|
+
for (const interceptor of interceptors) {
|
|
5749
|
+
await interceptor({ ...context, stage });
|
|
5750
|
+
}
|
|
5751
|
+
}
|
|
5752
|
+
};
|
|
5753
|
+
function mergeHierarchicalMetadata(base, patch) {
|
|
5754
|
+
if (!base && !patch)
|
|
5755
|
+
return void 0;
|
|
5756
|
+
if (!base)
|
|
5757
|
+
return patch;
|
|
5758
|
+
if (!patch)
|
|
5759
|
+
return base;
|
|
5760
|
+
const result = { ...base };
|
|
5761
|
+
for (const [key, value] of Object.entries(patch)) {
|
|
5762
|
+
const current = result[key];
|
|
5763
|
+
if (typeof current === "object" && current !== null && !Array.isArray(current) && typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
5764
|
+
result[key] = mergeHierarchicalMetadata(
|
|
5765
|
+
current,
|
|
5766
|
+
value
|
|
5767
|
+
);
|
|
5768
|
+
} else {
|
|
5769
|
+
result[key] = value;
|
|
5770
|
+
}
|
|
5771
|
+
}
|
|
5772
|
+
return result;
|
|
5773
|
+
}
|
|
5774
|
+
|
|
5775
|
+
// src/core/tag-taxonomy.ts
|
|
5776
|
+
var TAG_NAMESPACES = {
|
|
5777
|
+
SYSTEM: "sys:",
|
|
5778
|
+
QUALITY: "q:",
|
|
5779
|
+
PROJECT: "proj:",
|
|
5780
|
+
TOPIC: "topic:",
|
|
5781
|
+
TEMPORAL: "t:",
|
|
5782
|
+
USER: "user:",
|
|
5783
|
+
AGENT: "agent:"
|
|
5784
|
+
};
|
|
5785
|
+
var VALID_TAG_NAMESPACES = new Set(Object.values(TAG_NAMESPACES));
|
|
5786
|
+
function parseTag(tag) {
|
|
5787
|
+
const value = (tag || "").trim();
|
|
5788
|
+
const idx = value.indexOf(":");
|
|
5789
|
+
if (idx <= 0)
|
|
5790
|
+
return { value };
|
|
5791
|
+
const namespace = `${value.slice(0, idx)}:`;
|
|
5792
|
+
const tagValue = value.slice(idx + 1);
|
|
5793
|
+
if (!tagValue)
|
|
5794
|
+
return { value };
|
|
5795
|
+
return { namespace, value: tagValue };
|
|
5796
|
+
}
|
|
5797
|
+
function validateTag(tag) {
|
|
5798
|
+
const normalized = (tag || "").trim();
|
|
5799
|
+
if (!normalized)
|
|
5800
|
+
return false;
|
|
5801
|
+
const { namespace } = parseTag(normalized);
|
|
5802
|
+
if (!namespace)
|
|
5803
|
+
return true;
|
|
5804
|
+
return VALID_TAG_NAMESPACES.has(namespace);
|
|
5805
|
+
}
|
|
5806
|
+
function normalizeTags(tags) {
|
|
5807
|
+
if (!Array.isArray(tags))
|
|
5808
|
+
return [];
|
|
5809
|
+
const dedup = /* @__PURE__ */ new Set();
|
|
5810
|
+
for (const item of tags) {
|
|
5811
|
+
if (typeof item !== "string")
|
|
5812
|
+
continue;
|
|
5813
|
+
const normalized = item.trim();
|
|
5814
|
+
if (!validateTag(normalized))
|
|
5815
|
+
continue;
|
|
5816
|
+
dedup.add(normalized);
|
|
5817
|
+
}
|
|
5818
|
+
return [...dedup];
|
|
5819
|
+
}
|
|
5820
|
+
|
|
4926
5821
|
// src/services/memory-service.ts
|
|
4927
5822
|
function normalizePath(projectPath) {
|
|
4928
|
-
const expanded = projectPath.startsWith("~") ?
|
|
5823
|
+
const expanded = projectPath.startsWith("~") ? path3.join(os.homedir(), projectPath.slice(1)) : projectPath;
|
|
4929
5824
|
try {
|
|
4930
|
-
return
|
|
5825
|
+
return fs4.realpathSync(expanded);
|
|
4931
5826
|
} catch {
|
|
4932
|
-
return
|
|
5827
|
+
return path3.resolve(expanded);
|
|
4933
5828
|
}
|
|
4934
5829
|
}
|
|
4935
5830
|
function hashProjectPath(projectPath) {
|
|
@@ -4938,14 +5833,14 @@ function hashProjectPath(projectPath) {
|
|
|
4938
5833
|
}
|
|
4939
5834
|
function getProjectStoragePath(projectPath) {
|
|
4940
5835
|
const hash = hashProjectPath(projectPath);
|
|
4941
|
-
return
|
|
5836
|
+
return path3.join(os.homedir(), ".claude-code", "memory", "projects", hash);
|
|
4942
5837
|
}
|
|
4943
|
-
var REGISTRY_PATH =
|
|
4944
|
-
var SHARED_STORAGE_PATH =
|
|
5838
|
+
var REGISTRY_PATH = path3.join(os.homedir(), ".claude-code", "memory", "session-registry.json");
|
|
5839
|
+
var SHARED_STORAGE_PATH = path3.join(os.homedir(), ".claude-code", "memory", "shared");
|
|
4945
5840
|
function loadSessionRegistry() {
|
|
4946
5841
|
try {
|
|
4947
|
-
if (
|
|
4948
|
-
const data =
|
|
5842
|
+
if (fs4.existsSync(REGISTRY_PATH)) {
|
|
5843
|
+
const data = fs4.readFileSync(REGISTRY_PATH, "utf-8");
|
|
4949
5844
|
return JSON.parse(data);
|
|
4950
5845
|
}
|
|
4951
5846
|
} catch (error) {
|
|
@@ -4954,13 +5849,13 @@ function loadSessionRegistry() {
|
|
|
4954
5849
|
return { version: 1, sessions: {} };
|
|
4955
5850
|
}
|
|
4956
5851
|
function saveSessionRegistry(registry) {
|
|
4957
|
-
const dir =
|
|
4958
|
-
if (!
|
|
4959
|
-
|
|
5852
|
+
const dir = path3.dirname(REGISTRY_PATH);
|
|
5853
|
+
if (!fs4.existsSync(dir)) {
|
|
5854
|
+
fs4.mkdirSync(dir, { recursive: true });
|
|
4960
5855
|
}
|
|
4961
5856
|
const tempPath = REGISTRY_PATH + ".tmp";
|
|
4962
|
-
|
|
4963
|
-
|
|
5857
|
+
fs4.writeFileSync(tempPath, JSON.stringify(registry, null, 2));
|
|
5858
|
+
fs4.renameSync(tempPath, REGISTRY_PATH);
|
|
4964
5859
|
}
|
|
4965
5860
|
function registerSession(sessionId, projectPath) {
|
|
4966
5861
|
const registry = loadSessionRegistry();
|
|
@@ -4992,6 +5887,7 @@ var MemoryService = class {
|
|
|
4992
5887
|
vectorWorker = null;
|
|
4993
5888
|
graduationWorker = null;
|
|
4994
5889
|
initialized = false;
|
|
5890
|
+
ingestInterceptors = new IngestInterceptorRegistry();
|
|
4995
5891
|
// Endless Mode components
|
|
4996
5892
|
workingSetStore = null;
|
|
4997
5893
|
consolidatedStore = null;
|
|
@@ -5005,20 +5901,27 @@ var MemoryService = class {
|
|
|
5005
5901
|
sharedPromoter = null;
|
|
5006
5902
|
sharedStoreConfig = null;
|
|
5007
5903
|
projectHash = null;
|
|
5904
|
+
projectPath = null;
|
|
5008
5905
|
readOnly;
|
|
5009
5906
|
lightweightMode;
|
|
5907
|
+
mdMirror;
|
|
5010
5908
|
constructor(config) {
|
|
5011
5909
|
const storagePath = this.expandPath(config.storagePath);
|
|
5012
5910
|
this.readOnly = config.readOnly ?? false;
|
|
5013
5911
|
this.lightweightMode = config.lightweightMode ?? false;
|
|
5014
|
-
|
|
5015
|
-
|
|
5912
|
+
this.mdMirror = new MarkdownMirror2(process.cwd());
|
|
5913
|
+
if (!this.readOnly && !fs4.existsSync(storagePath)) {
|
|
5914
|
+
fs4.mkdirSync(storagePath, { recursive: true });
|
|
5016
5915
|
}
|
|
5017
5916
|
this.projectHash = config.projectHash || null;
|
|
5917
|
+
this.projectPath = config.projectPath || null;
|
|
5018
5918
|
this.sharedStoreConfig = config.sharedStoreConfig ?? { enabled: true };
|
|
5019
5919
|
this.sqliteStore = new SQLiteEventStore(
|
|
5020
|
-
|
|
5021
|
-
{
|
|
5920
|
+
path3.join(storagePath, "events.sqlite"),
|
|
5921
|
+
{
|
|
5922
|
+
readonly: this.readOnly,
|
|
5923
|
+
markdownMirrorRoot: storagePath
|
|
5924
|
+
}
|
|
5022
5925
|
);
|
|
5023
5926
|
const analyticsEnabled = config.analyticsEnabled ?? this.readOnly;
|
|
5024
5927
|
if (!analyticsEnabled) {
|
|
@@ -5026,7 +5929,7 @@ var MemoryService = class {
|
|
|
5026
5929
|
} else if (this.readOnly) {
|
|
5027
5930
|
try {
|
|
5028
5931
|
this.analyticsStore = new EventStore(
|
|
5029
|
-
|
|
5932
|
+
path3.join(storagePath, "analytics.duckdb"),
|
|
5030
5933
|
{ readOnly: true }
|
|
5031
5934
|
);
|
|
5032
5935
|
} catch {
|
|
@@ -5034,11 +5937,11 @@ var MemoryService = class {
|
|
|
5034
5937
|
}
|
|
5035
5938
|
} else {
|
|
5036
5939
|
this.analyticsStore = new EventStore(
|
|
5037
|
-
|
|
5940
|
+
path3.join(storagePath, "analytics.duckdb"),
|
|
5038
5941
|
{ readOnly: false }
|
|
5039
5942
|
);
|
|
5040
5943
|
}
|
|
5041
|
-
this.vectorStore = new VectorStore(
|
|
5944
|
+
this.vectorStore = new VectorStore(path3.join(storagePath, "vectors"));
|
|
5042
5945
|
this.embedder = config.embeddingModel ? new Embedder(config.embeddingModel) : getDefaultEmbedder();
|
|
5043
5946
|
this.matcher = getDefaultMatcher();
|
|
5044
5947
|
this.retriever = createRetriever(
|
|
@@ -5048,6 +5951,7 @@ var MemoryService = class {
|
|
|
5048
5951
|
this.embedder,
|
|
5049
5952
|
this.matcher
|
|
5050
5953
|
);
|
|
5954
|
+
this.retriever.setQueryRewriter((q) => this.rewriteQueryIntent(q));
|
|
5051
5955
|
this.graduation = createGraduationPipeline(this.sqliteStore);
|
|
5052
5956
|
}
|
|
5053
5957
|
/**
|
|
@@ -5107,16 +6011,16 @@ var MemoryService = class {
|
|
|
5107
6011
|
*/
|
|
5108
6012
|
async initializeSharedStore() {
|
|
5109
6013
|
const sharedPath = this.sharedStoreConfig?.sharedStoragePath ? this.expandPath(this.sharedStoreConfig.sharedStoragePath) : SHARED_STORAGE_PATH;
|
|
5110
|
-
if (!
|
|
5111
|
-
|
|
6014
|
+
if (!fs4.existsSync(sharedPath)) {
|
|
6015
|
+
fs4.mkdirSync(sharedPath, { recursive: true });
|
|
5112
6016
|
}
|
|
5113
6017
|
this.sharedEventStore = createSharedEventStore(
|
|
5114
|
-
|
|
6018
|
+
path3.join(sharedPath, "shared.duckdb")
|
|
5115
6019
|
);
|
|
5116
6020
|
await this.sharedEventStore.initialize();
|
|
5117
6021
|
this.sharedStore = createSharedStore(this.sharedEventStore);
|
|
5118
6022
|
this.sharedVectorStore = createSharedVectorStore(
|
|
5119
|
-
|
|
6023
|
+
path3.join(sharedPath, "vectors")
|
|
5120
6024
|
);
|
|
5121
6025
|
await this.sharedVectorStore.initialize();
|
|
5122
6026
|
this.sharedPromoter = createSharedPromoter(
|
|
@@ -5127,6 +6031,86 @@ var MemoryService = class {
|
|
|
5127
6031
|
);
|
|
5128
6032
|
this.retriever.setSharedStores(this.sharedStore, this.sharedVectorStore);
|
|
5129
6033
|
}
|
|
6034
|
+
registerIngestBefore(interceptor) {
|
|
6035
|
+
return this.ingestInterceptors.registerBefore(interceptor);
|
|
6036
|
+
}
|
|
6037
|
+
registerIngestAfter(interceptor) {
|
|
6038
|
+
return this.ingestInterceptors.registerAfter(interceptor);
|
|
6039
|
+
}
|
|
6040
|
+
registerIngestOnError(interceptor) {
|
|
6041
|
+
return this.ingestInterceptors.registerOnError(interceptor);
|
|
6042
|
+
}
|
|
6043
|
+
async ingestWithInterceptors(operation, input, onSuccess) {
|
|
6044
|
+
const normalizedInput = {
|
|
6045
|
+
...input,
|
|
6046
|
+
metadata: mergeHierarchicalMetadata(
|
|
6047
|
+
{
|
|
6048
|
+
ingest: {
|
|
6049
|
+
operation,
|
|
6050
|
+
pipeline: "default",
|
|
6051
|
+
ts: (/* @__PURE__ */ new Date()).toISOString()
|
|
6052
|
+
},
|
|
6053
|
+
...this.projectHash ? {
|
|
6054
|
+
scope: {
|
|
6055
|
+
project: {
|
|
6056
|
+
hash: this.projectHash,
|
|
6057
|
+
...this.projectPath ? { path: this.projectPath } : {}
|
|
6058
|
+
}
|
|
6059
|
+
},
|
|
6060
|
+
tags: [`proj:${this.projectHash}`]
|
|
6061
|
+
} : {}
|
|
6062
|
+
},
|
|
6063
|
+
input.metadata
|
|
6064
|
+
)
|
|
6065
|
+
};
|
|
6066
|
+
if (this.projectHash && normalizedInput.metadata) {
|
|
6067
|
+
const meta = normalizedInput.metadata;
|
|
6068
|
+
const currentTags = Array.isArray(meta.tags) ? meta.tags.filter((x) => typeof x === "string") : [];
|
|
6069
|
+
const projectTag = `proj:${this.projectHash}`;
|
|
6070
|
+
if (!currentTags.includes(projectTag)) {
|
|
6071
|
+
meta.tags = [...currentTags, projectTag];
|
|
6072
|
+
}
|
|
6073
|
+
}
|
|
6074
|
+
if (normalizedInput.metadata) {
|
|
6075
|
+
const meta = normalizedInput.metadata;
|
|
6076
|
+
const normalizedTags = normalizeTags(meta.tags);
|
|
6077
|
+
if (normalizedTags.length > 0) {
|
|
6078
|
+
meta.tags = normalizedTags;
|
|
6079
|
+
}
|
|
6080
|
+
}
|
|
6081
|
+
await this.ingestInterceptors.run("before", {
|
|
6082
|
+
operation,
|
|
6083
|
+
sessionId: normalizedInput.sessionId,
|
|
6084
|
+
event: normalizedInput
|
|
6085
|
+
});
|
|
6086
|
+
try {
|
|
6087
|
+
const result = await this.sqliteStore.append(normalizedInput);
|
|
6088
|
+
if (result.success && !result.isDuplicate) {
|
|
6089
|
+
if (onSuccess) {
|
|
6090
|
+
await onSuccess(result.eventId);
|
|
6091
|
+
}
|
|
6092
|
+
try {
|
|
6093
|
+
await this.mdMirror.append(normalizedInput, result.eventId);
|
|
6094
|
+
} catch {
|
|
6095
|
+
}
|
|
6096
|
+
}
|
|
6097
|
+
await this.ingestInterceptors.run("after", {
|
|
6098
|
+
operation,
|
|
6099
|
+
sessionId: normalizedInput.sessionId,
|
|
6100
|
+
event: normalizedInput
|
|
6101
|
+
});
|
|
6102
|
+
return result;
|
|
6103
|
+
} catch (error) {
|
|
6104
|
+
const normalizedError = error instanceof Error ? error : new Error(String(error));
|
|
6105
|
+
await this.ingestInterceptors.run("error", {
|
|
6106
|
+
operation,
|
|
6107
|
+
sessionId: normalizedInput.sessionId,
|
|
6108
|
+
event: normalizedInput,
|
|
6109
|
+
error: normalizedError
|
|
6110
|
+
});
|
|
6111
|
+
throw error;
|
|
6112
|
+
}
|
|
6113
|
+
}
|
|
5130
6114
|
/**
|
|
5131
6115
|
* Start a new session
|
|
5132
6116
|
*/
|
|
@@ -5154,50 +6138,57 @@ var MemoryService = class {
|
|
|
5154
6138
|
*/
|
|
5155
6139
|
async storeUserPrompt(sessionId, content, metadata) {
|
|
5156
6140
|
await this.initialize();
|
|
5157
|
-
|
|
5158
|
-
|
|
5159
|
-
|
|
5160
|
-
|
|
5161
|
-
|
|
5162
|
-
|
|
5163
|
-
|
|
5164
|
-
|
|
5165
|
-
|
|
5166
|
-
|
|
5167
|
-
|
|
6141
|
+
return this.ingestWithInterceptors(
|
|
6142
|
+
"user_prompt",
|
|
6143
|
+
{
|
|
6144
|
+
eventType: "user_prompt",
|
|
6145
|
+
sessionId,
|
|
6146
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6147
|
+
content,
|
|
6148
|
+
metadata
|
|
6149
|
+
},
|
|
6150
|
+
async (eventId) => {
|
|
6151
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, content);
|
|
6152
|
+
}
|
|
6153
|
+
);
|
|
5168
6154
|
}
|
|
5169
6155
|
/**
|
|
5170
6156
|
* Store an agent response
|
|
5171
6157
|
*/
|
|
5172
6158
|
async storeAgentResponse(sessionId, content, metadata) {
|
|
5173
6159
|
await this.initialize();
|
|
5174
|
-
|
|
5175
|
-
|
|
5176
|
-
|
|
5177
|
-
|
|
5178
|
-
|
|
5179
|
-
|
|
5180
|
-
|
|
5181
|
-
|
|
5182
|
-
|
|
5183
|
-
|
|
5184
|
-
|
|
6160
|
+
return this.ingestWithInterceptors(
|
|
6161
|
+
"agent_response",
|
|
6162
|
+
{
|
|
6163
|
+
eventType: "agent_response",
|
|
6164
|
+
sessionId,
|
|
6165
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6166
|
+
content,
|
|
6167
|
+
metadata
|
|
6168
|
+
},
|
|
6169
|
+
async (eventId) => {
|
|
6170
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, content);
|
|
6171
|
+
}
|
|
6172
|
+
);
|
|
5185
6173
|
}
|
|
5186
6174
|
/**
|
|
5187
6175
|
* Store a session summary
|
|
5188
6176
|
*/
|
|
5189
|
-
async storeSessionSummary(sessionId, summary) {
|
|
6177
|
+
async storeSessionSummary(sessionId, summary, metadata) {
|
|
5190
6178
|
await this.initialize();
|
|
5191
|
-
|
|
5192
|
-
|
|
5193
|
-
|
|
5194
|
-
|
|
5195
|
-
|
|
5196
|
-
|
|
5197
|
-
|
|
5198
|
-
|
|
5199
|
-
|
|
5200
|
-
|
|
6179
|
+
return this.ingestWithInterceptors(
|
|
6180
|
+
"session_summary",
|
|
6181
|
+
{
|
|
6182
|
+
eventType: "session_summary",
|
|
6183
|
+
sessionId,
|
|
6184
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6185
|
+
content: summary,
|
|
6186
|
+
metadata
|
|
6187
|
+
},
|
|
6188
|
+
async (eventId) => {
|
|
6189
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, summary);
|
|
6190
|
+
}
|
|
6191
|
+
);
|
|
5201
6192
|
}
|
|
5202
6193
|
/**
|
|
5203
6194
|
* Store a tool observation
|
|
@@ -5206,40 +6197,181 @@ var MemoryService = class {
|
|
|
5206
6197
|
await this.initialize();
|
|
5207
6198
|
const content = JSON.stringify(payload);
|
|
5208
6199
|
const turnId = payload.metadata?.turnId;
|
|
5209
|
-
|
|
5210
|
-
|
|
5211
|
-
|
|
5212
|
-
|
|
5213
|
-
|
|
5214
|
-
|
|
5215
|
-
|
|
5216
|
-
|
|
5217
|
-
|
|
6200
|
+
return this.ingestWithInterceptors(
|
|
6201
|
+
"tool_observation",
|
|
6202
|
+
{
|
|
6203
|
+
eventType: "tool_observation",
|
|
6204
|
+
sessionId,
|
|
6205
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6206
|
+
content,
|
|
6207
|
+
metadata: {
|
|
6208
|
+
toolName: payload.toolName,
|
|
6209
|
+
success: payload.success,
|
|
6210
|
+
...turnId ? { turnId } : {}
|
|
6211
|
+
}
|
|
6212
|
+
},
|
|
6213
|
+
async (eventId) => {
|
|
6214
|
+
const embeddingContent = createToolObservationEmbedding(
|
|
6215
|
+
payload.toolName,
|
|
6216
|
+
payload.metadata || {},
|
|
6217
|
+
payload.success
|
|
6218
|
+
);
|
|
6219
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, embeddingContent);
|
|
5218
6220
|
}
|
|
5219
|
-
|
|
5220
|
-
if (result.success && !result.isDuplicate) {
|
|
5221
|
-
const embeddingContent = createToolObservationEmbedding(
|
|
5222
|
-
payload.toolName,
|
|
5223
|
-
payload.metadata || {},
|
|
5224
|
-
payload.success
|
|
5225
|
-
);
|
|
5226
|
-
await this.sqliteStore.enqueueForEmbedding(result.eventId, embeddingContent);
|
|
5227
|
-
}
|
|
5228
|
-
return result;
|
|
6221
|
+
);
|
|
5229
6222
|
}
|
|
5230
6223
|
/**
|
|
5231
6224
|
* Retrieve relevant memories for a query
|
|
5232
6225
|
*/
|
|
5233
6226
|
async retrieveMemories(query, options) {
|
|
5234
6227
|
await this.initialize();
|
|
6228
|
+
const rerankWeights = await this.getRerankWeights(options?.adaptiveRerank === true);
|
|
6229
|
+
let result;
|
|
5235
6230
|
if (options?.includeShared && this.sharedStore) {
|
|
5236
|
-
|
|
6231
|
+
result = await this.retriever.retrieveUnified(query, {
|
|
5237
6232
|
...options,
|
|
6233
|
+
intentRewrite: options?.intentRewrite === true,
|
|
6234
|
+
rerankWeights,
|
|
5238
6235
|
includeShared: true,
|
|
5239
|
-
projectHash: this.projectHash || void 0
|
|
6236
|
+
projectHash: this.projectHash || void 0,
|
|
6237
|
+
projectScopeMode: options?.projectScopeMode ?? (this.projectHash ? "strict" : "global"),
|
|
6238
|
+
allowedProjectHashes: options?.allowedProjectHashes
|
|
6239
|
+
});
|
|
6240
|
+
} else {
|
|
6241
|
+
result = await this.retriever.retrieve(query, {
|
|
6242
|
+
...options,
|
|
6243
|
+
intentRewrite: options?.intentRewrite === true,
|
|
6244
|
+
rerankWeights,
|
|
6245
|
+
projectHash: this.projectHash || void 0,
|
|
6246
|
+
projectScopeMode: options?.projectScopeMode ?? (this.projectHash ? "strict" : "global"),
|
|
6247
|
+
allowedProjectHashes: options?.allowedProjectHashes
|
|
6248
|
+
});
|
|
6249
|
+
}
|
|
6250
|
+
try {
|
|
6251
|
+
const selectedEventIds = result.memories.map((m) => m.event.id);
|
|
6252
|
+
const selectedDetails = (result.selectedDebug || []).map((d) => ({
|
|
6253
|
+
eventId: d.eventId,
|
|
6254
|
+
score: d.score,
|
|
6255
|
+
semanticScore: d.semanticScore,
|
|
6256
|
+
lexicalScore: d.lexicalScore,
|
|
6257
|
+
recencyScore: d.recencyScore
|
|
6258
|
+
}));
|
|
6259
|
+
const candidateDetails = (result.candidateDebug || []).map((d) => ({
|
|
6260
|
+
eventId: d.eventId,
|
|
6261
|
+
score: d.score,
|
|
6262
|
+
semanticScore: d.semanticScore,
|
|
6263
|
+
lexicalScore: d.lexicalScore,
|
|
6264
|
+
recencyScore: d.recencyScore
|
|
6265
|
+
}));
|
|
6266
|
+
const candidateEventIds = candidateDetails.length > 0 ? candidateDetails.map((d) => d.eventId) : selectedEventIds;
|
|
6267
|
+
await this.sqliteStore.recordRetrievalTrace({
|
|
6268
|
+
sessionId: options?.sessionId,
|
|
6269
|
+
projectHash: this.projectHash || void 0,
|
|
6270
|
+
queryText: query,
|
|
6271
|
+
strategy: options?.strategy || "auto",
|
|
6272
|
+
candidateEventIds,
|
|
6273
|
+
selectedEventIds,
|
|
6274
|
+
candidateDetails,
|
|
6275
|
+
selectedDetails,
|
|
6276
|
+
confidence: result.matchResult.confidence,
|
|
6277
|
+
fallbackTrace: result.fallbackTrace || []
|
|
6278
|
+
});
|
|
6279
|
+
} catch {
|
|
6280
|
+
}
|
|
6281
|
+
return result;
|
|
6282
|
+
}
|
|
6283
|
+
getConfiguredRerankWeights() {
|
|
6284
|
+
const semantic = Number(process.env.MEMORY_RERANK_WEIGHT_SEMANTIC ?? "");
|
|
6285
|
+
const lexical = Number(process.env.MEMORY_RERANK_WEIGHT_LEXICAL ?? "");
|
|
6286
|
+
const recency = Number(process.env.MEMORY_RERANK_WEIGHT_RECENCY ?? "");
|
|
6287
|
+
const allFinite = [semantic, lexical, recency].every((v) => Number.isFinite(v));
|
|
6288
|
+
if (!allFinite)
|
|
6289
|
+
return void 0;
|
|
6290
|
+
const nonNegative = [semantic, lexical, recency].every((v) => v >= 0);
|
|
6291
|
+
const total = semantic + lexical + recency;
|
|
6292
|
+
if (!nonNegative || total <= 0)
|
|
6293
|
+
return void 0;
|
|
6294
|
+
return {
|
|
6295
|
+
semantic: semantic / total,
|
|
6296
|
+
lexical: lexical / total,
|
|
6297
|
+
recency: recency / total
|
|
6298
|
+
};
|
|
6299
|
+
}
|
|
6300
|
+
async getRerankWeights(adaptive) {
|
|
6301
|
+
const configured = this.getConfiguredRerankWeights();
|
|
6302
|
+
if (configured)
|
|
6303
|
+
return configured;
|
|
6304
|
+
if (adaptive)
|
|
6305
|
+
return this.getAdaptiveRerankWeights();
|
|
6306
|
+
return void 0;
|
|
6307
|
+
}
|
|
6308
|
+
async rewriteQueryIntent(query) {
|
|
6309
|
+
if (process.env.MEMORY_INTENT_REWRITE_ENABLED !== "1")
|
|
6310
|
+
return null;
|
|
6311
|
+
const apiUrl = process.env.COMPANY_STOCK_API_URL || process.env.COMPANY_INT_API_URL;
|
|
6312
|
+
if (!apiUrl)
|
|
6313
|
+
return null;
|
|
6314
|
+
const controller = new AbortController();
|
|
6315
|
+
const timeoutMs = Number(process.env.MEMORY_INTENT_REWRITE_TIMEOUT_MS || 5e3);
|
|
6316
|
+
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
|
6317
|
+
try {
|
|
6318
|
+
const prompt = [
|
|
6319
|
+
"Rewrite user query for memory retrieval intent expansion.",
|
|
6320
|
+
"Return plain text only, one line, no markdown.",
|
|
6321
|
+
`Query: ${query}`
|
|
6322
|
+
].join("\n");
|
|
6323
|
+
const res = await fetch(apiUrl, {
|
|
6324
|
+
method: "POST",
|
|
6325
|
+
headers: {
|
|
6326
|
+
"Content-Type": "application/json",
|
|
6327
|
+
Accept: "*/*",
|
|
6328
|
+
Origin: process.env.COMPANY_INT_ORIGIN || "http://company-int.aplusai.ai",
|
|
6329
|
+
Referer: process.env.COMPANY_INT_REFERER || "http://company-int.aplusai.ai/"
|
|
6330
|
+
},
|
|
6331
|
+
body: JSON.stringify({
|
|
6332
|
+
question: prompt,
|
|
6333
|
+
company_name: null,
|
|
6334
|
+
conversation_id: null
|
|
6335
|
+
}),
|
|
6336
|
+
signal: controller.signal
|
|
5240
6337
|
});
|
|
6338
|
+
const text = (await res.text()).trim();
|
|
6339
|
+
if (!text)
|
|
6340
|
+
return null;
|
|
6341
|
+
const oneLine = text.replace(/^data:\s*/gm, "").split(/\r?\n/).map((x) => x.trim()).filter(Boolean).join(" ").slice(0, 240);
|
|
6342
|
+
if (!oneLine || oneLine.toLowerCase() === query.toLowerCase())
|
|
6343
|
+
return null;
|
|
6344
|
+
return oneLine;
|
|
6345
|
+
} catch {
|
|
6346
|
+
return null;
|
|
6347
|
+
} finally {
|
|
6348
|
+
clearTimeout(timeout);
|
|
6349
|
+
}
|
|
6350
|
+
}
|
|
6351
|
+
async getAdaptiveRerankWeights() {
|
|
6352
|
+
try {
|
|
6353
|
+
const s = await this.sqliteStore.getHelpfulnessStats();
|
|
6354
|
+
if (s.totalEvaluated < 20)
|
|
6355
|
+
return void 0;
|
|
6356
|
+
let semantic = 0.7;
|
|
6357
|
+
let lexical = 0.2;
|
|
6358
|
+
let recency = 0.1;
|
|
6359
|
+
if (s.avgScore < 0.45) {
|
|
6360
|
+
semantic -= 0.1;
|
|
6361
|
+
lexical += 0.1;
|
|
6362
|
+
} else if (s.avgScore > 0.75) {
|
|
6363
|
+
semantic += 0.05;
|
|
6364
|
+
lexical -= 0.05;
|
|
6365
|
+
}
|
|
6366
|
+
if (s.unhelpful > s.helpful) {
|
|
6367
|
+
recency += 0.05;
|
|
6368
|
+
semantic -= 0.03;
|
|
6369
|
+
lexical -= 0.02;
|
|
6370
|
+
}
|
|
6371
|
+
return { semantic, lexical, recency };
|
|
6372
|
+
} catch {
|
|
6373
|
+
return void 0;
|
|
5241
6374
|
}
|
|
5242
|
-
return this.retriever.retrieve(query, options);
|
|
5243
6375
|
}
|
|
5244
6376
|
/**
|
|
5245
6377
|
* Fast keyword search using SQLite FTS5
|
|
@@ -5281,6 +6413,18 @@ var MemoryService = class {
|
|
|
5281
6413
|
/**
|
|
5282
6414
|
* Get memory statistics
|
|
5283
6415
|
*/
|
|
6416
|
+
async getOutboxStats() {
|
|
6417
|
+
await this.initialize();
|
|
6418
|
+
return this.sqliteStore.getOutboxStats();
|
|
6419
|
+
}
|
|
6420
|
+
async getRetrievalTraceStats() {
|
|
6421
|
+
await this.initialize();
|
|
6422
|
+
return this.sqliteStore.getRetrievalTraceStats();
|
|
6423
|
+
}
|
|
6424
|
+
async getRecentRetrievalTraces(limit = 50) {
|
|
6425
|
+
await this.initialize();
|
|
6426
|
+
return this.sqliteStore.getRecentRetrievalTraces(limit);
|
|
6427
|
+
}
|
|
5284
6428
|
async getStats() {
|
|
5285
6429
|
await this.initialize();
|
|
5286
6430
|
const recentEvents = await this.sqliteStore.getRecentEvents(1e4);
|
|
@@ -5771,7 +6915,7 @@ var MemoryService = class {
|
|
|
5771
6915
|
*/
|
|
5772
6916
|
expandPath(p) {
|
|
5773
6917
|
if (p.startsWith("~")) {
|
|
5774
|
-
return
|
|
6918
|
+
return path3.join(os.homedir(), p.slice(1));
|
|
5775
6919
|
}
|
|
5776
6920
|
return p;
|
|
5777
6921
|
}
|
|
@@ -5784,6 +6928,7 @@ function getMemoryServiceForProject(projectPath, sharedStoreConfig) {
|
|
|
5784
6928
|
serviceCache.set(hash, new MemoryService({
|
|
5785
6929
|
storagePath,
|
|
5786
6930
|
projectHash: hash,
|
|
6931
|
+
projectPath,
|
|
5787
6932
|
// Override shared store config - hooks don't need DuckDB
|
|
5788
6933
|
sharedStoreConfig: sharedStoreConfig ?? { enabled: false },
|
|
5789
6934
|
analyticsEnabled: false
|