claude-memory-layer 1.0.10 → 1.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +60 -0
- package/README.md +166 -2
- package/bootstrap-kb/decisions/decisions.md +244 -0
- package/bootstrap-kb/glossary/glossary.md +46 -0
- package/bootstrap-kb/modules/.claude-plugin.md +22 -0
- package/bootstrap-kb/modules/agents.md.md +15 -0
- package/bootstrap-kb/modules/claude.md.md +15 -0
- package/bootstrap-kb/modules/context.md.md +15 -0
- package/bootstrap-kb/modules/docs.md +18 -0
- package/bootstrap-kb/modules/handoff.md.md +15 -0
- package/bootstrap-kb/modules/package-lock.json.md +15 -0
- package/bootstrap-kb/modules/package.json.md +15 -0
- package/bootstrap-kb/modules/plan.md.md +15 -0
- package/bootstrap-kb/modules/readme.md.md +15 -0
- package/bootstrap-kb/modules/scripts.md +26 -0
- package/bootstrap-kb/modules/spec.md.md +15 -0
- package/bootstrap-kb/modules/specs.md +20 -0
- package/bootstrap-kb/modules/src.md +51 -0
- package/bootstrap-kb/modules/tests.md +42 -0
- package/bootstrap-kb/modules/tsconfig.json.md +15 -0
- package/bootstrap-kb/modules/vitest.config.ts.md +15 -0
- package/bootstrap-kb/overview/overview.md +40 -0
- package/bootstrap-kb/sources/manifest.json +950 -0
- package/bootstrap-kb/sources/manifest.md +227 -0
- package/bootstrap-kb/timeline/timeline.md +57 -0
- package/d.sh +3 -0
- package/deploy.sh +3 -0
- package/dist/cli/index.js +3577 -389
- package/dist/cli/index.js.map +4 -4
- package/dist/core/index.js +1383 -138
- package/dist/core/index.js.map +4 -4
- package/dist/hooks/post-tool-use.js +1917 -214
- package/dist/hooks/post-tool-use.js.map +4 -4
- package/dist/hooks/session-end.js +1813 -231
- package/dist/hooks/session-end.js.map +4 -4
- package/dist/hooks/session-start.js +1802 -205
- package/dist/hooks/session-start.js.map +4 -4
- package/dist/hooks/stop.js +1909 -248
- package/dist/hooks/stop.js.map +4 -4
- package/dist/hooks/user-prompt-submit.js +1861 -206
- package/dist/hooks/user-prompt-submit.js.map +4 -4
- package/dist/server/api/index.js +2341 -217
- package/dist/server/api/index.js.map +4 -4
- package/dist/server/index.js +2350 -226
- package/dist/server/index.js.map +4 -4
- package/dist/services/memory-service.js +1805 -206
- package/dist/services/memory-service.js.map +4 -4
- package/dist/ui/app.js +1447 -55
- package/dist/ui/index.html +318 -147
- package/dist/ui/style.css +892 -0
- package/docs/MCP_MEMORY_SERVICE_COMPARATIVE_REVIEW.md +271 -0
- package/docs/MEMU_ADOPTION.md +40 -0
- package/docs/OPERATIONS.md +18 -0
- package/memory/.claude-plugin/commands/2026-02-25.md +263 -0
- package/memory/_index.md +405 -0
- package/memory/default/uncategorized/2026-02-25.md +4839 -0
- package/memory/specs/20260207-dashboard-upgrade/2026-02-25.md +142 -0
- package/memory/specs/citations-system/2026-02-25.md +1121 -0
- package/memory/specs/endless-mode/2026-02-25.md +1392 -0
- package/memory/specs/entity-edge-model/2026-02-25.md +1263 -0
- package/memory/specs/evidence-aligner-v2/2026-02-25.md +1028 -0
- package/memory/specs/mcp-desktop-integration/2026-02-25.md +1334 -0
- package/memory/specs/post-tool-use-hook/2026-02-25.md +1164 -0
- package/memory/specs/private-tags/2026-02-25.md +1057 -0
- package/memory/specs/progressive-disclosure/2026-02-25.md +1436 -0
- package/memory/specs/task-entity-system/2026-02-25.md +924 -0
- package/memory/specs/vector-outbox-v2/2026-02-25.md +1510 -0
- package/memory/specs/web-viewer-ui/2026-02-25.md +1709 -0
- package/package.json +9 -2
- package/scripts/build.ts +6 -0
- package/scripts/fix-sync-gap.js +32 -0
- package/scripts/heartbeat-memory-orchestrator.sh +28 -0
- package/scripts/report-sync-gap.js +26 -0
- package/scripts/review-queue-auto-resolve.js +21 -0
- package/scripts/sync-gap-auto-heal.sh +17 -0
- package/specs/20260207-dashboard-upgrade/context.md +38 -0
- package/specs/20260207-dashboard-upgrade/spec.md +96 -0
- package/src/cli/index.ts +391 -60
- package/src/core/consolidated-store.ts +63 -1
- package/src/core/consolidation-worker.ts +115 -6
- package/src/core/event-store.ts +14 -0
- package/src/core/index.ts +1 -0
- package/src/core/ingest-interceptor.ts +80 -0
- package/src/core/markdown-mirror.ts +70 -0
- package/src/core/md-mirror.ts +92 -0
- package/src/core/mongo-sync-config.ts +165 -0
- package/src/core/mongo-sync-worker.ts +381 -0
- package/src/core/retriever.ts +540 -150
- package/src/core/sqlite-event-store.ts +794 -7
- package/src/core/sqlite-wrapper.ts +8 -0
- package/src/core/tag-taxonomy.ts +51 -0
- package/src/core/turn-state.ts +159 -0
- package/src/core/types.ts +51 -8
- package/src/core/vector-store.ts +21 -3
- package/src/hooks/post-tool-use.ts +68 -23
- package/src/hooks/session-end.ts +8 -3
- package/src/hooks/stop.ts +96 -25
- package/src/hooks/user-prompt-submit.ts +44 -5
- package/src/server/api/chat.ts +244 -0
- package/src/server/api/citations.ts +3 -3
- package/src/server/api/events.ts +30 -5
- package/src/server/api/health.ts +53 -0
- package/src/server/api/index.ts +9 -1
- package/src/server/api/projects.ts +74 -0
- package/src/server/api/search.ts +3 -3
- package/src/server/api/sessions.ts +3 -3
- package/src/server/api/stats.ts +89 -8
- package/src/server/api/turns.ts +143 -0
- package/src/server/api/utils.ts +46 -0
- package/src/services/bootstrap-organizer.ts +443 -0
- package/src/services/codex-session-history-importer.ts +474 -0
- package/src/services/memory-service.ts +508 -71
- package/src/services/session-history-importer.ts +215 -51
- package/src/ui/app.js +1447 -55
- package/src/ui/index.html +318 -147
- package/src/ui/style.css +892 -0
- package/tests/bootstrap-organizer.test.ts +111 -0
- package/tests/consolidation-worker.test.ts +75 -0
- package/tests/ingest-interceptor.test.ts +38 -0
- package/tests/markdown-mirror.test.ts +85 -0
- package/tests/md-mirror.test.ts +50 -0
- package/tests/retriever-fallback-chain.test.ts +223 -0
- package/tests/retriever-strategy-scope.test.ts +97 -0
- package/tests/retriever.memu-adoption.test.ts +122 -0
- package/tests/sqlite-event-store-replication.test.ts +92 -0
- package/.claude/settings.local.json +0 -27
- package/.claude-memory/test.sqlite +0 -0
- package/.history/package_20260201112328.json +0 -45
- package/.history/package_20260201113602.json +0 -45
- package/.history/package_20260201113713.json +0 -45
- package/.history/package_20260201114110.json +0 -45
- package/.history/package_20260201114632.json +0 -46
- package/.history/package_20260201133143.json +0 -45
- package/.history/package_20260201134319.json +0 -45
- package/.history/package_20260201134326.json +0 -45
- package/.history/package_20260201134334.json +0 -45
- package/.history/package_20260201134912.json +0 -45
- package/.history/package_20260201142928.json +0 -46
- package/.history/package_20260201192048.json +0 -47
- package/.history/package_20260202114053.json +0 -49
- package/.history/package_20260202121115.json +0 -49
- package/test_access.js +0 -49
package/dist/server/api/index.js
CHANGED
|
@@ -4,17 +4,28 @@ import { dirname } from 'path';
|
|
|
4
4
|
const require = createRequire(import.meta.url);
|
|
5
5
|
const __filename = fileURLToPath(import.meta.url);
|
|
6
6
|
const __dirname = dirname(__filename);
|
|
7
|
+
var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
|
|
8
|
+
get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
|
|
9
|
+
}) : x)(function(x) {
|
|
10
|
+
if (typeof require !== "undefined")
|
|
11
|
+
return require.apply(this, arguments);
|
|
12
|
+
throw Error('Dynamic require of "' + x + '" is not supported');
|
|
13
|
+
});
|
|
7
14
|
|
|
8
15
|
// src/server/api/index.ts
|
|
9
|
-
import { Hono as
|
|
16
|
+
import { Hono as Hono10 } from "hono";
|
|
10
17
|
|
|
11
18
|
// src/server/api/sessions.ts
|
|
12
19
|
import { Hono } from "hono";
|
|
13
20
|
|
|
21
|
+
// src/server/api/utils.ts
|
|
22
|
+
import * as path4 from "path";
|
|
23
|
+
import * as os2 from "os";
|
|
24
|
+
|
|
14
25
|
// src/services/memory-service.ts
|
|
15
|
-
import * as
|
|
26
|
+
import * as path3 from "path";
|
|
16
27
|
import * as os from "os";
|
|
17
|
-
import * as
|
|
28
|
+
import * as fs4 from "fs";
|
|
18
29
|
import * as crypto2 from "crypto";
|
|
19
30
|
|
|
20
31
|
// src/core/event-store.ts
|
|
@@ -72,11 +83,11 @@ function toDate(value) {
|
|
|
72
83
|
return new Date(value);
|
|
73
84
|
return new Date(String(value));
|
|
74
85
|
}
|
|
75
|
-
function createDatabase(
|
|
86
|
+
function createDatabase(path6, options) {
|
|
76
87
|
if (options?.readOnly) {
|
|
77
|
-
return new duckdb.Database(
|
|
88
|
+
return new duckdb.Database(path6, { access_mode: "READ_ONLY" });
|
|
78
89
|
}
|
|
79
|
-
return new duckdb.Database(
|
|
90
|
+
return new duckdb.Database(path6);
|
|
80
91
|
}
|
|
81
92
|
function dbRun(db, sql, params = []) {
|
|
82
93
|
return new Promise((resolve2, reject) => {
|
|
@@ -340,6 +351,17 @@ var EventStore = class {
|
|
|
340
351
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
341
352
|
)
|
|
342
353
|
`);
|
|
354
|
+
await dbRun(this.db, `
|
|
355
|
+
CREATE TABLE IF NOT EXISTS consolidated_rules (
|
|
356
|
+
rule_id VARCHAR PRIMARY KEY,
|
|
357
|
+
rule TEXT NOT NULL,
|
|
358
|
+
topics JSON,
|
|
359
|
+
source_memory_ids JSON,
|
|
360
|
+
source_events JSON,
|
|
361
|
+
confidence FLOAT DEFAULT 0.5,
|
|
362
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
363
|
+
)
|
|
364
|
+
`);
|
|
343
365
|
await dbRun(this.db, `
|
|
344
366
|
CREATE TABLE IF NOT EXISTS endless_config (
|
|
345
367
|
key VARCHAR PRIMARY KEY,
|
|
@@ -359,6 +381,7 @@ var EventStore = class {
|
|
|
359
381
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_working_set_expires ON working_set(expires_at)`);
|
|
360
382
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_working_set_relevance ON working_set(relevance_score DESC)`);
|
|
361
383
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_consolidated_confidence ON consolidated_memories(confidence DESC)`);
|
|
384
|
+
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_consolidated_rules_confidence ON consolidated_rules(confidence DESC)`);
|
|
362
385
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_continuity_created ON continuity_log(created_at)`);
|
|
363
386
|
this.initialized = true;
|
|
364
387
|
}
|
|
@@ -746,8 +769,14 @@ import { randomUUID as randomUUID2 } from "crypto";
|
|
|
746
769
|
|
|
747
770
|
// src/core/sqlite-wrapper.ts
|
|
748
771
|
import Database from "better-sqlite3";
|
|
749
|
-
|
|
750
|
-
|
|
772
|
+
import * as fs from "fs";
|
|
773
|
+
import * as nodePath from "path";
|
|
774
|
+
function createSQLiteDatabase(path6, options) {
|
|
775
|
+
const dir = nodePath.dirname(path6);
|
|
776
|
+
if (!fs.existsSync(dir)) {
|
|
777
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
778
|
+
}
|
|
779
|
+
const db = new Database(path6, {
|
|
751
780
|
readonly: options?.readonly ?? false
|
|
752
781
|
});
|
|
753
782
|
if (!options?.readonly && (options?.walMode ?? true)) {
|
|
@@ -788,6 +817,64 @@ function toSQLiteTimestamp(date) {
|
|
|
788
817
|
return date.toISOString();
|
|
789
818
|
}
|
|
790
819
|
|
|
820
|
+
// src/core/markdown-mirror.ts
|
|
821
|
+
import * as fs2 from "fs/promises";
|
|
822
|
+
import * as path from "path";
|
|
823
|
+
var DEFAULT_NAMESPACE = "default";
|
|
824
|
+
var DEFAULT_CATEGORY = "uncategorized";
|
|
825
|
+
function sanitizeSegment(input, fallback) {
|
|
826
|
+
const raw = String(input ?? "").trim().toLowerCase();
|
|
827
|
+
const safe = raw.normalize("NFKD").replace(/[^a-z0-9_-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
828
|
+
if (!safe || safe === "." || safe === "..")
|
|
829
|
+
return fallback;
|
|
830
|
+
return safe;
|
|
831
|
+
}
|
|
832
|
+
function getCategorySegments(metadata, eventType) {
|
|
833
|
+
const raw = metadata?.categoryPath;
|
|
834
|
+
if (Array.isArray(raw) && raw.length > 0) {
|
|
835
|
+
return raw.map((s) => sanitizeSegment(s, DEFAULT_CATEGORY));
|
|
836
|
+
}
|
|
837
|
+
const single = metadata?.category;
|
|
838
|
+
if (typeof single === "string" && single.trim()) {
|
|
839
|
+
return [sanitizeSegment(single, DEFAULT_CATEGORY)];
|
|
840
|
+
}
|
|
841
|
+
return [sanitizeSegment(eventType, DEFAULT_CATEGORY)];
|
|
842
|
+
}
|
|
843
|
+
function buildMirrorPath(rootDir, event) {
|
|
844
|
+
const metadata = event.metadata;
|
|
845
|
+
const namespace = sanitizeSegment(metadata?.namespace, DEFAULT_NAMESPACE);
|
|
846
|
+
const categories = getCategorySegments(metadata, event.eventType);
|
|
847
|
+
const d = event.timestamp;
|
|
848
|
+
const yyyy = d.getFullYear();
|
|
849
|
+
const mm = String(d.getMonth() + 1).padStart(2, "0");
|
|
850
|
+
const dd = String(d.getDate()).padStart(2, "0");
|
|
851
|
+
return path.join(rootDir, "memory", namespace, ...categories, `${yyyy}-${mm}-${dd}.md`);
|
|
852
|
+
}
|
|
853
|
+
function formatMirrorEntry(event) {
|
|
854
|
+
const category = Array.isArray(event.metadata?.categoryPath) ? event.metadata.categoryPath.join("/") : String(event.metadata?.category ?? event.eventType);
|
|
855
|
+
return [
|
|
856
|
+
"",
|
|
857
|
+
`- ts: ${event.timestamp.toISOString()}`,
|
|
858
|
+
` id: ${event.id}`,
|
|
859
|
+
` type: ${event.eventType}`,
|
|
860
|
+
` session: ${event.sessionId}`,
|
|
861
|
+
` category: ${category}`,
|
|
862
|
+
" content: |",
|
|
863
|
+
...event.content.split("\n").map((line) => ` ${line}`)
|
|
864
|
+
].join("\n") + "\n";
|
|
865
|
+
}
|
|
866
|
+
var MarkdownMirror = class {
|
|
867
|
+
constructor(rootDir) {
|
|
868
|
+
this.rootDir = rootDir;
|
|
869
|
+
}
|
|
870
|
+
async append(event) {
|
|
871
|
+
const outPath = buildMirrorPath(this.rootDir, event);
|
|
872
|
+
await fs2.mkdir(path.dirname(outPath), { recursive: true });
|
|
873
|
+
await fs2.appendFile(outPath, formatMirrorEntry(event), "utf8");
|
|
874
|
+
return outPath;
|
|
875
|
+
}
|
|
876
|
+
};
|
|
877
|
+
|
|
791
878
|
// src/core/sqlite-event-store.ts
|
|
792
879
|
var SQLiteEventStore = class {
|
|
793
880
|
constructor(dbPath, options) {
|
|
@@ -797,10 +884,12 @@ var SQLiteEventStore = class {
|
|
|
797
884
|
readonly: this.readOnly,
|
|
798
885
|
walMode: !this.readOnly
|
|
799
886
|
});
|
|
887
|
+
this.markdownMirror = this.readOnly || !options?.markdownMirrorRoot ? null : new MarkdownMirror(options.markdownMirrorRoot);
|
|
800
888
|
}
|
|
801
889
|
db;
|
|
802
890
|
initialized = false;
|
|
803
891
|
readOnly;
|
|
892
|
+
markdownMirror;
|
|
804
893
|
/**
|
|
805
894
|
* Initialize database schema
|
|
806
895
|
*/
|
|
@@ -1007,6 +1096,17 @@ var SQLiteEventStore = class {
|
|
|
1007
1096
|
created_at TEXT DEFAULT (datetime('now'))
|
|
1008
1097
|
);
|
|
1009
1098
|
|
|
1099
|
+
-- Consolidated Rules table (long-term stable memory)
|
|
1100
|
+
CREATE TABLE IF NOT EXISTS consolidated_rules (
|
|
1101
|
+
rule_id TEXT PRIMARY KEY,
|
|
1102
|
+
rule TEXT NOT NULL,
|
|
1103
|
+
topics TEXT,
|
|
1104
|
+
source_memory_ids TEXT,
|
|
1105
|
+
source_events TEXT,
|
|
1106
|
+
confidence REAL DEFAULT 0.5,
|
|
1107
|
+
created_at TEXT DEFAULT (datetime('now'))
|
|
1108
|
+
);
|
|
1109
|
+
|
|
1010
1110
|
-- Endless Mode Config table
|
|
1011
1111
|
CREATE TABLE IF NOT EXISTS endless_config (
|
|
1012
1112
|
key TEXT PRIMARY KEY,
|
|
@@ -1014,6 +1114,41 @@ var SQLiteEventStore = class {
|
|
|
1014
1114
|
updated_at TEXT DEFAULT (datetime('now'))
|
|
1015
1115
|
);
|
|
1016
1116
|
|
|
1117
|
+
-- Memory Helpfulness tracking
|
|
1118
|
+
CREATE TABLE IF NOT EXISTS memory_helpfulness (
|
|
1119
|
+
id TEXT PRIMARY KEY,
|
|
1120
|
+
event_id TEXT NOT NULL,
|
|
1121
|
+
session_id TEXT NOT NULL,
|
|
1122
|
+
retrieval_score REAL DEFAULT 0,
|
|
1123
|
+
query_preview TEXT,
|
|
1124
|
+
session_continued INTEGER DEFAULT 0,
|
|
1125
|
+
prompt_count_after INTEGER DEFAULT 0,
|
|
1126
|
+
tool_success_count INTEGER DEFAULT 0,
|
|
1127
|
+
tool_total_count INTEGER DEFAULT 0,
|
|
1128
|
+
was_reasked INTEGER DEFAULT 0,
|
|
1129
|
+
helpfulness_score REAL DEFAULT 0.5,
|
|
1130
|
+
created_at TEXT DEFAULT (datetime('now')),
|
|
1131
|
+
measured_at TEXT
|
|
1132
|
+
);
|
|
1133
|
+
|
|
1134
|
+
-- Retrieval trace log (query -> candidates -> selected for context)
|
|
1135
|
+
CREATE TABLE IF NOT EXISTS retrieval_traces (
|
|
1136
|
+
trace_id TEXT PRIMARY KEY,
|
|
1137
|
+
session_id TEXT,
|
|
1138
|
+
project_hash TEXT,
|
|
1139
|
+
query_text TEXT NOT NULL,
|
|
1140
|
+
strategy TEXT,
|
|
1141
|
+
candidate_event_ids TEXT,
|
|
1142
|
+
selected_event_ids TEXT,
|
|
1143
|
+
candidate_details_json TEXT,
|
|
1144
|
+
selected_details_json TEXT,
|
|
1145
|
+
candidate_count INTEGER DEFAULT 0,
|
|
1146
|
+
selected_count INTEGER DEFAULT 0,
|
|
1147
|
+
confidence TEXT,
|
|
1148
|
+
fallback_trace TEXT,
|
|
1149
|
+
created_at TEXT DEFAULT (datetime('now'))
|
|
1150
|
+
);
|
|
1151
|
+
|
|
1017
1152
|
-- Sync position tracking (for SQLite -> DuckDB sync)
|
|
1018
1153
|
CREATE TABLE IF NOT EXISTS sync_positions (
|
|
1019
1154
|
target_name TEXT PRIMARY KEY,
|
|
@@ -1038,7 +1173,14 @@ var SQLiteEventStore = class {
|
|
|
1038
1173
|
CREATE INDEX IF NOT EXISTS idx_working_set_relevance ON working_set(relevance_score);
|
|
1039
1174
|
CREATE INDEX IF NOT EXISTS idx_consolidated_confidence ON consolidated_memories(confidence);
|
|
1040
1175
|
CREATE INDEX IF NOT EXISTS idx_continuity_created ON continuity_log(created_at);
|
|
1176
|
+
CREATE INDEX IF NOT EXISTS idx_consolidated_rules_confidence ON consolidated_rules(confidence);
|
|
1041
1177
|
CREATE INDEX IF NOT EXISTS idx_embedding_outbox_status ON embedding_outbox(status);
|
|
1178
|
+
CREATE INDEX IF NOT EXISTS idx_helpfulness_event ON memory_helpfulness(event_id);
|
|
1179
|
+
CREATE INDEX IF NOT EXISTS idx_helpfulness_session ON memory_helpfulness(session_id);
|
|
1180
|
+
CREATE INDEX IF NOT EXISTS idx_helpfulness_score ON memory_helpfulness(helpfulness_score DESC);
|
|
1181
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_created_at ON retrieval_traces(created_at DESC);
|
|
1182
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_project_hash ON retrieval_traces(project_hash);
|
|
1183
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_session_id ON retrieval_traces(session_id);
|
|
1042
1184
|
|
|
1043
1185
|
-- FTS5 Full-Text Search for fast keyword search
|
|
1044
1186
|
CREATE VIRTUAL TABLE IF NOT EXISTS events_fts USING fts5(
|
|
@@ -1062,6 +1204,14 @@ var SQLiteEventStore = class {
|
|
|
1062
1204
|
INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
|
|
1063
1205
|
END;
|
|
1064
1206
|
`);
|
|
1207
|
+
try {
|
|
1208
|
+
sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN selected_details_json TEXT;`);
|
|
1209
|
+
} catch {
|
|
1210
|
+
}
|
|
1211
|
+
try {
|
|
1212
|
+
sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN candidate_details_json TEXT;`);
|
|
1213
|
+
} catch {
|
|
1214
|
+
}
|
|
1065
1215
|
const tableInfo = sqliteAll(this.db, "PRAGMA table_info(events)", []);
|
|
1066
1216
|
const columnNames = tableInfo.map((col) => col.name);
|
|
1067
1217
|
if (!columnNames.includes("access_count")) {
|
|
@@ -1082,6 +1232,15 @@ var SQLiteEventStore = class {
|
|
|
1082
1232
|
console.error("Error adding last_accessed_at column:", err);
|
|
1083
1233
|
}
|
|
1084
1234
|
}
|
|
1235
|
+
if (!columnNames.includes("turn_id")) {
|
|
1236
|
+
try {
|
|
1237
|
+
sqliteExec(this.db, `
|
|
1238
|
+
ALTER TABLE events ADD COLUMN turn_id TEXT;
|
|
1239
|
+
`);
|
|
1240
|
+
} catch (err) {
|
|
1241
|
+
console.error("Error adding turn_id column:", err);
|
|
1242
|
+
}
|
|
1243
|
+
}
|
|
1085
1244
|
try {
|
|
1086
1245
|
sqliteExec(this.db, `
|
|
1087
1246
|
CREATE INDEX IF NOT EXISTS idx_events_access_count ON events(access_count DESC);
|
|
@@ -1094,6 +1253,12 @@ var SQLiteEventStore = class {
|
|
|
1094
1253
|
`);
|
|
1095
1254
|
} catch (err) {
|
|
1096
1255
|
}
|
|
1256
|
+
try {
|
|
1257
|
+
sqliteExec(this.db, `
|
|
1258
|
+
CREATE INDEX IF NOT EXISTS idx_events_turn_id ON events(turn_id);
|
|
1259
|
+
`);
|
|
1260
|
+
} catch (err) {
|
|
1261
|
+
}
|
|
1097
1262
|
this.initialized = true;
|
|
1098
1263
|
}
|
|
1099
1264
|
/**
|
|
@@ -1118,9 +1283,11 @@ var SQLiteEventStore = class {
|
|
|
1118
1283
|
const id = randomUUID2();
|
|
1119
1284
|
const timestamp = toSQLiteTimestamp(input.timestamp);
|
|
1120
1285
|
try {
|
|
1286
|
+
const metadata = input.metadata || {};
|
|
1287
|
+
const turnId = metadata.turnId || null;
|
|
1121
1288
|
const insertEvent = this.db.prepare(`
|
|
1122
|
-
INSERT INTO events (id, event_type, session_id, timestamp, content, canonical_key, dedupe_key, metadata)
|
|
1123
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
1289
|
+
INSERT INTO events (id, event_type, session_id, timestamp, content, canonical_key, dedupe_key, metadata, turn_id)
|
|
1290
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1124
1291
|
`);
|
|
1125
1292
|
const insertDedup = this.db.prepare(`
|
|
1126
1293
|
INSERT INTO event_dedup (dedupe_key, event_id) VALUES (?, ?)
|
|
@@ -1137,12 +1304,28 @@ var SQLiteEventStore = class {
|
|
|
1137
1304
|
input.content,
|
|
1138
1305
|
canonicalKey,
|
|
1139
1306
|
dedupeKey,
|
|
1140
|
-
JSON.stringify(
|
|
1307
|
+
JSON.stringify(metadata),
|
|
1308
|
+
turnId
|
|
1141
1309
|
);
|
|
1142
1310
|
insertDedup.run(dedupeKey, id);
|
|
1143
1311
|
insertLevel.run(id);
|
|
1144
1312
|
});
|
|
1145
1313
|
transaction();
|
|
1314
|
+
if (this.markdownMirror) {
|
|
1315
|
+
const event = {
|
|
1316
|
+
id,
|
|
1317
|
+
eventType: input.eventType,
|
|
1318
|
+
sessionId: input.sessionId,
|
|
1319
|
+
timestamp: input.timestamp,
|
|
1320
|
+
content: input.content,
|
|
1321
|
+
canonicalKey,
|
|
1322
|
+
dedupeKey,
|
|
1323
|
+
metadata
|
|
1324
|
+
};
|
|
1325
|
+
this.markdownMirror.append(event).catch((err) => {
|
|
1326
|
+
console.warn("[SQLiteEventStore] markdown mirror append failed:", err);
|
|
1327
|
+
});
|
|
1328
|
+
}
|
|
1146
1329
|
return { success: true, eventId: id, isDuplicate: false };
|
|
1147
1330
|
} catch (error) {
|
|
1148
1331
|
return {
|
|
@@ -1201,6 +1384,92 @@ var SQLiteEventStore = class {
|
|
|
1201
1384
|
);
|
|
1202
1385
|
return rows.map(this.rowToEvent);
|
|
1203
1386
|
}
|
|
1387
|
+
/**
|
|
1388
|
+
* Get events since a SQLite rowid (for robust incremental replication).
|
|
1389
|
+
* Rowid is monotonic for append-only tables, independent of client timestamps.
|
|
1390
|
+
*/
|
|
1391
|
+
async getEventsSinceRowid(lastRowid, limit = 1e3) {
|
|
1392
|
+
await this.initialize();
|
|
1393
|
+
const rows = sqliteAll(
|
|
1394
|
+
this.db,
|
|
1395
|
+
`SELECT rowid as _rowid, * FROM events WHERE rowid > ? ORDER BY rowid ASC LIMIT ?`,
|
|
1396
|
+
[lastRowid, limit]
|
|
1397
|
+
);
|
|
1398
|
+
return rows.map((row) => ({
|
|
1399
|
+
rowid: row._rowid,
|
|
1400
|
+
event: this.rowToEvent(row)
|
|
1401
|
+
}));
|
|
1402
|
+
}
|
|
1403
|
+
/**
|
|
1404
|
+
* Import events with fixed IDs (used for cross-machine replication).
|
|
1405
|
+
* Idempotent: skips if event id or dedupeKey already exists.
|
|
1406
|
+
*
|
|
1407
|
+
* NOTE: This bypasses the append() id generation to preserve stable IDs.
|
|
1408
|
+
*/
|
|
1409
|
+
async importEvents(events) {
|
|
1410
|
+
if (events.length === 0)
|
|
1411
|
+
return { inserted: 0, skipped: 0 };
|
|
1412
|
+
if (this.readOnly)
|
|
1413
|
+
return { inserted: 0, skipped: events.length };
|
|
1414
|
+
await this.initialize();
|
|
1415
|
+
const getById = this.db.prepare(`SELECT id FROM events WHERE id = ?`);
|
|
1416
|
+
const getByDedupe = this.db.prepare(`SELECT event_id FROM event_dedup WHERE dedupe_key = ?`);
|
|
1417
|
+
const insertEvent = this.db.prepare(`
|
|
1418
|
+
INSERT INTO events (id, event_type, session_id, timestamp, content, canonical_key, dedupe_key, metadata, turn_id)
|
|
1419
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1420
|
+
`);
|
|
1421
|
+
const insertDedup = this.db.prepare(`
|
|
1422
|
+
INSERT INTO event_dedup (dedupe_key, event_id) VALUES (?, ?)
|
|
1423
|
+
`);
|
|
1424
|
+
const insertLevel = this.db.prepare(`
|
|
1425
|
+
INSERT INTO memory_levels (event_id, level) VALUES (?, 'L0')
|
|
1426
|
+
`);
|
|
1427
|
+
let inserted = 0;
|
|
1428
|
+
let skipped = 0;
|
|
1429
|
+
const insertedEvents = [];
|
|
1430
|
+
const tx = this.db.transaction((batch) => {
|
|
1431
|
+
for (const ev of batch) {
|
|
1432
|
+
const existingById = getById.get(ev.id);
|
|
1433
|
+
if (existingById) {
|
|
1434
|
+
skipped++;
|
|
1435
|
+
continue;
|
|
1436
|
+
}
|
|
1437
|
+
const canonicalKey = ev.canonicalKey || makeCanonicalKey(ev.content);
|
|
1438
|
+
const dedupeKey = ev.dedupeKey || makeDedupeKey(ev.content, ev.sessionId);
|
|
1439
|
+
const existingByDedupe = getByDedupe.get(dedupeKey);
|
|
1440
|
+
if (existingByDedupe) {
|
|
1441
|
+
skipped++;
|
|
1442
|
+
continue;
|
|
1443
|
+
}
|
|
1444
|
+
const metadata = ev.metadata || {};
|
|
1445
|
+
const turnId = metadata.turnId;
|
|
1446
|
+
insertEvent.run(
|
|
1447
|
+
ev.id,
|
|
1448
|
+
ev.eventType,
|
|
1449
|
+
ev.sessionId,
|
|
1450
|
+
toSQLiteTimestamp(ev.timestamp),
|
|
1451
|
+
ev.content,
|
|
1452
|
+
canonicalKey,
|
|
1453
|
+
dedupeKey,
|
|
1454
|
+
JSON.stringify(metadata),
|
|
1455
|
+
turnId ?? null
|
|
1456
|
+
);
|
|
1457
|
+
insertDedup.run(dedupeKey, ev.id);
|
|
1458
|
+
insertLevel.run(ev.id);
|
|
1459
|
+
inserted++;
|
|
1460
|
+
insertedEvents.push(ev);
|
|
1461
|
+
}
|
|
1462
|
+
});
|
|
1463
|
+
tx(events);
|
|
1464
|
+
if (this.markdownMirror && insertedEvents.length > 0) {
|
|
1465
|
+
for (const ev of insertedEvents) {
|
|
1466
|
+
this.markdownMirror.append(ev).catch((err) => {
|
|
1467
|
+
console.warn("[SQLiteEventStore] markdown mirror append failed:", err);
|
|
1468
|
+
});
|
|
1469
|
+
}
|
|
1470
|
+
}
|
|
1471
|
+
return { inserted, skipped };
|
|
1472
|
+
}
|
|
1204
1473
|
/**
|
|
1205
1474
|
* Create or update session
|
|
1206
1475
|
*/
|
|
@@ -1363,6 +1632,35 @@ var SQLiteEventStore = class {
|
|
|
1363
1632
|
[error, ...ids]
|
|
1364
1633
|
);
|
|
1365
1634
|
}
|
|
1635
|
+
/**
|
|
1636
|
+
* Get embedding/vector outbox health statistics
|
|
1637
|
+
*/
|
|
1638
|
+
async getOutboxStats() {
|
|
1639
|
+
await this.initialize();
|
|
1640
|
+
const embeddingRows = sqliteAll(
|
|
1641
|
+
this.db,
|
|
1642
|
+
`SELECT status, COUNT(*) as count FROM embedding_outbox GROUP BY status`
|
|
1643
|
+
);
|
|
1644
|
+
const vectorRows = sqliteAll(
|
|
1645
|
+
this.db,
|
|
1646
|
+
`SELECT status, COUNT(*) as count FROM vector_outbox GROUP BY status`
|
|
1647
|
+
);
|
|
1648
|
+
const fromRows = (rows) => {
|
|
1649
|
+
const out = { pending: 0, processing: 0, failed: 0, total: 0 };
|
|
1650
|
+
for (const row of rows) {
|
|
1651
|
+
const key = row.status;
|
|
1652
|
+
if (key === "pending" || key === "processing" || key === "failed") {
|
|
1653
|
+
out[key] += row.count;
|
|
1654
|
+
}
|
|
1655
|
+
out.total += row.count;
|
|
1656
|
+
}
|
|
1657
|
+
return out;
|
|
1658
|
+
};
|
|
1659
|
+
return {
|
|
1660
|
+
embedding: fromRows(embeddingRows),
|
|
1661
|
+
vector: fromRows(vectorRows)
|
|
1662
|
+
};
|
|
1663
|
+
}
|
|
1366
1664
|
/**
|
|
1367
1665
|
* Update memory level
|
|
1368
1666
|
*/
|
|
@@ -1487,11 +1785,11 @@ var SQLiteEventStore = class {
|
|
|
1487
1785
|
);
|
|
1488
1786
|
}
|
|
1489
1787
|
/**
|
|
1490
|
-
* Get most accessed memories
|
|
1788
|
+
* Get most accessed memories (falls back to recent events if none accessed)
|
|
1491
1789
|
*/
|
|
1492
1790
|
async getMostAccessed(limit = 10) {
|
|
1493
1791
|
await this.initialize();
|
|
1494
|
-
|
|
1792
|
+
let rows = sqliteAll(
|
|
1495
1793
|
this.db,
|
|
1496
1794
|
`SELECT * FROM events
|
|
1497
1795
|
WHERE access_count > 0
|
|
@@ -1499,8 +1797,166 @@ var SQLiteEventStore = class {
|
|
|
1499
1797
|
LIMIT ?`,
|
|
1500
1798
|
[limit]
|
|
1501
1799
|
);
|
|
1800
|
+
if (rows.length === 0) {
|
|
1801
|
+
rows = sqliteAll(
|
|
1802
|
+
this.db,
|
|
1803
|
+
`SELECT * FROM events
|
|
1804
|
+
ORDER BY timestamp DESC
|
|
1805
|
+
LIMIT ?`,
|
|
1806
|
+
[limit]
|
|
1807
|
+
);
|
|
1808
|
+
}
|
|
1502
1809
|
return rows.map((row) => this.rowToEvent(row));
|
|
1503
1810
|
}
|
|
1811
|
+
/**
|
|
1812
|
+
* Record a memory retrieval for helpfulness tracking
|
|
1813
|
+
*/
|
|
1814
|
+
async recordRetrieval(eventId, sessionId, score, query) {
|
|
1815
|
+
if (this.readOnly)
|
|
1816
|
+
return;
|
|
1817
|
+
await this.initialize();
|
|
1818
|
+
const id = randomUUID2();
|
|
1819
|
+
sqliteRun(
|
|
1820
|
+
this.db,
|
|
1821
|
+
`INSERT INTO memory_helpfulness (id, event_id, session_id, retrieval_score, query_preview, created_at)
|
|
1822
|
+
VALUES (?, ?, ?, ?, ?, datetime('now'))`,
|
|
1823
|
+
[id, eventId, sessionId, score, query.slice(0, 100)]
|
|
1824
|
+
);
|
|
1825
|
+
}
|
|
1826
|
+
/**
|
|
1827
|
+
* Evaluate helpfulness for all retrievals in a session
|
|
1828
|
+
* Called at session end - uses behavioral signals to compute score
|
|
1829
|
+
*/
|
|
1830
|
+
async evaluateSessionHelpfulness(sessionId) {
|
|
1831
|
+
if (this.readOnly)
|
|
1832
|
+
return;
|
|
1833
|
+
await this.initialize();
|
|
1834
|
+
const retrievals = sqliteAll(
|
|
1835
|
+
this.db,
|
|
1836
|
+
`SELECT * FROM memory_helpfulness WHERE session_id = ? AND measured_at IS NULL`,
|
|
1837
|
+
[sessionId]
|
|
1838
|
+
);
|
|
1839
|
+
if (retrievals.length === 0)
|
|
1840
|
+
return;
|
|
1841
|
+
const sessionEvents = sqliteAll(
|
|
1842
|
+
this.db,
|
|
1843
|
+
`SELECT * FROM events WHERE session_id = ? ORDER BY timestamp ASC`,
|
|
1844
|
+
[sessionId]
|
|
1845
|
+
);
|
|
1846
|
+
const promptEvents = sessionEvents.filter((e) => e.event_type === "user_prompt");
|
|
1847
|
+
const toolEvents = sessionEvents.filter((e) => e.event_type === "tool_observation");
|
|
1848
|
+
let toolSuccessCount = 0;
|
|
1849
|
+
let toolTotalCount = toolEvents.length;
|
|
1850
|
+
for (const t of toolEvents) {
|
|
1851
|
+
try {
|
|
1852
|
+
const content = JSON.parse(t.content);
|
|
1853
|
+
if (content.success !== false)
|
|
1854
|
+
toolSuccessCount++;
|
|
1855
|
+
} catch {
|
|
1856
|
+
toolSuccessCount++;
|
|
1857
|
+
}
|
|
1858
|
+
}
|
|
1859
|
+
const toolSuccessRatio = toolTotalCount > 0 ? toolSuccessCount / toolTotalCount : 0.5;
|
|
1860
|
+
for (const retrieval of retrievals) {
|
|
1861
|
+
const retrievalTime = retrieval.created_at;
|
|
1862
|
+
const eventsAfter = sessionEvents.filter((e) => e.timestamp > retrievalTime);
|
|
1863
|
+
const sessionContinued = eventsAfter.length > 0 ? 1 : 0;
|
|
1864
|
+
const promptsAfter = promptEvents.filter((e) => e.timestamp > retrievalTime);
|
|
1865
|
+
const promptCountAfter = promptsAfter.length;
|
|
1866
|
+
const queryWords = new Set((retrieval.query_preview || "").toLowerCase().split(/\s+/).filter((w) => w.length > 2));
|
|
1867
|
+
let wasReasked = 0;
|
|
1868
|
+
for (const p of promptsAfter) {
|
|
1869
|
+
const pWords = new Set(p.content.toLowerCase().split(/\s+/).filter((w) => w.length > 2));
|
|
1870
|
+
let overlap = 0;
|
|
1871
|
+
for (const w of queryWords) {
|
|
1872
|
+
if (pWords.has(w))
|
|
1873
|
+
overlap++;
|
|
1874
|
+
}
|
|
1875
|
+
if (queryWords.size > 0 && overlap / queryWords.size > 0.5) {
|
|
1876
|
+
wasReasked = 1;
|
|
1877
|
+
break;
|
|
1878
|
+
}
|
|
1879
|
+
}
|
|
1880
|
+
const retrievalScore = retrieval.retrieval_score || 0;
|
|
1881
|
+
const helpfulnessScore = 0.3 * Math.min(retrievalScore, 1) + 0.25 * (sessionContinued ? 1 : 0) + 0.25 * toolSuccessRatio + 0.2 * (wasReasked ? 0 : 1);
|
|
1882
|
+
sqliteRun(
|
|
1883
|
+
this.db,
|
|
1884
|
+
`UPDATE memory_helpfulness
|
|
1885
|
+
SET session_continued = ?, prompt_count_after = ?,
|
|
1886
|
+
tool_success_count = ?, tool_total_count = ?,
|
|
1887
|
+
was_reasked = ?, helpfulness_score = ?,
|
|
1888
|
+
measured_at = datetime('now')
|
|
1889
|
+
WHERE id = ?`,
|
|
1890
|
+
[
|
|
1891
|
+
sessionContinued,
|
|
1892
|
+
promptCountAfter,
|
|
1893
|
+
toolSuccessCount,
|
|
1894
|
+
toolTotalCount,
|
|
1895
|
+
wasReasked,
|
|
1896
|
+
helpfulnessScore,
|
|
1897
|
+
retrieval.id
|
|
1898
|
+
]
|
|
1899
|
+
);
|
|
1900
|
+
}
|
|
1901
|
+
}
|
|
1902
|
+
/**
|
|
1903
|
+
* Get most helpful memories ranked by helpfulness score
|
|
1904
|
+
*/
|
|
1905
|
+
async getHelpfulMemories(limit = 10) {
|
|
1906
|
+
await this.initialize();
|
|
1907
|
+
const rows = sqliteAll(
|
|
1908
|
+
this.db,
|
|
1909
|
+
`SELECT
|
|
1910
|
+
mh.event_id,
|
|
1911
|
+
AVG(mh.helpfulness_score) as avg_score,
|
|
1912
|
+
COUNT(*) as eval_count,
|
|
1913
|
+
e.content,
|
|
1914
|
+
e.access_count
|
|
1915
|
+
FROM memory_helpfulness mh
|
|
1916
|
+
JOIN events e ON e.id = mh.event_id
|
|
1917
|
+
WHERE mh.measured_at IS NOT NULL
|
|
1918
|
+
GROUP BY mh.event_id
|
|
1919
|
+
ORDER BY avg_score DESC
|
|
1920
|
+
LIMIT ?`,
|
|
1921
|
+
[limit]
|
|
1922
|
+
);
|
|
1923
|
+
return rows.map((r) => ({
|
|
1924
|
+
eventId: r.event_id,
|
|
1925
|
+
summary: r.content.substring(0, 200) + (r.content.length > 200 ? "..." : ""),
|
|
1926
|
+
helpfulnessScore: Math.round(r.avg_score * 100) / 100,
|
|
1927
|
+
accessCount: r.access_count || 0,
|
|
1928
|
+
evaluationCount: r.eval_count
|
|
1929
|
+
}));
|
|
1930
|
+
}
|
|
1931
|
+
/**
|
|
1932
|
+
* Get helpfulness statistics for dashboard
|
|
1933
|
+
*/
|
|
1934
|
+
async getHelpfulnessStats() {
|
|
1935
|
+
await this.initialize();
|
|
1936
|
+
const stats = sqliteGet(
|
|
1937
|
+
this.db,
|
|
1938
|
+
`SELECT
|
|
1939
|
+
AVG(helpfulness_score) as avg_score,
|
|
1940
|
+
COUNT(*) as total_evaluated,
|
|
1941
|
+
SUM(CASE WHEN helpfulness_score >= 0.7 THEN 1 ELSE 0 END) as helpful,
|
|
1942
|
+
SUM(CASE WHEN helpfulness_score >= 0.4 AND helpfulness_score < 0.7 THEN 1 ELSE 0 END) as neutral,
|
|
1943
|
+
SUM(CASE WHEN helpfulness_score < 0.4 THEN 1 ELSE 0 END) as unhelpful
|
|
1944
|
+
FROM memory_helpfulness
|
|
1945
|
+
WHERE measured_at IS NOT NULL`
|
|
1946
|
+
);
|
|
1947
|
+
const totalRow = sqliteGet(
|
|
1948
|
+
this.db,
|
|
1949
|
+
`SELECT COUNT(*) as total FROM memory_helpfulness`
|
|
1950
|
+
);
|
|
1951
|
+
return {
|
|
1952
|
+
avgScore: Math.round((stats?.avg_score || 0) * 100) / 100,
|
|
1953
|
+
totalEvaluated: stats?.total_evaluated || 0,
|
|
1954
|
+
totalRetrievals: totalRow?.total || 0,
|
|
1955
|
+
helpful: stats?.helpful || 0,
|
|
1956
|
+
neutral: stats?.neutral || 0,
|
|
1957
|
+
unhelpful: stats?.unhelpful || 0
|
|
1958
|
+
};
|
|
1959
|
+
}
|
|
1504
1960
|
/**
|
|
1505
1961
|
* Fast keyword search using FTS5
|
|
1506
1962
|
* Returns events matching the search query, ranked by relevance
|
|
@@ -1563,12 +2019,222 @@ var SQLiteEventStore = class {
|
|
|
1563
2019
|
getDatabase() {
|
|
1564
2020
|
return this.db;
|
|
1565
2021
|
}
|
|
2022
|
+
async recordRetrievalTrace(input) {
|
|
2023
|
+
await this.initialize();
|
|
2024
|
+
const traceId = randomUUID2();
|
|
2025
|
+
sqliteRun(
|
|
2026
|
+
this.db,
|
|
2027
|
+
`INSERT INTO retrieval_traces (
|
|
2028
|
+
trace_id, session_id, project_hash, query_text, strategy,
|
|
2029
|
+
candidate_event_ids, selected_event_ids, candidate_details_json, selected_details_json,
|
|
2030
|
+
candidate_count, selected_count, confidence, fallback_trace
|
|
2031
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
2032
|
+
[
|
|
2033
|
+
traceId,
|
|
2034
|
+
input.sessionId || null,
|
|
2035
|
+
input.projectHash || null,
|
|
2036
|
+
input.queryText,
|
|
2037
|
+
input.strategy || null,
|
|
2038
|
+
JSON.stringify(input.candidateEventIds || []),
|
|
2039
|
+
JSON.stringify(input.selectedEventIds || []),
|
|
2040
|
+
JSON.stringify(input.candidateDetails || []),
|
|
2041
|
+
JSON.stringify(input.selectedDetails || []),
|
|
2042
|
+
(input.candidateEventIds || []).length,
|
|
2043
|
+
(input.selectedEventIds || []).length,
|
|
2044
|
+
input.confidence || null,
|
|
2045
|
+
JSON.stringify(input.fallbackTrace || [])
|
|
2046
|
+
]
|
|
2047
|
+
);
|
|
2048
|
+
}
|
|
2049
|
+
async getRecentRetrievalTraces(limit = 50) {
|
|
2050
|
+
await this.initialize();
|
|
2051
|
+
const rows = sqliteAll(
|
|
2052
|
+
this.db,
|
|
2053
|
+
`SELECT * FROM retrieval_traces ORDER BY created_at DESC LIMIT ?`,
|
|
2054
|
+
[limit]
|
|
2055
|
+
);
|
|
2056
|
+
return rows.map((row) => ({
|
|
2057
|
+
traceId: row.trace_id,
|
|
2058
|
+
sessionId: row.session_id || void 0,
|
|
2059
|
+
projectHash: row.project_hash || void 0,
|
|
2060
|
+
queryText: row.query_text,
|
|
2061
|
+
strategy: row.strategy || void 0,
|
|
2062
|
+
candidateEventIds: row.candidate_event_ids ? JSON.parse(row.candidate_event_ids) : [],
|
|
2063
|
+
selectedEventIds: row.selected_event_ids ? JSON.parse(row.selected_event_ids) : [],
|
|
2064
|
+
candidateDetails: row.candidate_details_json ? JSON.parse(row.candidate_details_json) : [],
|
|
2065
|
+
selectedDetails: row.selected_details_json ? JSON.parse(row.selected_details_json) : [],
|
|
2066
|
+
candidateCount: Number(row.candidate_count || 0),
|
|
2067
|
+
selectedCount: Number(row.selected_count || 0),
|
|
2068
|
+
confidence: row.confidence || void 0,
|
|
2069
|
+
fallbackTrace: row.fallback_trace ? JSON.parse(row.fallback_trace) : [],
|
|
2070
|
+
createdAt: toDateFromSQLite(row.created_at)
|
|
2071
|
+
}));
|
|
2072
|
+
}
|
|
2073
|
+
async getRetrievalTraceStats() {
|
|
2074
|
+
await this.initialize();
|
|
2075
|
+
const row = sqliteGet(
|
|
2076
|
+
this.db,
|
|
2077
|
+
`SELECT
|
|
2078
|
+
COUNT(*) as total_queries,
|
|
2079
|
+
AVG(candidate_count) as avg_candidate_count,
|
|
2080
|
+
AVG(selected_count) as avg_selected_count,
|
|
2081
|
+
CASE
|
|
2082
|
+
WHEN SUM(candidate_count) > 0 THEN (SUM(selected_count) * 1.0 / SUM(candidate_count))
|
|
2083
|
+
ELSE 0
|
|
2084
|
+
END as selection_rate
|
|
2085
|
+
FROM retrieval_traces`,
|
|
2086
|
+
[]
|
|
2087
|
+
);
|
|
2088
|
+
return {
|
|
2089
|
+
totalQueries: Number(row?.total_queries || 0),
|
|
2090
|
+
avgCandidateCount: Number(row?.avg_candidate_count || 0),
|
|
2091
|
+
avgSelectedCount: Number(row?.avg_selected_count || 0),
|
|
2092
|
+
selectionRate: Number(row?.selection_rate || 0)
|
|
2093
|
+
};
|
|
2094
|
+
}
|
|
1566
2095
|
/**
|
|
1567
2096
|
* Close database connection
|
|
1568
2097
|
*/
|
|
1569
2098
|
async close() {
|
|
1570
2099
|
sqliteClose(this.db);
|
|
1571
2100
|
}
|
|
2101
|
+
/**
|
|
2102
|
+
* Get events grouped by turn_id for a session
|
|
2103
|
+
* Returns turns ordered by first event timestamp (newest first)
|
|
2104
|
+
*/
|
|
2105
|
+
async getSessionTurns(sessionId, options) {
|
|
2106
|
+
await this.initialize();
|
|
2107
|
+
const limit = options?.limit || 20;
|
|
2108
|
+
const offset = options?.offset || 0;
|
|
2109
|
+
const turnRows = sqliteAll(
|
|
2110
|
+
this.db,
|
|
2111
|
+
`SELECT turn_id, MIN(timestamp) as min_ts
|
|
2112
|
+
FROM events
|
|
2113
|
+
WHERE session_id = ? AND turn_id IS NOT NULL
|
|
2114
|
+
GROUP BY turn_id
|
|
2115
|
+
ORDER BY min_ts DESC
|
|
2116
|
+
LIMIT ? OFFSET ?`,
|
|
2117
|
+
[sessionId, limit, offset]
|
|
2118
|
+
);
|
|
2119
|
+
const turns = [];
|
|
2120
|
+
for (const turnRow of turnRows) {
|
|
2121
|
+
const events = await this.getEventsByTurn(turnRow.turn_id);
|
|
2122
|
+
const promptEvent = events.find((e) => e.eventType === "user_prompt");
|
|
2123
|
+
const toolEvents = events.filter((e) => e.eventType === "tool_observation");
|
|
2124
|
+
const hasResponse = events.some((e) => e.eventType === "agent_response");
|
|
2125
|
+
turns.push({
|
|
2126
|
+
turnId: turnRow.turn_id,
|
|
2127
|
+
events,
|
|
2128
|
+
startedAt: toDateFromSQLite(turnRow.min_ts),
|
|
2129
|
+
promptPreview: promptEvent ? promptEvent.content.slice(0, 200) + (promptEvent.content.length > 200 ? "..." : "") : "(no prompt)",
|
|
2130
|
+
eventCount: events.length,
|
|
2131
|
+
toolCount: toolEvents.length,
|
|
2132
|
+
hasResponse
|
|
2133
|
+
});
|
|
2134
|
+
}
|
|
2135
|
+
return turns;
|
|
2136
|
+
}
|
|
2137
|
+
/**
|
|
2138
|
+
* Get all events for a specific turn_id
|
|
2139
|
+
*/
|
|
2140
|
+
async getEventsByTurn(turnId) {
|
|
2141
|
+
await this.initialize();
|
|
2142
|
+
const rows = sqliteAll(
|
|
2143
|
+
this.db,
|
|
2144
|
+
`SELECT * FROM events WHERE turn_id = ? ORDER BY timestamp ASC`,
|
|
2145
|
+
[turnId]
|
|
2146
|
+
);
|
|
2147
|
+
return rows.map(this.rowToEvent);
|
|
2148
|
+
}
|
|
2149
|
+
/**
|
|
2150
|
+
* Count total turns for a session
|
|
2151
|
+
*/
|
|
2152
|
+
async countSessionTurns(sessionId) {
|
|
2153
|
+
await this.initialize();
|
|
2154
|
+
const row = sqliteGet(
|
|
2155
|
+
this.db,
|
|
2156
|
+
`SELECT COUNT(DISTINCT turn_id) as count
|
|
2157
|
+
FROM events
|
|
2158
|
+
WHERE session_id = ? AND turn_id IS NOT NULL`,
|
|
2159
|
+
[sessionId]
|
|
2160
|
+
);
|
|
2161
|
+
return row?.count || 0;
|
|
2162
|
+
}
|
|
2163
|
+
/**
|
|
2164
|
+
* Migrate existing events: backfill turn_id for events that have turnId in metadata
|
|
2165
|
+
* but no turn_id column value (for events stored before this migration)
|
|
2166
|
+
*/
|
|
2167
|
+
async backfillTurnIds() {
|
|
2168
|
+
await this.initialize();
|
|
2169
|
+
const rows = sqliteAll(
|
|
2170
|
+
this.db,
|
|
2171
|
+
`SELECT id, metadata FROM events
|
|
2172
|
+
WHERE turn_id IS NULL AND metadata IS NOT NULL AND metadata LIKE '%turnId%'`
|
|
2173
|
+
);
|
|
2174
|
+
let updated = 0;
|
|
2175
|
+
for (const row of rows) {
|
|
2176
|
+
try {
|
|
2177
|
+
const metadata = JSON.parse(row.metadata);
|
|
2178
|
+
if (metadata.turnId) {
|
|
2179
|
+
sqliteRun(
|
|
2180
|
+
this.db,
|
|
2181
|
+
`UPDATE events SET turn_id = ? WHERE id = ?`,
|
|
2182
|
+
[metadata.turnId, row.id]
|
|
2183
|
+
);
|
|
2184
|
+
updated++;
|
|
2185
|
+
}
|
|
2186
|
+
} catch {
|
|
2187
|
+
}
|
|
2188
|
+
}
|
|
2189
|
+
return updated;
|
|
2190
|
+
}
|
|
2191
|
+
/**
|
|
2192
|
+
* Delete all events for a session (for force reimport)
|
|
2193
|
+
*/
|
|
2194
|
+
async deleteSessionEvents(sessionId) {
|
|
2195
|
+
await this.initialize();
|
|
2196
|
+
const events = sqliteAll(
|
|
2197
|
+
this.db,
|
|
2198
|
+
`SELECT id FROM events WHERE session_id = ?`,
|
|
2199
|
+
[sessionId]
|
|
2200
|
+
);
|
|
2201
|
+
if (events.length === 0)
|
|
2202
|
+
return 0;
|
|
2203
|
+
const eventIds = events.map((e) => e.id);
|
|
2204
|
+
const placeholders = eventIds.map(() => "?").join(",");
|
|
2205
|
+
const ftsTriggersDropped = [];
|
|
2206
|
+
for (const triggerName of ["events_fts_delete", "events_fts_update", "events_fts_insert"]) {
|
|
2207
|
+
try {
|
|
2208
|
+
sqliteRun(this.db, `DROP TRIGGER IF EXISTS ${triggerName}`);
|
|
2209
|
+
ftsTriggersDropped.push(triggerName);
|
|
2210
|
+
} catch {
|
|
2211
|
+
}
|
|
2212
|
+
}
|
|
2213
|
+
for (const table of ["event_dedup", "memory_levels", "embedding_queue", "embedding_outbox", "vector_outbox"]) {
|
|
2214
|
+
try {
|
|
2215
|
+
sqliteRun(this.db, `DELETE FROM ${table} WHERE event_id IN (${placeholders})`, eventIds);
|
|
2216
|
+
} catch {
|
|
2217
|
+
}
|
|
2218
|
+
}
|
|
2219
|
+
const result = sqliteRun(this.db, `DELETE FROM events WHERE session_id = ?`, [sessionId]);
|
|
2220
|
+
if (ftsTriggersDropped.length > 0) {
|
|
2221
|
+
try {
|
|
2222
|
+
sqliteRun(this.db, `INSERT INTO events_fts(events_fts) VALUES('rebuild')`);
|
|
2223
|
+
sqliteRun(this.db, `CREATE TRIGGER IF NOT EXISTS events_fts_insert AFTER INSERT ON events BEGIN
|
|
2224
|
+
INSERT INTO events_fts(rowid, content) VALUES (NEW.rowid, NEW.content);
|
|
2225
|
+
END`);
|
|
2226
|
+
sqliteRun(this.db, `CREATE TRIGGER IF NOT EXISTS events_fts_delete AFTER DELETE ON events BEGIN
|
|
2227
|
+
INSERT INTO events_fts(events_fts, rowid, content) VALUES('delete', OLD.rowid, OLD.content);
|
|
2228
|
+
END`);
|
|
2229
|
+
sqliteRun(this.db, `CREATE TRIGGER IF NOT EXISTS events_fts_update AFTER UPDATE ON events BEGIN
|
|
2230
|
+
INSERT INTO events_fts(events_fts, rowid, content) VALUES('delete', OLD.rowid, OLD.content);
|
|
2231
|
+
INSERT INTO events_fts(rowid, content) VALUES (NEW.rowid, NEW.content);
|
|
2232
|
+
END`);
|
|
2233
|
+
} catch {
|
|
2234
|
+
}
|
|
2235
|
+
}
|
|
2236
|
+
return result.changes || 0;
|
|
2237
|
+
}
|
|
1572
2238
|
/**
|
|
1573
2239
|
* Convert database row to MemoryEvent
|
|
1574
2240
|
*/
|
|
@@ -1589,6 +2255,9 @@ var SQLiteEventStore = class {
|
|
|
1589
2255
|
if (row.last_accessed_at !== void 0) {
|
|
1590
2256
|
event.last_accessed_at = row.last_accessed_at;
|
|
1591
2257
|
}
|
|
2258
|
+
if (row.turn_id !== void 0 && row.turn_id !== null) {
|
|
2259
|
+
event.turn_id = row.turn_id;
|
|
2260
|
+
}
|
|
1592
2261
|
return event;
|
|
1593
2262
|
}
|
|
1594
2263
|
};
|
|
@@ -1800,7 +2469,16 @@ var VectorStore = class {
|
|
|
1800
2469
|
metadata: JSON.stringify(record.metadata || {})
|
|
1801
2470
|
};
|
|
1802
2471
|
if (!this.table) {
|
|
1803
|
-
|
|
2472
|
+
try {
|
|
2473
|
+
this.table = await this.db.createTable(this.tableName, [data]);
|
|
2474
|
+
} catch (e) {
|
|
2475
|
+
if (e?.message?.includes("already exists")) {
|
|
2476
|
+
this.table = await this.db.openTable(this.tableName);
|
|
2477
|
+
await this.table.add([data]);
|
|
2478
|
+
} else {
|
|
2479
|
+
throw e;
|
|
2480
|
+
}
|
|
2481
|
+
}
|
|
1804
2482
|
} else {
|
|
1805
2483
|
await this.table.add([data]);
|
|
1806
2484
|
}
|
|
@@ -1826,7 +2504,16 @@ var VectorStore = class {
|
|
|
1826
2504
|
metadata: JSON.stringify(record.metadata || {})
|
|
1827
2505
|
}));
|
|
1828
2506
|
if (!this.table) {
|
|
1829
|
-
|
|
2507
|
+
try {
|
|
2508
|
+
this.table = await this.db.createTable(this.tableName, data);
|
|
2509
|
+
} catch (e) {
|
|
2510
|
+
if (e?.message?.includes("already exists")) {
|
|
2511
|
+
this.table = await this.db.openTable(this.tableName);
|
|
2512
|
+
await this.table.add(data);
|
|
2513
|
+
} else {
|
|
2514
|
+
throw e;
|
|
2515
|
+
}
|
|
2516
|
+
}
|
|
1830
2517
|
} else {
|
|
1831
2518
|
await this.table.add(data);
|
|
1832
2519
|
}
|
|
@@ -2266,7 +2953,20 @@ var DEFAULT_OPTIONS = {
|
|
|
2266
2953
|
topK: 5,
|
|
2267
2954
|
minScore: 0.7,
|
|
2268
2955
|
maxTokens: 2e3,
|
|
2269
|
-
includeSessionContext: true
|
|
2956
|
+
includeSessionContext: true,
|
|
2957
|
+
strategy: "auto",
|
|
2958
|
+
rerankWithKeyword: true,
|
|
2959
|
+
decayPolicy: {
|
|
2960
|
+
enabled: true,
|
|
2961
|
+
windowDays: 30,
|
|
2962
|
+
maxPenalty: 0.15
|
|
2963
|
+
},
|
|
2964
|
+
graphHop: {
|
|
2965
|
+
enabled: true,
|
|
2966
|
+
maxHops: 1,
|
|
2967
|
+
hopPenalty: 0.08
|
|
2968
|
+
},
|
|
2969
|
+
projectScopeMode: "global"
|
|
2270
2970
|
};
|
|
2271
2971
|
var Retriever = class {
|
|
2272
2972
|
eventStore;
|
|
@@ -2276,6 +2976,7 @@ var Retriever = class {
|
|
|
2276
2976
|
sharedStore;
|
|
2277
2977
|
sharedVectorStore;
|
|
2278
2978
|
graduation;
|
|
2979
|
+
queryRewriter;
|
|
2279
2980
|
constructor(eventStore, vectorStore, embedder, matcher, sharedOptions) {
|
|
2280
2981
|
this.eventStore = eventStore;
|
|
2281
2982
|
this.vectorStore = vectorStore;
|
|
@@ -2284,47 +2985,105 @@ var Retriever = class {
|
|
|
2284
2985
|
this.sharedStore = sharedOptions?.sharedStore;
|
|
2285
2986
|
this.sharedVectorStore = sharedOptions?.sharedVectorStore;
|
|
2286
2987
|
}
|
|
2287
|
-
/**
|
|
2288
|
-
* Set graduation pipeline for access tracking
|
|
2289
|
-
*/
|
|
2290
2988
|
setGraduationPipeline(graduation) {
|
|
2291
2989
|
this.graduation = graduation;
|
|
2292
2990
|
}
|
|
2293
|
-
/**
|
|
2294
|
-
* Set shared stores after construction
|
|
2295
|
-
*/
|
|
2296
2991
|
setSharedStores(sharedStore, sharedVectorStore) {
|
|
2297
2992
|
this.sharedStore = sharedStore;
|
|
2298
2993
|
this.sharedVectorStore = sharedVectorStore;
|
|
2299
2994
|
}
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2995
|
+
setQueryRewriter(rewriter) {
|
|
2996
|
+
this.queryRewriter = rewriter;
|
|
2997
|
+
}
|
|
2303
2998
|
async retrieve(query, options = {}) {
|
|
2304
2999
|
const opts = { ...DEFAULT_OPTIONS, ...options };
|
|
2305
|
-
const
|
|
2306
|
-
const
|
|
2307
|
-
|
|
2308
|
-
|
|
3000
|
+
const sessionFilter = opts.scope?.sessionId ?? opts.sessionId;
|
|
3001
|
+
const fallbackTrace = [];
|
|
3002
|
+
const fallbackEnabled = (opts.strategy ?? "auto") === "auto";
|
|
3003
|
+
const primaryStrategy = opts.strategy === "auto" ? "fast" : opts.strategy || "fast";
|
|
3004
|
+
let current = await this.runStage(query, {
|
|
3005
|
+
strategy: primaryStrategy,
|
|
3006
|
+
topK: opts.topK,
|
|
2309
3007
|
minScore: opts.minScore,
|
|
2310
|
-
sessionId:
|
|
3008
|
+
sessionId: sessionFilter,
|
|
3009
|
+
scope: opts.scope,
|
|
3010
|
+
rerankWithKeyword: opts.rerankWithKeyword !== false,
|
|
3011
|
+
rerankWeights: opts.rerankWeights,
|
|
3012
|
+
decayPolicy: opts.decayPolicy,
|
|
3013
|
+
intentRewrite: opts.intentRewrite === true,
|
|
3014
|
+
graphHop: opts.graphHop,
|
|
3015
|
+
projectScopeMode: opts.projectScopeMode,
|
|
3016
|
+
projectHash: opts.projectHash,
|
|
3017
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
2311
3018
|
});
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
3019
|
+
fallbackTrace.push(`stage:primary:${primaryStrategy}`);
|
|
3020
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results) && primaryStrategy !== "deep") {
|
|
3021
|
+
current = await this.runStage(query, {
|
|
3022
|
+
strategy: "deep",
|
|
3023
|
+
topK: opts.topK,
|
|
3024
|
+
minScore: opts.minScore,
|
|
3025
|
+
sessionId: sessionFilter,
|
|
3026
|
+
scope: opts.scope,
|
|
3027
|
+
rerankWithKeyword: opts.rerankWithKeyword !== false,
|
|
3028
|
+
rerankWeights: opts.rerankWeights,
|
|
3029
|
+
decayPolicy: opts.decayPolicy,
|
|
3030
|
+
graphHop: opts.graphHop,
|
|
3031
|
+
projectScopeMode: opts.projectScopeMode,
|
|
3032
|
+
projectHash: opts.projectHash,
|
|
3033
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
3034
|
+
});
|
|
3035
|
+
fallbackTrace.push("fallback:deep");
|
|
3036
|
+
}
|
|
3037
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results)) {
|
|
3038
|
+
current = await this.runStage(query, {
|
|
3039
|
+
strategy: "deep",
|
|
3040
|
+
topK: opts.topK,
|
|
3041
|
+
minScore: Math.max(0.5, opts.minScore - 0.15),
|
|
3042
|
+
sessionId: void 0,
|
|
3043
|
+
scope: void 0,
|
|
3044
|
+
rerankWithKeyword: true,
|
|
3045
|
+
rerankWeights: opts.rerankWeights,
|
|
3046
|
+
decayPolicy: opts.decayPolicy,
|
|
3047
|
+
graphHop: opts.graphHop,
|
|
3048
|
+
projectScopeMode: opts.projectScopeMode,
|
|
3049
|
+
projectHash: opts.projectHash,
|
|
3050
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
3051
|
+
});
|
|
3052
|
+
fallbackTrace.push("fallback:scope-expanded");
|
|
3053
|
+
}
|
|
3054
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results)) {
|
|
3055
|
+
const summary = await this.buildSummaryFallback(query, opts.topK);
|
|
3056
|
+
current = {
|
|
3057
|
+
results: summary,
|
|
3058
|
+
candidateResults: summary,
|
|
3059
|
+
matchResult: this.matcher.matchSearchResults(summary, () => 0)
|
|
3060
|
+
};
|
|
3061
|
+
fallbackTrace.push("fallback:summary");
|
|
3062
|
+
}
|
|
3063
|
+
const memories = await this.enrichResults(current.results.slice(0, opts.topK), opts);
|
|
2317
3064
|
const context = this.buildContext(memories, opts.maxTokens);
|
|
2318
3065
|
return {
|
|
2319
3066
|
memories,
|
|
2320
|
-
matchResult,
|
|
3067
|
+
matchResult: current.matchResult,
|
|
2321
3068
|
totalTokens: this.estimateTokens(context),
|
|
2322
|
-
context
|
|
3069
|
+
context,
|
|
3070
|
+
fallbackTrace,
|
|
3071
|
+
selectedDebug: current.results.slice(0, opts.topK).map((r) => ({
|
|
3072
|
+
eventId: r.eventId,
|
|
3073
|
+
score: r.score,
|
|
3074
|
+
semanticScore: r.semanticScore,
|
|
3075
|
+
lexicalScore: r.lexicalScore,
|
|
3076
|
+
recencyScore: r.recencyScore
|
|
3077
|
+
})),
|
|
3078
|
+
candidateDebug: (current.candidateResults || []).slice(0, Math.max(opts.topK * 3, 20)).map((r) => ({
|
|
3079
|
+
eventId: r.eventId,
|
|
3080
|
+
score: r.score,
|
|
3081
|
+
semanticScore: r.semanticScore,
|
|
3082
|
+
lexicalScore: r.lexicalScore,
|
|
3083
|
+
recencyScore: r.recencyScore
|
|
3084
|
+
}))
|
|
2323
3085
|
};
|
|
2324
3086
|
}
|
|
2325
|
-
/**
|
|
2326
|
-
* Retrieve with unified search (project + shared)
|
|
2327
|
-
*/
|
|
2328
3087
|
async retrieveUnified(query, options = {}) {
|
|
2329
3088
|
const projectResult = await this.retrieve(query, options);
|
|
2330
3089
|
if (!options.includeShared || !this.sharedStore || !this.sharedVectorStore) {
|
|
@@ -2332,22 +3091,19 @@ var Retriever = class {
|
|
|
2332
3091
|
}
|
|
2333
3092
|
try {
|
|
2334
3093
|
const queryEmbedding = await this.embedder.embed(query);
|
|
2335
|
-
const sharedVectorResults = await this.sharedVectorStore.search(
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
excludeProjectHash: options.projectHash
|
|
2341
|
-
}
|
|
2342
|
-
);
|
|
3094
|
+
const sharedVectorResults = await this.sharedVectorStore.search(queryEmbedding.vector, {
|
|
3095
|
+
limit: options.topK || 5,
|
|
3096
|
+
minScore: options.minScore || 0.7,
|
|
3097
|
+
excludeProjectHash: options.projectHash
|
|
3098
|
+
});
|
|
2343
3099
|
const sharedMemories = [];
|
|
2344
3100
|
for (const result of sharedVectorResults) {
|
|
2345
3101
|
const entry = await this.sharedStore.get(result.entryId);
|
|
2346
|
-
if (entry)
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
|
|
3102
|
+
if (!entry)
|
|
3103
|
+
continue;
|
|
3104
|
+
if (!options.projectHash || entry.sourceProjectHash !== options.projectHash) {
|
|
3105
|
+
sharedMemories.push(entry);
|
|
3106
|
+
await this.sharedStore.recordUsage(entry.entryId);
|
|
2351
3107
|
}
|
|
2352
3108
|
}
|
|
2353
3109
|
const unifiedContext = this.buildUnifiedContext(projectResult, sharedMemories);
|
|
@@ -2362,50 +3118,243 @@ var Retriever = class {
|
|
|
2362
3118
|
return projectResult;
|
|
2363
3119
|
}
|
|
2364
3120
|
}
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
|
|
2368
|
-
|
|
2369
|
-
|
|
2370
|
-
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2375
|
-
|
|
2376
|
-
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
|
|
2385
|
-
|
|
3121
|
+
async runStage(query, input) {
|
|
3122
|
+
let initialResults = await this.searchByStrategy(query, {
|
|
3123
|
+
strategy: input.strategy,
|
|
3124
|
+
topK: input.topK,
|
|
3125
|
+
minScore: input.minScore,
|
|
3126
|
+
sessionId: input.sessionId
|
|
3127
|
+
});
|
|
3128
|
+
if (input.intentRewrite && input.strategy === "deep" && this.queryRewriter) {
|
|
3129
|
+
const rewritten = (await this.queryRewriter(query))?.trim();
|
|
3130
|
+
if (rewritten && rewritten !== query) {
|
|
3131
|
+
const rewrittenResults = await this.searchByStrategy(rewritten, {
|
|
3132
|
+
strategy: "deep",
|
|
3133
|
+
topK: input.topK,
|
|
3134
|
+
minScore: Math.max(0.5, input.minScore - 0.1),
|
|
3135
|
+
sessionId: input.sessionId
|
|
3136
|
+
});
|
|
3137
|
+
initialResults = this.mergeResults(initialResults, rewrittenResults, input.topK * 3);
|
|
3138
|
+
}
|
|
3139
|
+
}
|
|
3140
|
+
const expandedResults = input.graphHop?.enabled === false ? initialResults : await this.expandGraphHops(initialResults, {
|
|
3141
|
+
maxHops: Math.max(1, input.graphHop?.maxHops ?? 1),
|
|
3142
|
+
hopPenalty: Math.max(0, input.graphHop?.hopPenalty ?? 0.08),
|
|
3143
|
+
limit: input.topK * 4
|
|
3144
|
+
});
|
|
3145
|
+
const rerankedResults = input.rerankWithKeyword ? this.rerankByKeywordOverlap(expandedResults, query, input.rerankWeights, input.decayPolicy) : expandedResults;
|
|
3146
|
+
const filtered = await this.applyScopeFilters(rerankedResults, {
|
|
3147
|
+
scope: input.scope,
|
|
3148
|
+
projectScopeMode: input.projectScopeMode,
|
|
3149
|
+
projectHash: input.projectHash,
|
|
3150
|
+
allowedProjectHashes: input.allowedProjectHashes
|
|
3151
|
+
});
|
|
3152
|
+
const top = filtered.slice(0, input.topK);
|
|
3153
|
+
const matchResult = this.matcher.matchSearchResults(top, () => 0);
|
|
3154
|
+
return { results: top, candidateResults: filtered, matchResult };
|
|
3155
|
+
}
|
|
3156
|
+
mergeResults(primary, secondary, limit) {
|
|
3157
|
+
const byId = /* @__PURE__ */ new Map();
|
|
3158
|
+
for (const row of primary)
|
|
3159
|
+
byId.set(row.eventId, row);
|
|
3160
|
+
for (const row of secondary) {
|
|
3161
|
+
const prev = byId.get(row.eventId);
|
|
3162
|
+
if (!prev || row.score > prev.score) {
|
|
3163
|
+
byId.set(row.eventId, row);
|
|
3164
|
+
}
|
|
3165
|
+
}
|
|
3166
|
+
return [...byId.values()].sort((a, b) => b.score - a.score).slice(0, limit);
|
|
3167
|
+
}
|
|
3168
|
+
async expandGraphHops(seeds, opts) {
|
|
3169
|
+
const byId = /* @__PURE__ */ new Map();
|
|
3170
|
+
for (const s of seeds)
|
|
3171
|
+
byId.set(s.eventId, s);
|
|
3172
|
+
let frontier = seeds.map((s) => ({ row: s, hop: 0 }));
|
|
3173
|
+
for (let hop = 1; hop <= opts.maxHops; hop += 1) {
|
|
3174
|
+
const next = [];
|
|
3175
|
+
for (const f of frontier) {
|
|
3176
|
+
const ev = await this.eventStore.getEvent(f.row.eventId);
|
|
3177
|
+
if (!ev)
|
|
3178
|
+
continue;
|
|
3179
|
+
const rel = ev.metadata?.relatedEventIds ?? [];
|
|
3180
|
+
const relatedIds = Array.isArray(rel) ? rel.filter((x) => typeof x === "string") : [];
|
|
3181
|
+
for (const rid of relatedIds) {
|
|
3182
|
+
if (byId.has(rid))
|
|
3183
|
+
continue;
|
|
3184
|
+
const target = await this.eventStore.getEvent(rid);
|
|
3185
|
+
if (!target)
|
|
3186
|
+
continue;
|
|
3187
|
+
const score = Math.max(0, f.row.score - opts.hopPenalty * hop);
|
|
3188
|
+
const row = {
|
|
3189
|
+
id: `hop-${hop}-${rid}`,
|
|
3190
|
+
eventId: target.id,
|
|
3191
|
+
content: target.content,
|
|
3192
|
+
score,
|
|
3193
|
+
sessionId: target.sessionId,
|
|
3194
|
+
eventType: target.eventType,
|
|
3195
|
+
timestamp: target.timestamp.toISOString()
|
|
3196
|
+
};
|
|
3197
|
+
byId.set(row.eventId, row);
|
|
3198
|
+
next.push({ row, hop });
|
|
3199
|
+
if (byId.size >= opts.limit)
|
|
3200
|
+
break;
|
|
2386
3201
|
}
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
`;
|
|
3202
|
+
if (byId.size >= opts.limit)
|
|
3203
|
+
break;
|
|
2390
3204
|
}
|
|
3205
|
+
frontier = next;
|
|
3206
|
+
if (frontier.length === 0 || byId.size >= opts.limit)
|
|
3207
|
+
break;
|
|
2391
3208
|
}
|
|
2392
|
-
return
|
|
3209
|
+
return [...byId.values()].sort((a, b) => b.score - a.score).slice(0, opts.limit);
|
|
3210
|
+
}
|
|
3211
|
+
shouldFallback(matchResult, results) {
|
|
3212
|
+
if (results.length === 0)
|
|
3213
|
+
return true;
|
|
3214
|
+
if (matchResult.confidence === "none")
|
|
3215
|
+
return true;
|
|
3216
|
+
return false;
|
|
3217
|
+
}
|
|
3218
|
+
async buildSummaryFallback(query, topK) {
|
|
3219
|
+
const recent = await this.eventStore.getRecentEvents(Math.max(topK * 6, 20));
|
|
3220
|
+
const q = this.tokenize(query);
|
|
3221
|
+
const ranked = recent.map((e) => ({ e, overlap: this.keywordOverlap(q, this.tokenize(e.content)) })).filter((r) => r.overlap > 0).sort((a, b) => b.overlap - a.overlap).slice(0, topK).map((row, idx) => ({
|
|
3222
|
+
id: `summary-${row.e.id}`,
|
|
3223
|
+
eventId: row.e.id,
|
|
3224
|
+
content: row.e.content,
|
|
3225
|
+
score: Math.max(0.25, 0.6 - idx * 0.05),
|
|
3226
|
+
sessionId: row.e.sessionId,
|
|
3227
|
+
eventType: row.e.eventType,
|
|
3228
|
+
timestamp: row.e.timestamp.toISOString()
|
|
3229
|
+
}));
|
|
3230
|
+
return ranked;
|
|
3231
|
+
}
|
|
3232
|
+
async searchByStrategy(query, input) {
|
|
3233
|
+
const strategy = input.strategy === "auto" ? "deep" : input.strategy;
|
|
3234
|
+
if (strategy === "fast") {
|
|
3235
|
+
const keyword = await this.searchByKeyword(query, {
|
|
3236
|
+
limit: Math.max(5, input.topK * 3),
|
|
3237
|
+
sessionId: input.sessionId
|
|
3238
|
+
});
|
|
3239
|
+
return keyword;
|
|
3240
|
+
}
|
|
3241
|
+
const queryEmbedding = await this.embedder.embed(query);
|
|
3242
|
+
return this.vectorStore.search(queryEmbedding.vector, {
|
|
3243
|
+
limit: Math.max(5, input.topK * 3),
|
|
3244
|
+
minScore: input.minScore,
|
|
3245
|
+
sessionId: input.sessionId
|
|
3246
|
+
});
|
|
3247
|
+
}
|
|
3248
|
+
async searchByKeyword(query, input) {
|
|
3249
|
+
if (this.eventStore.keywordSearch) {
|
|
3250
|
+
const rows = await this.eventStore.keywordSearch(query, input.limit);
|
|
3251
|
+
const filtered2 = input.sessionId ? rows.filter((r) => r.event.sessionId === input.sessionId) : rows;
|
|
3252
|
+
return filtered2.map((row, idx) => ({
|
|
3253
|
+
id: `kw-${row.event.id}`,
|
|
3254
|
+
eventId: row.event.id,
|
|
3255
|
+
content: row.event.content,
|
|
3256
|
+
score: Math.max(0.4, 1 - idx * 0.04),
|
|
3257
|
+
sessionId: row.event.sessionId,
|
|
3258
|
+
eventType: row.event.eventType,
|
|
3259
|
+
timestamp: row.event.timestamp.toISOString()
|
|
3260
|
+
}));
|
|
3261
|
+
}
|
|
3262
|
+
const recent = await this.eventStore.getRecentEvents(input.limit * 4);
|
|
3263
|
+
const tokens = this.tokenize(query);
|
|
3264
|
+
const filtered = recent.filter((e) => input.sessionId ? e.sessionId === input.sessionId : true).map((e) => ({ e, overlap: this.keywordOverlap(tokens, this.tokenize(e.content)) })).filter((r) => r.overlap > 0).sort((a, b) => b.overlap - a.overlap).slice(0, input.limit);
|
|
3265
|
+
return filtered.map((row, idx) => ({
|
|
3266
|
+
id: `kw-fallback-${row.e.id}`,
|
|
3267
|
+
eventId: row.e.id,
|
|
3268
|
+
content: row.e.content,
|
|
3269
|
+
score: Math.max(0.3, 0.9 - idx * 0.05),
|
|
3270
|
+
sessionId: row.e.sessionId,
|
|
3271
|
+
eventType: row.e.eventType,
|
|
3272
|
+
timestamp: row.e.timestamp.toISOString()
|
|
3273
|
+
}));
|
|
3274
|
+
}
|
|
3275
|
+
rerankByKeywordOverlap(results, query, weights, decayPolicy) {
|
|
3276
|
+
const q = this.tokenize(query);
|
|
3277
|
+
const now = Date.now();
|
|
3278
|
+
const sw = Math.max(0, weights?.semantic ?? 0.7);
|
|
3279
|
+
const lw = Math.max(0, weights?.lexical ?? 0.2);
|
|
3280
|
+
const rw = Math.max(0, weights?.recency ?? 0.1);
|
|
3281
|
+
const total = sw + lw + rw || 1;
|
|
3282
|
+
const decayEnabled = decayPolicy?.enabled !== false;
|
|
3283
|
+
const decayWindow = Math.max(1, decayPolicy?.windowDays ?? 30);
|
|
3284
|
+
const decayMaxPenalty = Math.max(0, decayPolicy?.maxPenalty ?? 0.15);
|
|
3285
|
+
return [...results].map((r) => {
|
|
3286
|
+
const overlap = this.keywordOverlap(q, this.tokenize(r.content));
|
|
3287
|
+
const recencyDays = Math.max(0, (now - new Date(r.timestamp).getTime()) / (1e3 * 60 * 60 * 24));
|
|
3288
|
+
const recency = Math.max(0, 1 - recencyDays / decayWindow);
|
|
3289
|
+
let blended = (r.score * sw + overlap * lw + recency * rw) / total;
|
|
3290
|
+
if (decayEnabled && recencyDays > decayWindow && overlap < 0.5) {
|
|
3291
|
+
const ageFactor = Math.min(1, (recencyDays - decayWindow) / decayWindow);
|
|
3292
|
+
blended -= decayMaxPenalty * ageFactor;
|
|
3293
|
+
}
|
|
3294
|
+
return { ...r, score: Math.max(0, blended), semanticScore: r.score, lexicalScore: overlap, recencyScore: recency };
|
|
3295
|
+
}).sort((a, b) => b.score - a.score);
|
|
3296
|
+
}
|
|
3297
|
+
async applyScopeFilters(results, options) {
|
|
3298
|
+
const scope = options?.scope;
|
|
3299
|
+
const projectScopeMode = options?.projectScopeMode ?? "global";
|
|
3300
|
+
const allowedProjectHashes = new Set(
|
|
3301
|
+
[options?.projectHash, ...options?.allowedProjectHashes || []].filter(
|
|
3302
|
+
(value) => typeof value === "string" && value.length > 0
|
|
3303
|
+
)
|
|
3304
|
+
);
|
|
3305
|
+
if (!scope && projectScopeMode === "global")
|
|
3306
|
+
return results;
|
|
3307
|
+
const normalizedIncludes = (scope?.contentIncludes || []).map((s) => s.toLowerCase());
|
|
3308
|
+
const filtered = [];
|
|
3309
|
+
for (const result of results) {
|
|
3310
|
+
if (scope?.sessionId && result.sessionId !== scope.sessionId)
|
|
3311
|
+
continue;
|
|
3312
|
+
if (scope?.sessionIdPrefix && !result.sessionId.startsWith(scope.sessionIdPrefix))
|
|
3313
|
+
continue;
|
|
3314
|
+
if (scope?.eventTypes && scope.eventTypes.length > 0 && !scope.eventTypes.includes(result.eventType))
|
|
3315
|
+
continue;
|
|
3316
|
+
const event = await this.eventStore.getEvent(result.eventId);
|
|
3317
|
+
if (!event)
|
|
3318
|
+
continue;
|
|
3319
|
+
if (scope?.canonicalKeyPrefix && !event.canonicalKey.startsWith(scope.canonicalKeyPrefix))
|
|
3320
|
+
continue;
|
|
3321
|
+
if (normalizedIncludes.length > 0) {
|
|
3322
|
+
const lc = event.content.toLowerCase();
|
|
3323
|
+
if (!normalizedIncludes.some((needle) => lc.includes(needle)))
|
|
3324
|
+
continue;
|
|
3325
|
+
}
|
|
3326
|
+
if (scope?.metadata && !this.matchesMetadataScope(event.metadata, scope.metadata))
|
|
3327
|
+
continue;
|
|
3328
|
+
const projectHash = this.extractProjectHash(event.metadata);
|
|
3329
|
+
filtered.push({ result, projectHash });
|
|
3330
|
+
}
|
|
3331
|
+
if (projectScopeMode === "global" || allowedProjectHashes.size === 0) {
|
|
3332
|
+
return filtered.map((x) => x.result);
|
|
3333
|
+
}
|
|
3334
|
+
const projectMatched = filtered.filter((x) => x.projectHash && allowedProjectHashes.has(x.projectHash));
|
|
3335
|
+
if (projectScopeMode === "strict") {
|
|
3336
|
+
return projectMatched.map((x) => x.result);
|
|
3337
|
+
}
|
|
3338
|
+
return (projectMatched.length > 0 ? projectMatched : filtered).map((x) => x.result);
|
|
3339
|
+
}
|
|
3340
|
+
extractProjectHash(metadata) {
|
|
3341
|
+
if (!metadata || typeof metadata !== "object")
|
|
3342
|
+
return void 0;
|
|
3343
|
+
const scope = metadata.scope;
|
|
3344
|
+
if (!scope || typeof scope !== "object")
|
|
3345
|
+
return void 0;
|
|
3346
|
+
const project = scope.project;
|
|
3347
|
+
if (!project || typeof project !== "object")
|
|
3348
|
+
return void 0;
|
|
3349
|
+
const hash = project.hash;
|
|
3350
|
+
return typeof hash === "string" && hash.length > 0 ? hash : void 0;
|
|
2393
3351
|
}
|
|
2394
|
-
/**
|
|
2395
|
-
* Retrieve memories from a specific session
|
|
2396
|
-
*/
|
|
2397
3352
|
async retrieveFromSession(sessionId) {
|
|
2398
3353
|
return this.eventStore.getSessionEvents(sessionId);
|
|
2399
3354
|
}
|
|
2400
|
-
/**
|
|
2401
|
-
* Get recent memories across all sessions
|
|
2402
|
-
*/
|
|
2403
3355
|
async retrieveRecent(limit = 100) {
|
|
2404
3356
|
return this.eventStore.getRecentEvents(limit);
|
|
2405
3357
|
}
|
|
2406
|
-
/**
|
|
2407
|
-
* Enrich search results with full event data
|
|
2408
|
-
*/
|
|
2409
3358
|
async enrichResults(results, options) {
|
|
2410
3359
|
const memories = [];
|
|
2411
3360
|
for (const result of results) {
|
|
@@ -2413,27 +3362,16 @@ var Retriever = class {
|
|
|
2413
3362
|
if (!event)
|
|
2414
3363
|
continue;
|
|
2415
3364
|
if (this.graduation) {
|
|
2416
|
-
this.graduation.recordAccess(
|
|
2417
|
-
event.id,
|
|
2418
|
-
options.sessionId || "unknown",
|
|
2419
|
-
result.score
|
|
2420
|
-
);
|
|
3365
|
+
this.graduation.recordAccess(event.id, options.sessionId || "unknown", result.score);
|
|
2421
3366
|
}
|
|
2422
3367
|
let sessionContext;
|
|
2423
3368
|
if (options.includeSessionContext) {
|
|
2424
3369
|
sessionContext = await this.getSessionContext(event.sessionId, event.id);
|
|
2425
3370
|
}
|
|
2426
|
-
memories.push({
|
|
2427
|
-
event,
|
|
2428
|
-
score: result.score,
|
|
2429
|
-
sessionContext
|
|
2430
|
-
});
|
|
3371
|
+
memories.push({ event, score: result.score, sessionContext });
|
|
2431
3372
|
}
|
|
2432
3373
|
return memories;
|
|
2433
3374
|
}
|
|
2434
|
-
/**
|
|
2435
|
-
* Get surrounding context from the same session
|
|
2436
|
-
*/
|
|
2437
3375
|
async getSessionContext(sessionId, eventId) {
|
|
2438
3376
|
const sessionEvents = await this.eventStore.getSessionEvents(sessionId);
|
|
2439
3377
|
const eventIndex = sessionEvents.findIndex((e) => e.id === eventId);
|
|
@@ -2446,55 +3384,86 @@ var Retriever = class {
|
|
|
2446
3384
|
return void 0;
|
|
2447
3385
|
return contextEvents.filter((e) => e.id !== eventId).map((e) => `[${e.eventType}]: ${e.content.slice(0, 200)}...`).join("\n");
|
|
2448
3386
|
}
|
|
2449
|
-
|
|
2450
|
-
|
|
2451
|
-
|
|
3387
|
+
buildUnifiedContext(projectResult, sharedMemories) {
|
|
3388
|
+
let context = projectResult.context;
|
|
3389
|
+
if (sharedMemories.length === 0)
|
|
3390
|
+
return context;
|
|
3391
|
+
context += "\n\n## Cross-Project Knowledge\n\n";
|
|
3392
|
+
for (const memory of sharedMemories.slice(0, 3)) {
|
|
3393
|
+
context += `### ${memory.title}
|
|
3394
|
+
`;
|
|
3395
|
+
if (memory.symptoms.length > 0)
|
|
3396
|
+
context += `**Symptoms:** ${memory.symptoms.join(", ")}
|
|
3397
|
+
`;
|
|
3398
|
+
context += `**Root Cause:** ${memory.rootCause}
|
|
3399
|
+
`;
|
|
3400
|
+
context += `**Solution:** ${memory.solution}
|
|
3401
|
+
`;
|
|
3402
|
+
if (memory.technologies && memory.technologies.length > 0)
|
|
3403
|
+
context += `**Technologies:** ${memory.technologies.join(", ")}
|
|
3404
|
+
`;
|
|
3405
|
+
context += `_Confidence: ${(memory.confidence * 100).toFixed(0)}%_
|
|
3406
|
+
|
|
3407
|
+
`;
|
|
3408
|
+
}
|
|
3409
|
+
return context;
|
|
3410
|
+
}
|
|
2452
3411
|
buildContext(memories, maxTokens) {
|
|
2453
3412
|
const parts = [];
|
|
2454
3413
|
let currentTokens = 0;
|
|
2455
3414
|
for (const memory of memories) {
|
|
2456
3415
|
const memoryText = this.formatMemory(memory);
|
|
2457
3416
|
const memoryTokens = this.estimateTokens(memoryText);
|
|
2458
|
-
if (currentTokens + memoryTokens > maxTokens)
|
|
3417
|
+
if (currentTokens + memoryTokens > maxTokens)
|
|
2459
3418
|
break;
|
|
2460
|
-
}
|
|
2461
3419
|
parts.push(memoryText);
|
|
2462
3420
|
currentTokens += memoryTokens;
|
|
2463
3421
|
}
|
|
2464
|
-
if (parts.length === 0)
|
|
3422
|
+
if (parts.length === 0)
|
|
2465
3423
|
return "";
|
|
2466
|
-
}
|
|
2467
3424
|
return `## Relevant Memories
|
|
2468
3425
|
|
|
2469
3426
|
${parts.join("\n\n---\n\n")}`;
|
|
2470
3427
|
}
|
|
2471
|
-
/**
|
|
2472
|
-
* Format a single memory for context
|
|
2473
|
-
*/
|
|
2474
3428
|
formatMemory(memory) {
|
|
2475
3429
|
const { event, score, sessionContext } = memory;
|
|
2476
3430
|
const date = event.timestamp.toISOString().split("T")[0];
|
|
2477
3431
|
let text = `**${event.eventType}** (${date}, score: ${score.toFixed(2)})
|
|
2478
3432
|
${event.content}`;
|
|
2479
|
-
if (sessionContext)
|
|
3433
|
+
if (sessionContext)
|
|
2480
3434
|
text += `
|
|
2481
3435
|
|
|
2482
3436
|
_Context:_ ${sessionContext}`;
|
|
2483
|
-
}
|
|
2484
3437
|
return text;
|
|
2485
3438
|
}
|
|
2486
|
-
|
|
2487
|
-
|
|
2488
|
-
|
|
3439
|
+
matchesMetadataScope(metadata, expected) {
|
|
3440
|
+
if (!metadata)
|
|
3441
|
+
return false;
|
|
3442
|
+
return Object.entries(expected).every(([path6, value]) => {
|
|
3443
|
+
const actual = path6.split(".").reduce((acc, key) => {
|
|
3444
|
+
if (typeof acc !== "object" || acc === null)
|
|
3445
|
+
return void 0;
|
|
3446
|
+
return acc[key];
|
|
3447
|
+
}, metadata);
|
|
3448
|
+
return actual === value;
|
|
3449
|
+
});
|
|
3450
|
+
}
|
|
3451
|
+
tokenize(text) {
|
|
3452
|
+
return text.toLowerCase().replace(/[^\p{L}\p{N}\s]/gu, " ").split(/\s+/).filter((t) => t.length >= 2).slice(0, 64);
|
|
3453
|
+
}
|
|
3454
|
+
keywordOverlap(a, b) {
|
|
3455
|
+
if (a.length === 0 || b.length === 0)
|
|
3456
|
+
return 0;
|
|
3457
|
+
const bs = new Set(b);
|
|
3458
|
+
let hit = 0;
|
|
3459
|
+
for (const t of a)
|
|
3460
|
+
if (bs.has(t))
|
|
3461
|
+
hit += 1;
|
|
3462
|
+
return hit / a.length;
|
|
3463
|
+
}
|
|
2489
3464
|
estimateTokens(text) {
|
|
2490
3465
|
return Math.ceil(text.length / 4);
|
|
2491
3466
|
}
|
|
2492
|
-
/**
|
|
2493
|
-
* Get event age in days (for recency scoring)
|
|
2494
|
-
*/
|
|
2495
|
-
getEventAgeDays(eventId) {
|
|
2496
|
-
return 0;
|
|
2497
|
-
}
|
|
2498
3467
|
};
|
|
2499
3468
|
function createRetriever(eventStore, vectorStore, embedder, matcher) {
|
|
2500
3469
|
return new Retriever(eventStore, vectorStore, embedder, matcher);
|
|
@@ -3784,6 +4753,59 @@ var ConsolidatedStore = class {
|
|
|
3784
4753
|
[memoryId]
|
|
3785
4754
|
);
|
|
3786
4755
|
}
|
|
4756
|
+
/**
|
|
4757
|
+
* Create a long-term rule promoted from stable summaries
|
|
4758
|
+
*/
|
|
4759
|
+
async createRule(input) {
|
|
4760
|
+
const ruleId = randomUUID6();
|
|
4761
|
+
await dbRun(
|
|
4762
|
+
this.db,
|
|
4763
|
+
`INSERT INTO consolidated_rules
|
|
4764
|
+
(rule_id, rule, topics, source_memory_ids, source_events, confidence, created_at)
|
|
4765
|
+
VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)`,
|
|
4766
|
+
[
|
|
4767
|
+
ruleId,
|
|
4768
|
+
input.rule,
|
|
4769
|
+
JSON.stringify(input.topics),
|
|
4770
|
+
JSON.stringify(input.sourceMemoryIds),
|
|
4771
|
+
JSON.stringify(input.sourceEvents),
|
|
4772
|
+
input.confidence
|
|
4773
|
+
]
|
|
4774
|
+
);
|
|
4775
|
+
return ruleId;
|
|
4776
|
+
}
|
|
4777
|
+
async getRules(options) {
|
|
4778
|
+
const limit = options?.limit || 100;
|
|
4779
|
+
const rows = await dbAll(
|
|
4780
|
+
this.db,
|
|
4781
|
+
`SELECT * FROM consolidated_rules ORDER BY confidence DESC, created_at DESC LIMIT ?`,
|
|
4782
|
+
[limit]
|
|
4783
|
+
);
|
|
4784
|
+
return rows.map((row) => ({
|
|
4785
|
+
ruleId: row.rule_id,
|
|
4786
|
+
rule: row.rule,
|
|
4787
|
+
topics: JSON.parse(row.topics || "[]"),
|
|
4788
|
+
sourceMemoryIds: JSON.parse(row.source_memory_ids || "[]"),
|
|
4789
|
+
sourceEvents: JSON.parse(row.source_events || "[]"),
|
|
4790
|
+
confidence: Number(row.confidence ?? 0.5),
|
|
4791
|
+
createdAt: toDate(row.created_at) || /* @__PURE__ */ new Date()
|
|
4792
|
+
}));
|
|
4793
|
+
}
|
|
4794
|
+
async countRules() {
|
|
4795
|
+
const result = await dbAll(
|
|
4796
|
+
this.db,
|
|
4797
|
+
`SELECT COUNT(*) as count FROM consolidated_rules`
|
|
4798
|
+
);
|
|
4799
|
+
return result[0]?.count || 0;
|
|
4800
|
+
}
|
|
4801
|
+
async hasRuleForSourceMemory(memoryId) {
|
|
4802
|
+
const rows = await dbAll(
|
|
4803
|
+
this.db,
|
|
4804
|
+
`SELECT COUNT(*) as count FROM consolidated_rules WHERE source_memory_ids LIKE ?`,
|
|
4805
|
+
[`%"${memoryId}"%`]
|
|
4806
|
+
);
|
|
4807
|
+
return (rows[0]?.count || 0) > 0;
|
|
4808
|
+
}
|
|
3787
4809
|
/**
|
|
3788
4810
|
* Get count of consolidated memories
|
|
3789
4811
|
*/
|
|
@@ -3933,7 +4955,14 @@ var ConsolidationWorker = class {
|
|
|
3933
4955
|
* Force a consolidation run (manual trigger)
|
|
3934
4956
|
*/
|
|
3935
4957
|
async forceRun() {
|
|
3936
|
-
|
|
4958
|
+
const out = await this.consolidateWithReport();
|
|
4959
|
+
return out.consolidatedCount;
|
|
4960
|
+
}
|
|
4961
|
+
/**
|
|
4962
|
+
* Force a consolidation run and return metrics report
|
|
4963
|
+
*/
|
|
4964
|
+
async forceRunWithReport() {
|
|
4965
|
+
return this.consolidateWithReport();
|
|
3937
4966
|
}
|
|
3938
4967
|
/**
|
|
3939
4968
|
* Schedule the next consolidation check
|
|
@@ -3973,12 +5002,21 @@ var ConsolidationWorker = class {
|
|
|
3973
5002
|
* Perform consolidation
|
|
3974
5003
|
*/
|
|
3975
5004
|
async consolidate() {
|
|
5005
|
+
const out = await this.consolidateWithReport();
|
|
5006
|
+
return out.consolidatedCount;
|
|
5007
|
+
}
|
|
5008
|
+
async consolidateWithReport() {
|
|
3976
5009
|
const workingSet = await this.workingSetStore.get();
|
|
3977
5010
|
if (workingSet.recentEvents.length < 3) {
|
|
3978
|
-
return
|
|
5011
|
+
return {
|
|
5012
|
+
consolidatedCount: 0,
|
|
5013
|
+
promotedRuleCount: 0,
|
|
5014
|
+
report: this.buildCostQualityReport(workingSet.recentEvents, [], 0)
|
|
5015
|
+
};
|
|
3979
5016
|
}
|
|
3980
5017
|
const groups = this.groupByTopic(workingSet.recentEvents);
|
|
3981
5018
|
let consolidatedCount = 0;
|
|
5019
|
+
const createdMemoryIds = [];
|
|
3982
5020
|
for (const group of groups) {
|
|
3983
5021
|
if (group.events.length < 3)
|
|
3984
5022
|
continue;
|
|
@@ -3987,14 +5025,16 @@ var ConsolidationWorker = class {
|
|
|
3987
5025
|
if (alreadyConsolidated)
|
|
3988
5026
|
continue;
|
|
3989
5027
|
const summary = await this.summarize(group);
|
|
3990
|
-
await this.consolidatedStore.create({
|
|
5028
|
+
const memoryId = await this.consolidatedStore.create({
|
|
3991
5029
|
summary,
|
|
3992
5030
|
topics: group.topics,
|
|
3993
5031
|
sourceEvents: eventIds,
|
|
3994
5032
|
confidence: this.calculateConfidence(group)
|
|
3995
5033
|
});
|
|
5034
|
+
createdMemoryIds.push(memoryId);
|
|
3996
5035
|
consolidatedCount++;
|
|
3997
5036
|
}
|
|
5037
|
+
const promotedRuleCount = await this.promoteStableSummariesToRules(createdMemoryIds);
|
|
3998
5038
|
if (consolidatedCount > 0) {
|
|
3999
5039
|
const consolidatedEventIds = groups.filter((g) => g.events.length >= 3).flatMap((g) => g.events.map((e) => e.id));
|
|
4000
5040
|
const oldEventIds = consolidatedEventIds.filter((id) => {
|
|
@@ -4008,7 +5048,61 @@ var ConsolidationWorker = class {
|
|
|
4008
5048
|
await this.workingSetStore.prune(oldEventIds);
|
|
4009
5049
|
}
|
|
4010
5050
|
}
|
|
4011
|
-
|
|
5051
|
+
const report = this.buildCostQualityReport(workingSet.recentEvents, groups, consolidatedCount);
|
|
5052
|
+
return { consolidatedCount, promotedRuleCount, report };
|
|
5053
|
+
}
|
|
5054
|
+
async promoteStableSummariesToRules(memoryIds) {
|
|
5055
|
+
let promoted = 0;
|
|
5056
|
+
for (const memoryId of memoryIds) {
|
|
5057
|
+
const memory = await this.consolidatedStore.get(memoryId);
|
|
5058
|
+
if (!memory)
|
|
5059
|
+
continue;
|
|
5060
|
+
if (memory.confidence < 0.55)
|
|
5061
|
+
continue;
|
|
5062
|
+
if (memory.sourceEvents.length < 4)
|
|
5063
|
+
continue;
|
|
5064
|
+
const exists = await this.consolidatedStore.hasRuleForSourceMemory(memoryId);
|
|
5065
|
+
if (exists)
|
|
5066
|
+
continue;
|
|
5067
|
+
const rule = this.buildRuleFromSummary(memory.summary, memory.topics);
|
|
5068
|
+
if (!rule)
|
|
5069
|
+
continue;
|
|
5070
|
+
await this.consolidatedStore.createRule({
|
|
5071
|
+
rule,
|
|
5072
|
+
topics: memory.topics,
|
|
5073
|
+
sourceMemoryIds: [memory.memoryId],
|
|
5074
|
+
sourceEvents: memory.sourceEvents,
|
|
5075
|
+
confidence: Math.min(1, memory.confidence + 0.08)
|
|
5076
|
+
});
|
|
5077
|
+
promoted++;
|
|
5078
|
+
}
|
|
5079
|
+
return promoted;
|
|
5080
|
+
}
|
|
5081
|
+
buildRuleFromSummary(summary, topics) {
|
|
5082
|
+
const lines = summary.split(/\r?\n/).map((l) => l.trim()).filter(Boolean).filter((l) => !l.toLowerCase().startsWith("topics:"));
|
|
5083
|
+
const bullet = lines.find((l) => l.startsWith("- "))?.replace(/^-\s*/, "");
|
|
5084
|
+
const seed = bullet || lines[0];
|
|
5085
|
+
if (!seed || seed.length < 8)
|
|
5086
|
+
return null;
|
|
5087
|
+
const topicPrefix = topics.length > 0 ? `[${topics.slice(0, 2).join(", ")}] ` : "";
|
|
5088
|
+
return `${topicPrefix}${seed}`;
|
|
5089
|
+
}
|
|
5090
|
+
buildCostQualityReport(events, groups, consolidatedCount) {
|
|
5091
|
+
const beforeTokenEstimate = events.reduce((acc, e) => acc + this.estimateTokens(e.content), 0);
|
|
5092
|
+
const afterSummaries = groups.filter((g) => g.events.length >= 3).slice(0, Math.max(consolidatedCount, 1));
|
|
5093
|
+
const afterTokenEstimate = afterSummaries.length > 0 ? afterSummaries.reduce((acc, g) => acc + this.estimateTokens(this.ruleBasedSummary(g)), 0) : beforeTokenEstimate;
|
|
5094
|
+
const reductionRatio = beforeTokenEstimate > 0 ? Math.max(0, (beforeTokenEstimate - afterTokenEstimate) / beforeTokenEstimate) : 0;
|
|
5095
|
+
const qualityGuardPassed = consolidatedCount === 0 ? true : groups.filter((g) => g.events.length >= 3).every((g) => this.calculateConfidence(g) >= 0.55);
|
|
5096
|
+
return {
|
|
5097
|
+
beforeTokenEstimate,
|
|
5098
|
+
afterTokenEstimate,
|
|
5099
|
+
reductionRatio,
|
|
5100
|
+
qualityGuardPassed,
|
|
5101
|
+
details: `groups=${groups.length}, consolidated=${consolidatedCount}`
|
|
5102
|
+
};
|
|
5103
|
+
}
|
|
5104
|
+
estimateTokens(text) {
|
|
5105
|
+
return Math.ceil((text || "").length / 4);
|
|
4012
5106
|
}
|
|
4013
5107
|
/**
|
|
4014
5108
|
* Check if consolidation should run
|
|
@@ -4568,13 +5662,185 @@ function createGraduationWorker(eventStore, graduation, config) {
|
|
|
4568
5662
|
);
|
|
4569
5663
|
}
|
|
4570
5664
|
|
|
5665
|
+
// src/core/md-mirror.ts
|
|
5666
|
+
import * as fs3 from "node:fs";
|
|
5667
|
+
import * as path2 from "node:path";
|
|
5668
|
+
function sanitizeSegment2(input, fallback) {
|
|
5669
|
+
const v = (input || "").trim().toLowerCase().replace(/[^a-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
5670
|
+
return v || fallback;
|
|
5671
|
+
}
|
|
5672
|
+
function getAtPath(obj, dotted) {
|
|
5673
|
+
if (!obj)
|
|
5674
|
+
return void 0;
|
|
5675
|
+
return dotted.split(".").reduce((acc, key) => {
|
|
5676
|
+
if (!acc || typeof acc !== "object")
|
|
5677
|
+
return void 0;
|
|
5678
|
+
return acc[key];
|
|
5679
|
+
}, obj);
|
|
5680
|
+
}
|
|
5681
|
+
function buildMirrorPath2(rootDir, event) {
|
|
5682
|
+
const meta = event.metadata;
|
|
5683
|
+
const namespaceRaw = getAtPath(meta, "namespace") ?? getAtPath(meta, "scope.namespace") ?? event.eventType;
|
|
5684
|
+
const namespace = sanitizeSegment2(typeof namespaceRaw === "string" ? namespaceRaw : void 0, "general");
|
|
5685
|
+
const categoryRaw = getAtPath(meta, "categoryPath") ?? getAtPath(meta, "scope.categoryPath");
|
|
5686
|
+
const categoryPath = Array.isArray(categoryRaw) && categoryRaw.length > 0 ? categoryRaw.map((x) => sanitizeSegment2(typeof x === "string" ? x : void 0, "uncategorized")) : ["uncategorized"];
|
|
5687
|
+
const d = event.timestamp;
|
|
5688
|
+
const yyyy = d.getFullYear();
|
|
5689
|
+
const mm = String(d.getMonth() + 1).padStart(2, "0");
|
|
5690
|
+
const dd = String(d.getDate()).padStart(2, "0");
|
|
5691
|
+
return path2.join(rootDir, "memory", namespace, ...categoryPath, `${yyyy}-${mm}-${dd}.md`);
|
|
5692
|
+
}
|
|
5693
|
+
var MarkdownMirror2 = class {
|
|
5694
|
+
constructor(rootDir) {
|
|
5695
|
+
this.rootDir = rootDir;
|
|
5696
|
+
}
|
|
5697
|
+
async append(event, eventId) {
|
|
5698
|
+
const out = buildMirrorPath2(this.rootDir, event);
|
|
5699
|
+
fs3.mkdirSync(path2.dirname(out), { recursive: true });
|
|
5700
|
+
const lines = [
|
|
5701
|
+
"",
|
|
5702
|
+
`## ${event.timestamp.toISOString()} | ${eventId ?? "pending-id"}`,
|
|
5703
|
+
`- type: ${event.eventType}`,
|
|
5704
|
+
`- session: ${event.sessionId}`,
|
|
5705
|
+
event.content
|
|
5706
|
+
];
|
|
5707
|
+
await fs3.promises.appendFile(out, lines.join("\n"), "utf8");
|
|
5708
|
+
await this.refreshIndex();
|
|
5709
|
+
}
|
|
5710
|
+
async refreshIndex() {
|
|
5711
|
+
const memoryRoot = path2.join(this.rootDir, "memory");
|
|
5712
|
+
await fs3.promises.mkdir(memoryRoot, { recursive: true });
|
|
5713
|
+
const files = [];
|
|
5714
|
+
await this.walk(memoryRoot, files);
|
|
5715
|
+
const mdFiles = files.filter((f) => f.endsWith(".md")).map((f) => path2.relative(this.rootDir, f)).filter((rel) => rel !== path2.join("memory", "_index.md")).sort();
|
|
5716
|
+
const index = [
|
|
5717
|
+
"# Memory Index",
|
|
5718
|
+
"",
|
|
5719
|
+
"Generated automatically by MarkdownMirror.",
|
|
5720
|
+
"",
|
|
5721
|
+
...mdFiles.map((rel) => `- ${rel}`),
|
|
5722
|
+
""
|
|
5723
|
+
].join("\n");
|
|
5724
|
+
await fs3.promises.writeFile(path2.join(memoryRoot, "_index.md"), index, "utf8");
|
|
5725
|
+
}
|
|
5726
|
+
async walk(dir, out) {
|
|
5727
|
+
const entries = await fs3.promises.readdir(dir, { withFileTypes: true });
|
|
5728
|
+
for (const e of entries) {
|
|
5729
|
+
const full = path2.join(dir, e.name);
|
|
5730
|
+
if (e.isDirectory()) {
|
|
5731
|
+
await this.walk(full, out);
|
|
5732
|
+
} else {
|
|
5733
|
+
out.push(full);
|
|
5734
|
+
}
|
|
5735
|
+
}
|
|
5736
|
+
}
|
|
5737
|
+
};
|
|
5738
|
+
|
|
5739
|
+
// src/core/ingest-interceptor.ts
|
|
5740
|
+
var IngestInterceptorRegistry = class {
|
|
5741
|
+
before = [];
|
|
5742
|
+
after = [];
|
|
5743
|
+
onError = [];
|
|
5744
|
+
registerBefore(interceptor) {
|
|
5745
|
+
this.before.push(interceptor);
|
|
5746
|
+
return () => {
|
|
5747
|
+
this.before = this.before.filter((i) => i !== interceptor);
|
|
5748
|
+
};
|
|
5749
|
+
}
|
|
5750
|
+
registerAfter(interceptor) {
|
|
5751
|
+
this.after.push(interceptor);
|
|
5752
|
+
return () => {
|
|
5753
|
+
this.after = this.after.filter((i) => i !== interceptor);
|
|
5754
|
+
};
|
|
5755
|
+
}
|
|
5756
|
+
registerOnError(interceptor) {
|
|
5757
|
+
this.onError.push(interceptor);
|
|
5758
|
+
return () => {
|
|
5759
|
+
this.onError = this.onError.filter((i) => i !== interceptor);
|
|
5760
|
+
};
|
|
5761
|
+
}
|
|
5762
|
+
async run(stage, context) {
|
|
5763
|
+
const interceptors = stage === "before" ? this.before : stage === "after" ? this.after : this.onError;
|
|
5764
|
+
for (const interceptor of interceptors) {
|
|
5765
|
+
await interceptor({ ...context, stage });
|
|
5766
|
+
}
|
|
5767
|
+
}
|
|
5768
|
+
};
|
|
5769
|
+
function mergeHierarchicalMetadata(base, patch) {
|
|
5770
|
+
if (!base && !patch)
|
|
5771
|
+
return void 0;
|
|
5772
|
+
if (!base)
|
|
5773
|
+
return patch;
|
|
5774
|
+
if (!patch)
|
|
5775
|
+
return base;
|
|
5776
|
+
const result = { ...base };
|
|
5777
|
+
for (const [key, value] of Object.entries(patch)) {
|
|
5778
|
+
const current = result[key];
|
|
5779
|
+
if (typeof current === "object" && current !== null && !Array.isArray(current) && typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
5780
|
+
result[key] = mergeHierarchicalMetadata(
|
|
5781
|
+
current,
|
|
5782
|
+
value
|
|
5783
|
+
);
|
|
5784
|
+
} else {
|
|
5785
|
+
result[key] = value;
|
|
5786
|
+
}
|
|
5787
|
+
}
|
|
5788
|
+
return result;
|
|
5789
|
+
}
|
|
5790
|
+
|
|
5791
|
+
// src/core/tag-taxonomy.ts
|
|
5792
|
+
var TAG_NAMESPACES = {
|
|
5793
|
+
SYSTEM: "sys:",
|
|
5794
|
+
QUALITY: "q:",
|
|
5795
|
+
PROJECT: "proj:",
|
|
5796
|
+
TOPIC: "topic:",
|
|
5797
|
+
TEMPORAL: "t:",
|
|
5798
|
+
USER: "user:",
|
|
5799
|
+
AGENT: "agent:"
|
|
5800
|
+
};
|
|
5801
|
+
var VALID_TAG_NAMESPACES = new Set(Object.values(TAG_NAMESPACES));
|
|
5802
|
+
function parseTag(tag) {
|
|
5803
|
+
const value = (tag || "").trim();
|
|
5804
|
+
const idx = value.indexOf(":");
|
|
5805
|
+
if (idx <= 0)
|
|
5806
|
+
return { value };
|
|
5807
|
+
const namespace = `${value.slice(0, idx)}:`;
|
|
5808
|
+
const tagValue = value.slice(idx + 1);
|
|
5809
|
+
if (!tagValue)
|
|
5810
|
+
return { value };
|
|
5811
|
+
return { namespace, value: tagValue };
|
|
5812
|
+
}
|
|
5813
|
+
function validateTag(tag) {
|
|
5814
|
+
const normalized = (tag || "").trim();
|
|
5815
|
+
if (!normalized)
|
|
5816
|
+
return false;
|
|
5817
|
+
const { namespace } = parseTag(normalized);
|
|
5818
|
+
if (!namespace)
|
|
5819
|
+
return true;
|
|
5820
|
+
return VALID_TAG_NAMESPACES.has(namespace);
|
|
5821
|
+
}
|
|
5822
|
+
function normalizeTags(tags) {
|
|
5823
|
+
if (!Array.isArray(tags))
|
|
5824
|
+
return [];
|
|
5825
|
+
const dedup = /* @__PURE__ */ new Set();
|
|
5826
|
+
for (const item of tags) {
|
|
5827
|
+
if (typeof item !== "string")
|
|
5828
|
+
continue;
|
|
5829
|
+
const normalized = item.trim();
|
|
5830
|
+
if (!validateTag(normalized))
|
|
5831
|
+
continue;
|
|
5832
|
+
dedup.add(normalized);
|
|
5833
|
+
}
|
|
5834
|
+
return [...dedup];
|
|
5835
|
+
}
|
|
5836
|
+
|
|
4571
5837
|
// src/services/memory-service.ts
|
|
4572
5838
|
function normalizePath(projectPath) {
|
|
4573
|
-
const expanded = projectPath.startsWith("~") ?
|
|
5839
|
+
const expanded = projectPath.startsWith("~") ? path3.join(os.homedir(), projectPath.slice(1)) : projectPath;
|
|
4574
5840
|
try {
|
|
4575
|
-
return
|
|
5841
|
+
return fs4.realpathSync(expanded);
|
|
4576
5842
|
} catch {
|
|
4577
|
-
return
|
|
5843
|
+
return path3.resolve(expanded);
|
|
4578
5844
|
}
|
|
4579
5845
|
}
|
|
4580
5846
|
function hashProjectPath(projectPath) {
|
|
@@ -4583,10 +5849,21 @@ function hashProjectPath(projectPath) {
|
|
|
4583
5849
|
}
|
|
4584
5850
|
function getProjectStoragePath(projectPath) {
|
|
4585
5851
|
const hash = hashProjectPath(projectPath);
|
|
4586
|
-
return
|
|
5852
|
+
return path3.join(os.homedir(), ".claude-code", "memory", "projects", hash);
|
|
5853
|
+
}
|
|
5854
|
+
var REGISTRY_PATH = path3.join(os.homedir(), ".claude-code", "memory", "session-registry.json");
|
|
5855
|
+
var SHARED_STORAGE_PATH = path3.join(os.homedir(), ".claude-code", "memory", "shared");
|
|
5856
|
+
function loadSessionRegistry() {
|
|
5857
|
+
try {
|
|
5858
|
+
if (fs4.existsSync(REGISTRY_PATH)) {
|
|
5859
|
+
const data = fs4.readFileSync(REGISTRY_PATH, "utf-8");
|
|
5860
|
+
return JSON.parse(data);
|
|
5861
|
+
}
|
|
5862
|
+
} catch (error) {
|
|
5863
|
+
console.error("Failed to load session registry:", error);
|
|
5864
|
+
}
|
|
5865
|
+
return { version: 1, sessions: {} };
|
|
4587
5866
|
}
|
|
4588
|
-
var REGISTRY_PATH = path.join(os.homedir(), ".claude-code", "memory", "session-registry.json");
|
|
4589
|
-
var SHARED_STORAGE_PATH = path.join(os.homedir(), ".claude-code", "memory", "shared");
|
|
4590
5867
|
var MemoryService = class {
|
|
4591
5868
|
// Primary store: SQLite (WAL mode) - for hooks, always available
|
|
4592
5869
|
sqliteStore;
|
|
@@ -4601,6 +5878,7 @@ var MemoryService = class {
|
|
|
4601
5878
|
vectorWorker = null;
|
|
4602
5879
|
graduationWorker = null;
|
|
4603
5880
|
initialized = false;
|
|
5881
|
+
ingestInterceptors = new IngestInterceptorRegistry();
|
|
4604
5882
|
// Endless Mode components
|
|
4605
5883
|
workingSetStore = null;
|
|
4606
5884
|
consolidatedStore = null;
|
|
@@ -4614,20 +5892,27 @@ var MemoryService = class {
|
|
|
4614
5892
|
sharedPromoter = null;
|
|
4615
5893
|
sharedStoreConfig = null;
|
|
4616
5894
|
projectHash = null;
|
|
5895
|
+
projectPath = null;
|
|
4617
5896
|
readOnly;
|
|
4618
5897
|
lightweightMode;
|
|
5898
|
+
mdMirror;
|
|
4619
5899
|
constructor(config) {
|
|
4620
5900
|
const storagePath = this.expandPath(config.storagePath);
|
|
4621
5901
|
this.readOnly = config.readOnly ?? false;
|
|
4622
5902
|
this.lightweightMode = config.lightweightMode ?? false;
|
|
4623
|
-
|
|
4624
|
-
|
|
5903
|
+
this.mdMirror = new MarkdownMirror2(process.cwd());
|
|
5904
|
+
if (!this.readOnly && !fs4.existsSync(storagePath)) {
|
|
5905
|
+
fs4.mkdirSync(storagePath, { recursive: true });
|
|
4625
5906
|
}
|
|
4626
5907
|
this.projectHash = config.projectHash || null;
|
|
5908
|
+
this.projectPath = config.projectPath || null;
|
|
4627
5909
|
this.sharedStoreConfig = config.sharedStoreConfig ?? { enabled: true };
|
|
4628
5910
|
this.sqliteStore = new SQLiteEventStore(
|
|
4629
|
-
|
|
4630
|
-
{
|
|
5911
|
+
path3.join(storagePath, "events.sqlite"),
|
|
5912
|
+
{
|
|
5913
|
+
readonly: this.readOnly,
|
|
5914
|
+
markdownMirrorRoot: storagePath
|
|
5915
|
+
}
|
|
4631
5916
|
);
|
|
4632
5917
|
const analyticsEnabled = config.analyticsEnabled ?? this.readOnly;
|
|
4633
5918
|
if (!analyticsEnabled) {
|
|
@@ -4635,7 +5920,7 @@ var MemoryService = class {
|
|
|
4635
5920
|
} else if (this.readOnly) {
|
|
4636
5921
|
try {
|
|
4637
5922
|
this.analyticsStore = new EventStore(
|
|
4638
|
-
|
|
5923
|
+
path3.join(storagePath, "analytics.duckdb"),
|
|
4639
5924
|
{ readOnly: true }
|
|
4640
5925
|
);
|
|
4641
5926
|
} catch {
|
|
@@ -4643,11 +5928,11 @@ var MemoryService = class {
|
|
|
4643
5928
|
}
|
|
4644
5929
|
} else {
|
|
4645
5930
|
this.analyticsStore = new EventStore(
|
|
4646
|
-
|
|
5931
|
+
path3.join(storagePath, "analytics.duckdb"),
|
|
4647
5932
|
{ readOnly: false }
|
|
4648
5933
|
);
|
|
4649
5934
|
}
|
|
4650
|
-
this.vectorStore = new VectorStore(
|
|
5935
|
+
this.vectorStore = new VectorStore(path3.join(storagePath, "vectors"));
|
|
4651
5936
|
this.embedder = config.embeddingModel ? new Embedder(config.embeddingModel) : getDefaultEmbedder();
|
|
4652
5937
|
this.matcher = getDefaultMatcher();
|
|
4653
5938
|
this.retriever = createRetriever(
|
|
@@ -4657,6 +5942,7 @@ var MemoryService = class {
|
|
|
4657
5942
|
this.embedder,
|
|
4658
5943
|
this.matcher
|
|
4659
5944
|
);
|
|
5945
|
+
this.retriever.setQueryRewriter((q) => this.rewriteQueryIntent(q));
|
|
4660
5946
|
this.graduation = createGraduationPipeline(this.sqliteStore);
|
|
4661
5947
|
}
|
|
4662
5948
|
/**
|
|
@@ -4716,16 +6002,16 @@ var MemoryService = class {
|
|
|
4716
6002
|
*/
|
|
4717
6003
|
async initializeSharedStore() {
|
|
4718
6004
|
const sharedPath = this.sharedStoreConfig?.sharedStoragePath ? this.expandPath(this.sharedStoreConfig.sharedStoragePath) : SHARED_STORAGE_PATH;
|
|
4719
|
-
if (!
|
|
4720
|
-
|
|
6005
|
+
if (!fs4.existsSync(sharedPath)) {
|
|
6006
|
+
fs4.mkdirSync(sharedPath, { recursive: true });
|
|
4721
6007
|
}
|
|
4722
6008
|
this.sharedEventStore = createSharedEventStore(
|
|
4723
|
-
|
|
6009
|
+
path3.join(sharedPath, "shared.duckdb")
|
|
4724
6010
|
);
|
|
4725
6011
|
await this.sharedEventStore.initialize();
|
|
4726
6012
|
this.sharedStore = createSharedStore(this.sharedEventStore);
|
|
4727
6013
|
this.sharedVectorStore = createSharedVectorStore(
|
|
4728
|
-
|
|
6014
|
+
path3.join(sharedPath, "vectors")
|
|
4729
6015
|
);
|
|
4730
6016
|
await this.sharedVectorStore.initialize();
|
|
4731
6017
|
this.sharedPromoter = createSharedPromoter(
|
|
@@ -4736,6 +6022,86 @@ var MemoryService = class {
|
|
|
4736
6022
|
);
|
|
4737
6023
|
this.retriever.setSharedStores(this.sharedStore, this.sharedVectorStore);
|
|
4738
6024
|
}
|
|
6025
|
+
registerIngestBefore(interceptor) {
|
|
6026
|
+
return this.ingestInterceptors.registerBefore(interceptor);
|
|
6027
|
+
}
|
|
6028
|
+
registerIngestAfter(interceptor) {
|
|
6029
|
+
return this.ingestInterceptors.registerAfter(interceptor);
|
|
6030
|
+
}
|
|
6031
|
+
registerIngestOnError(interceptor) {
|
|
6032
|
+
return this.ingestInterceptors.registerOnError(interceptor);
|
|
6033
|
+
}
|
|
6034
|
+
async ingestWithInterceptors(operation, input, onSuccess) {
|
|
6035
|
+
const normalizedInput = {
|
|
6036
|
+
...input,
|
|
6037
|
+
metadata: mergeHierarchicalMetadata(
|
|
6038
|
+
{
|
|
6039
|
+
ingest: {
|
|
6040
|
+
operation,
|
|
6041
|
+
pipeline: "default",
|
|
6042
|
+
ts: (/* @__PURE__ */ new Date()).toISOString()
|
|
6043
|
+
},
|
|
6044
|
+
...this.projectHash ? {
|
|
6045
|
+
scope: {
|
|
6046
|
+
project: {
|
|
6047
|
+
hash: this.projectHash,
|
|
6048
|
+
...this.projectPath ? { path: this.projectPath } : {}
|
|
6049
|
+
}
|
|
6050
|
+
},
|
|
6051
|
+
tags: [`proj:${this.projectHash}`]
|
|
6052
|
+
} : {}
|
|
6053
|
+
},
|
|
6054
|
+
input.metadata
|
|
6055
|
+
)
|
|
6056
|
+
};
|
|
6057
|
+
if (this.projectHash && normalizedInput.metadata) {
|
|
6058
|
+
const meta = normalizedInput.metadata;
|
|
6059
|
+
const currentTags = Array.isArray(meta.tags) ? meta.tags.filter((x) => typeof x === "string") : [];
|
|
6060
|
+
const projectTag = `proj:${this.projectHash}`;
|
|
6061
|
+
if (!currentTags.includes(projectTag)) {
|
|
6062
|
+
meta.tags = [...currentTags, projectTag];
|
|
6063
|
+
}
|
|
6064
|
+
}
|
|
6065
|
+
if (normalizedInput.metadata) {
|
|
6066
|
+
const meta = normalizedInput.metadata;
|
|
6067
|
+
const normalizedTags = normalizeTags(meta.tags);
|
|
6068
|
+
if (normalizedTags.length > 0) {
|
|
6069
|
+
meta.tags = normalizedTags;
|
|
6070
|
+
}
|
|
6071
|
+
}
|
|
6072
|
+
await this.ingestInterceptors.run("before", {
|
|
6073
|
+
operation,
|
|
6074
|
+
sessionId: normalizedInput.sessionId,
|
|
6075
|
+
event: normalizedInput
|
|
6076
|
+
});
|
|
6077
|
+
try {
|
|
6078
|
+
const result = await this.sqliteStore.append(normalizedInput);
|
|
6079
|
+
if (result.success && !result.isDuplicate) {
|
|
6080
|
+
if (onSuccess) {
|
|
6081
|
+
await onSuccess(result.eventId);
|
|
6082
|
+
}
|
|
6083
|
+
try {
|
|
6084
|
+
await this.mdMirror.append(normalizedInput, result.eventId);
|
|
6085
|
+
} catch {
|
|
6086
|
+
}
|
|
6087
|
+
}
|
|
6088
|
+
await this.ingestInterceptors.run("after", {
|
|
6089
|
+
operation,
|
|
6090
|
+
sessionId: normalizedInput.sessionId,
|
|
6091
|
+
event: normalizedInput
|
|
6092
|
+
});
|
|
6093
|
+
return result;
|
|
6094
|
+
} catch (error) {
|
|
6095
|
+
const normalizedError = error instanceof Error ? error : new Error(String(error));
|
|
6096
|
+
await this.ingestInterceptors.run("error", {
|
|
6097
|
+
operation,
|
|
6098
|
+
sessionId: normalizedInput.sessionId,
|
|
6099
|
+
event: normalizedInput,
|
|
6100
|
+
error: normalizedError
|
|
6101
|
+
});
|
|
6102
|
+
throw error;
|
|
6103
|
+
}
|
|
6104
|
+
}
|
|
4739
6105
|
/**
|
|
4740
6106
|
* Start a new session
|
|
4741
6107
|
*/
|
|
@@ -4763,50 +6129,57 @@ var MemoryService = class {
|
|
|
4763
6129
|
*/
|
|
4764
6130
|
async storeUserPrompt(sessionId, content, metadata) {
|
|
4765
6131
|
await this.initialize();
|
|
4766
|
-
|
|
4767
|
-
|
|
4768
|
-
|
|
4769
|
-
|
|
4770
|
-
|
|
4771
|
-
|
|
4772
|
-
|
|
4773
|
-
|
|
4774
|
-
|
|
4775
|
-
|
|
4776
|
-
|
|
6132
|
+
return this.ingestWithInterceptors(
|
|
6133
|
+
"user_prompt",
|
|
6134
|
+
{
|
|
6135
|
+
eventType: "user_prompt",
|
|
6136
|
+
sessionId,
|
|
6137
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6138
|
+
content,
|
|
6139
|
+
metadata
|
|
6140
|
+
},
|
|
6141
|
+
async (eventId) => {
|
|
6142
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, content);
|
|
6143
|
+
}
|
|
6144
|
+
);
|
|
4777
6145
|
}
|
|
4778
6146
|
/**
|
|
4779
6147
|
* Store an agent response
|
|
4780
6148
|
*/
|
|
4781
6149
|
async storeAgentResponse(sessionId, content, metadata) {
|
|
4782
6150
|
await this.initialize();
|
|
4783
|
-
|
|
4784
|
-
|
|
4785
|
-
|
|
4786
|
-
|
|
4787
|
-
|
|
4788
|
-
|
|
4789
|
-
|
|
4790
|
-
|
|
4791
|
-
|
|
4792
|
-
|
|
4793
|
-
|
|
6151
|
+
return this.ingestWithInterceptors(
|
|
6152
|
+
"agent_response",
|
|
6153
|
+
{
|
|
6154
|
+
eventType: "agent_response",
|
|
6155
|
+
sessionId,
|
|
6156
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6157
|
+
content,
|
|
6158
|
+
metadata
|
|
6159
|
+
},
|
|
6160
|
+
async (eventId) => {
|
|
6161
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, content);
|
|
6162
|
+
}
|
|
6163
|
+
);
|
|
4794
6164
|
}
|
|
4795
6165
|
/**
|
|
4796
6166
|
* Store a session summary
|
|
4797
6167
|
*/
|
|
4798
|
-
async storeSessionSummary(sessionId, summary) {
|
|
6168
|
+
async storeSessionSummary(sessionId, summary, metadata) {
|
|
4799
6169
|
await this.initialize();
|
|
4800
|
-
|
|
4801
|
-
|
|
4802
|
-
|
|
4803
|
-
|
|
4804
|
-
|
|
4805
|
-
|
|
4806
|
-
|
|
4807
|
-
|
|
4808
|
-
|
|
4809
|
-
|
|
6170
|
+
return this.ingestWithInterceptors(
|
|
6171
|
+
"session_summary",
|
|
6172
|
+
{
|
|
6173
|
+
eventType: "session_summary",
|
|
6174
|
+
sessionId,
|
|
6175
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6176
|
+
content: summary,
|
|
6177
|
+
metadata
|
|
6178
|
+
},
|
|
6179
|
+
async (eventId) => {
|
|
6180
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, summary);
|
|
6181
|
+
}
|
|
6182
|
+
);
|
|
4810
6183
|
}
|
|
4811
6184
|
/**
|
|
4812
6185
|
* Store a tool observation
|
|
@@ -4814,39 +6187,182 @@ var MemoryService = class {
|
|
|
4814
6187
|
async storeToolObservation(sessionId, payload) {
|
|
4815
6188
|
await this.initialize();
|
|
4816
6189
|
const content = JSON.stringify(payload);
|
|
4817
|
-
const
|
|
4818
|
-
|
|
4819
|
-
|
|
4820
|
-
|
|
4821
|
-
|
|
4822
|
-
|
|
4823
|
-
|
|
4824
|
-
|
|
6190
|
+
const turnId = payload.metadata?.turnId;
|
|
6191
|
+
return this.ingestWithInterceptors(
|
|
6192
|
+
"tool_observation",
|
|
6193
|
+
{
|
|
6194
|
+
eventType: "tool_observation",
|
|
6195
|
+
sessionId,
|
|
6196
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6197
|
+
content,
|
|
6198
|
+
metadata: {
|
|
6199
|
+
toolName: payload.toolName,
|
|
6200
|
+
success: payload.success,
|
|
6201
|
+
...turnId ? { turnId } : {}
|
|
6202
|
+
}
|
|
6203
|
+
},
|
|
6204
|
+
async (eventId) => {
|
|
6205
|
+
const embeddingContent = createToolObservationEmbedding(
|
|
6206
|
+
payload.toolName,
|
|
6207
|
+
payload.metadata || {},
|
|
6208
|
+
payload.success
|
|
6209
|
+
);
|
|
6210
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, embeddingContent);
|
|
4825
6211
|
}
|
|
4826
|
-
|
|
4827
|
-
if (result.success && !result.isDuplicate) {
|
|
4828
|
-
const embeddingContent = createToolObservationEmbedding(
|
|
4829
|
-
payload.toolName,
|
|
4830
|
-
payload.metadata || {},
|
|
4831
|
-
payload.success
|
|
4832
|
-
);
|
|
4833
|
-
await this.sqliteStore.enqueueForEmbedding(result.eventId, embeddingContent);
|
|
4834
|
-
}
|
|
4835
|
-
return result;
|
|
6212
|
+
);
|
|
4836
6213
|
}
|
|
4837
6214
|
/**
|
|
4838
6215
|
* Retrieve relevant memories for a query
|
|
4839
6216
|
*/
|
|
4840
6217
|
async retrieveMemories(query, options) {
|
|
4841
6218
|
await this.initialize();
|
|
6219
|
+
const rerankWeights = await this.getRerankWeights(options?.adaptiveRerank === true);
|
|
6220
|
+
let result;
|
|
4842
6221
|
if (options?.includeShared && this.sharedStore) {
|
|
4843
|
-
|
|
6222
|
+
result = await this.retriever.retrieveUnified(query, {
|
|
4844
6223
|
...options,
|
|
6224
|
+
intentRewrite: options?.intentRewrite === true,
|
|
6225
|
+
rerankWeights,
|
|
4845
6226
|
includeShared: true,
|
|
4846
|
-
projectHash: this.projectHash || void 0
|
|
6227
|
+
projectHash: this.projectHash || void 0,
|
|
6228
|
+
projectScopeMode: options?.projectScopeMode ?? (this.projectHash ? "strict" : "global"),
|
|
6229
|
+
allowedProjectHashes: options?.allowedProjectHashes
|
|
6230
|
+
});
|
|
6231
|
+
} else {
|
|
6232
|
+
result = await this.retriever.retrieve(query, {
|
|
6233
|
+
...options,
|
|
6234
|
+
intentRewrite: options?.intentRewrite === true,
|
|
6235
|
+
rerankWeights,
|
|
6236
|
+
projectHash: this.projectHash || void 0,
|
|
6237
|
+
projectScopeMode: options?.projectScopeMode ?? (this.projectHash ? "strict" : "global"),
|
|
6238
|
+
allowedProjectHashes: options?.allowedProjectHashes
|
|
6239
|
+
});
|
|
6240
|
+
}
|
|
6241
|
+
try {
|
|
6242
|
+
const selectedEventIds = result.memories.map((m) => m.event.id);
|
|
6243
|
+
const selectedDetails = (result.selectedDebug || []).map((d) => ({
|
|
6244
|
+
eventId: d.eventId,
|
|
6245
|
+
score: d.score,
|
|
6246
|
+
semanticScore: d.semanticScore,
|
|
6247
|
+
lexicalScore: d.lexicalScore,
|
|
6248
|
+
recencyScore: d.recencyScore
|
|
6249
|
+
}));
|
|
6250
|
+
const candidateDetails = (result.candidateDebug || []).map((d) => ({
|
|
6251
|
+
eventId: d.eventId,
|
|
6252
|
+
score: d.score,
|
|
6253
|
+
semanticScore: d.semanticScore,
|
|
6254
|
+
lexicalScore: d.lexicalScore,
|
|
6255
|
+
recencyScore: d.recencyScore
|
|
6256
|
+
}));
|
|
6257
|
+
const candidateEventIds = candidateDetails.length > 0 ? candidateDetails.map((d) => d.eventId) : selectedEventIds;
|
|
6258
|
+
await this.sqliteStore.recordRetrievalTrace({
|
|
6259
|
+
sessionId: options?.sessionId,
|
|
6260
|
+
projectHash: this.projectHash || void 0,
|
|
6261
|
+
queryText: query,
|
|
6262
|
+
strategy: options?.strategy || "auto",
|
|
6263
|
+
candidateEventIds,
|
|
6264
|
+
selectedEventIds,
|
|
6265
|
+
candidateDetails,
|
|
6266
|
+
selectedDetails,
|
|
6267
|
+
confidence: result.matchResult.confidence,
|
|
6268
|
+
fallbackTrace: result.fallbackTrace || []
|
|
4847
6269
|
});
|
|
6270
|
+
} catch {
|
|
6271
|
+
}
|
|
6272
|
+
return result;
|
|
6273
|
+
}
|
|
6274
|
+
getConfiguredRerankWeights() {
|
|
6275
|
+
const semantic = Number(process.env.MEMORY_RERANK_WEIGHT_SEMANTIC ?? "");
|
|
6276
|
+
const lexical = Number(process.env.MEMORY_RERANK_WEIGHT_LEXICAL ?? "");
|
|
6277
|
+
const recency = Number(process.env.MEMORY_RERANK_WEIGHT_RECENCY ?? "");
|
|
6278
|
+
const allFinite = [semantic, lexical, recency].every((v) => Number.isFinite(v));
|
|
6279
|
+
if (!allFinite)
|
|
6280
|
+
return void 0;
|
|
6281
|
+
const nonNegative = [semantic, lexical, recency].every((v) => v >= 0);
|
|
6282
|
+
const total = semantic + lexical + recency;
|
|
6283
|
+
if (!nonNegative || total <= 0)
|
|
6284
|
+
return void 0;
|
|
6285
|
+
return {
|
|
6286
|
+
semantic: semantic / total,
|
|
6287
|
+
lexical: lexical / total,
|
|
6288
|
+
recency: recency / total
|
|
6289
|
+
};
|
|
6290
|
+
}
|
|
6291
|
+
async getRerankWeights(adaptive) {
|
|
6292
|
+
const configured = this.getConfiguredRerankWeights();
|
|
6293
|
+
if (configured)
|
|
6294
|
+
return configured;
|
|
6295
|
+
if (adaptive)
|
|
6296
|
+
return this.getAdaptiveRerankWeights();
|
|
6297
|
+
return void 0;
|
|
6298
|
+
}
|
|
6299
|
+
async rewriteQueryIntent(query) {
|
|
6300
|
+
if (process.env.MEMORY_INTENT_REWRITE_ENABLED !== "1")
|
|
6301
|
+
return null;
|
|
6302
|
+
const apiUrl = process.env.COMPANY_STOCK_API_URL || process.env.COMPANY_INT_API_URL;
|
|
6303
|
+
if (!apiUrl)
|
|
6304
|
+
return null;
|
|
6305
|
+
const controller = new AbortController();
|
|
6306
|
+
const timeoutMs = Number(process.env.MEMORY_INTENT_REWRITE_TIMEOUT_MS || 5e3);
|
|
6307
|
+
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
|
6308
|
+
try {
|
|
6309
|
+
const prompt = [
|
|
6310
|
+
"Rewrite user query for memory retrieval intent expansion.",
|
|
6311
|
+
"Return plain text only, one line, no markdown.",
|
|
6312
|
+
`Query: ${query}`
|
|
6313
|
+
].join("\n");
|
|
6314
|
+
const res = await fetch(apiUrl, {
|
|
6315
|
+
method: "POST",
|
|
6316
|
+
headers: {
|
|
6317
|
+
"Content-Type": "application/json",
|
|
6318
|
+
Accept: "*/*",
|
|
6319
|
+
Origin: process.env.COMPANY_INT_ORIGIN || "http://company-int.aplusai.ai",
|
|
6320
|
+
Referer: process.env.COMPANY_INT_REFERER || "http://company-int.aplusai.ai/"
|
|
6321
|
+
},
|
|
6322
|
+
body: JSON.stringify({
|
|
6323
|
+
question: prompt,
|
|
6324
|
+
company_name: null,
|
|
6325
|
+
conversation_id: null
|
|
6326
|
+
}),
|
|
6327
|
+
signal: controller.signal
|
|
6328
|
+
});
|
|
6329
|
+
const text = (await res.text()).trim();
|
|
6330
|
+
if (!text)
|
|
6331
|
+
return null;
|
|
6332
|
+
const oneLine = text.replace(/^data:\s*/gm, "").split(/\r?\n/).map((x) => x.trim()).filter(Boolean).join(" ").slice(0, 240);
|
|
6333
|
+
if (!oneLine || oneLine.toLowerCase() === query.toLowerCase())
|
|
6334
|
+
return null;
|
|
6335
|
+
return oneLine;
|
|
6336
|
+
} catch {
|
|
6337
|
+
return null;
|
|
6338
|
+
} finally {
|
|
6339
|
+
clearTimeout(timeout);
|
|
6340
|
+
}
|
|
6341
|
+
}
|
|
6342
|
+
async getAdaptiveRerankWeights() {
|
|
6343
|
+
try {
|
|
6344
|
+
const s = await this.sqliteStore.getHelpfulnessStats();
|
|
6345
|
+
if (s.totalEvaluated < 20)
|
|
6346
|
+
return void 0;
|
|
6347
|
+
let semantic = 0.7;
|
|
6348
|
+
let lexical = 0.2;
|
|
6349
|
+
let recency = 0.1;
|
|
6350
|
+
if (s.avgScore < 0.45) {
|
|
6351
|
+
semantic -= 0.1;
|
|
6352
|
+
lexical += 0.1;
|
|
6353
|
+
} else if (s.avgScore > 0.75) {
|
|
6354
|
+
semantic += 0.05;
|
|
6355
|
+
lexical -= 0.05;
|
|
6356
|
+
}
|
|
6357
|
+
if (s.unhelpful > s.helpful) {
|
|
6358
|
+
recency += 0.05;
|
|
6359
|
+
semantic -= 0.03;
|
|
6360
|
+
lexical -= 0.02;
|
|
6361
|
+
}
|
|
6362
|
+
return { semantic, lexical, recency };
|
|
6363
|
+
} catch {
|
|
6364
|
+
return void 0;
|
|
4848
6365
|
}
|
|
4849
|
-
return this.retriever.retrieve(query, options);
|
|
4850
6366
|
}
|
|
4851
6367
|
/**
|
|
4852
6368
|
* Fast keyword search using SQLite FTS5
|
|
@@ -4888,6 +6404,18 @@ var MemoryService = class {
|
|
|
4888
6404
|
/**
|
|
4889
6405
|
* Get memory statistics
|
|
4890
6406
|
*/
|
|
6407
|
+
async getOutboxStats() {
|
|
6408
|
+
await this.initialize();
|
|
6409
|
+
return this.sqliteStore.getOutboxStats();
|
|
6410
|
+
}
|
|
6411
|
+
async getRetrievalTraceStats() {
|
|
6412
|
+
await this.initialize();
|
|
6413
|
+
return this.sqliteStore.getRetrievalTraceStats();
|
|
6414
|
+
}
|
|
6415
|
+
async getRecentRetrievalTraces(limit = 50) {
|
|
6416
|
+
await this.initialize();
|
|
6417
|
+
return this.sqliteStore.getRecentRetrievalTraces(limit);
|
|
6418
|
+
}
|
|
4891
6419
|
async getStats() {
|
|
4892
6420
|
await this.initialize();
|
|
4893
6421
|
const recentEvents = await this.sqliteStore.getRecentEvents(1e4);
|
|
@@ -5104,6 +6632,31 @@ var MemoryService = class {
|
|
|
5104
6632
|
return [];
|
|
5105
6633
|
return this.consolidatedStore.getAll({ limit });
|
|
5106
6634
|
}
|
|
6635
|
+
/**
|
|
6636
|
+
* Extract topic keywords from event content (markdown headings and key terms)
|
|
6637
|
+
*/
|
|
6638
|
+
extractTopicsFromContent(content) {
|
|
6639
|
+
const topics = /* @__PURE__ */ new Set();
|
|
6640
|
+
const headings = content.match(/^#{1,3}\s+(.+)$/gm);
|
|
6641
|
+
if (headings) {
|
|
6642
|
+
for (const h of headings.slice(0, 5)) {
|
|
6643
|
+
const text = h.replace(/^#+\s+/, "").replace(/[*_`#]/g, "").trim();
|
|
6644
|
+
if (text.length > 2 && text.length < 50) {
|
|
6645
|
+
topics.add(text);
|
|
6646
|
+
}
|
|
6647
|
+
}
|
|
6648
|
+
}
|
|
6649
|
+
const boldTerms = content.match(/\*\*([^*]+)\*\*/g);
|
|
6650
|
+
if (boldTerms) {
|
|
6651
|
+
for (const b of boldTerms.slice(0, 5)) {
|
|
6652
|
+
const text = b.replace(/\*\*/g, "").trim();
|
|
6653
|
+
if (text.length > 2 && text.length < 30) {
|
|
6654
|
+
topics.add(text);
|
|
6655
|
+
}
|
|
6656
|
+
}
|
|
6657
|
+
}
|
|
6658
|
+
return Array.from(topics).slice(0, 5);
|
|
6659
|
+
}
|
|
5107
6660
|
/**
|
|
5108
6661
|
* Increment access count for memories that were used in prompts
|
|
5109
6662
|
*/
|
|
@@ -5127,8 +6680,7 @@ var MemoryService = class {
|
|
|
5127
6680
|
return events.map((event) => ({
|
|
5128
6681
|
memoryId: event.id,
|
|
5129
6682
|
summary: event.content.substring(0, 200) + (event.content.length > 200 ? "..." : ""),
|
|
5130
|
-
topics:
|
|
5131
|
-
// Could extract topics from content if needed
|
|
6683
|
+
topics: this.extractTopicsFromContent(event.content),
|
|
5132
6684
|
accessCount: event.access_count || 0,
|
|
5133
6685
|
lastAccessed: event.last_accessed_at || null,
|
|
5134
6686
|
confidence: 1,
|
|
@@ -5149,6 +6701,34 @@ var MemoryService = class {
|
|
|
5149
6701
|
}
|
|
5150
6702
|
return [];
|
|
5151
6703
|
}
|
|
6704
|
+
/**
|
|
6705
|
+
* Record a memory retrieval for helpfulness tracking
|
|
6706
|
+
*/
|
|
6707
|
+
async recordRetrieval(eventId, sessionId, score, query) {
|
|
6708
|
+
await this.initialize();
|
|
6709
|
+
await this.sqliteStore.recordRetrieval(eventId, sessionId, score, query);
|
|
6710
|
+
}
|
|
6711
|
+
/**
|
|
6712
|
+
* Evaluate helpfulness of retrievals in a session (called at session end)
|
|
6713
|
+
*/
|
|
6714
|
+
async evaluateSessionHelpfulness(sessionId) {
|
|
6715
|
+
await this.initialize();
|
|
6716
|
+
await this.sqliteStore.evaluateSessionHelpfulness(sessionId);
|
|
6717
|
+
}
|
|
6718
|
+
/**
|
|
6719
|
+
* Get most helpful memories ranked by helpfulness score
|
|
6720
|
+
*/
|
|
6721
|
+
async getHelpfulMemories(limit = 10) {
|
|
6722
|
+
await this.initialize();
|
|
6723
|
+
return this.sqliteStore.getHelpfulMemories(limit);
|
|
6724
|
+
}
|
|
6725
|
+
/**
|
|
6726
|
+
* Get helpfulness statistics for dashboard
|
|
6727
|
+
*/
|
|
6728
|
+
async getHelpfulnessStats() {
|
|
6729
|
+
await this.initialize();
|
|
6730
|
+
return this.sqliteStore.getHelpfulnessStats();
|
|
6731
|
+
}
|
|
5152
6732
|
/**
|
|
5153
6733
|
* Mark a consolidated memory as accessed
|
|
5154
6734
|
*/
|
|
@@ -5212,6 +6792,44 @@ var MemoryService = class {
|
|
|
5212
6792
|
lastConsolidation
|
|
5213
6793
|
};
|
|
5214
6794
|
}
|
|
6795
|
+
// ============================================================
|
|
6796
|
+
// Turn Grouping Methods
|
|
6797
|
+
// ============================================================
|
|
6798
|
+
/**
|
|
6799
|
+
* Get events grouped by turn for a session
|
|
6800
|
+
*/
|
|
6801
|
+
async getSessionTurns(sessionId, options) {
|
|
6802
|
+
await this.initialize();
|
|
6803
|
+
return this.sqliteStore.getSessionTurns(sessionId, options);
|
|
6804
|
+
}
|
|
6805
|
+
/**
|
|
6806
|
+
* Get all events for a specific turn
|
|
6807
|
+
*/
|
|
6808
|
+
async getEventsByTurn(turnId) {
|
|
6809
|
+
await this.initialize();
|
|
6810
|
+
return this.sqliteStore.getEventsByTurn(turnId);
|
|
6811
|
+
}
|
|
6812
|
+
/**
|
|
6813
|
+
* Count total turns for a session
|
|
6814
|
+
*/
|
|
6815
|
+
async countSessionTurns(sessionId) {
|
|
6816
|
+
await this.initialize();
|
|
6817
|
+
return this.sqliteStore.countSessionTurns(sessionId);
|
|
6818
|
+
}
|
|
6819
|
+
/**
|
|
6820
|
+
* Backfill turn_ids from metadata for events stored before the migration
|
|
6821
|
+
*/
|
|
6822
|
+
async backfillTurnIds() {
|
|
6823
|
+
await this.initialize();
|
|
6824
|
+
return this.sqliteStore.backfillTurnIds();
|
|
6825
|
+
}
|
|
6826
|
+
/**
|
|
6827
|
+
* Delete all events for a session (for force reimport)
|
|
6828
|
+
*/
|
|
6829
|
+
async deleteSessionEvents(sessionId) {
|
|
6830
|
+
await this.initialize();
|
|
6831
|
+
return this.sqliteStore.deleteSessionEvents(sessionId);
|
|
6832
|
+
}
|
|
5215
6833
|
/**
|
|
5216
6834
|
* Format Endless Mode context for Claude
|
|
5217
6835
|
*/
|
|
@@ -5288,7 +6906,7 @@ var MemoryService = class {
|
|
|
5288
6906
|
*/
|
|
5289
6907
|
expandPath(p) {
|
|
5290
6908
|
if (p.startsWith("~")) {
|
|
5291
|
-
return
|
|
6909
|
+
return path3.join(os.homedir(), p.slice(1));
|
|
5292
6910
|
}
|
|
5293
6911
|
return p;
|
|
5294
6912
|
}
|
|
@@ -5311,6 +6929,7 @@ function getMemoryServiceForProject(projectPath, sharedStoreConfig) {
|
|
|
5311
6929
|
serviceCache.set(hash, new MemoryService({
|
|
5312
6930
|
storagePath,
|
|
5313
6931
|
projectHash: hash,
|
|
6932
|
+
projectPath,
|
|
5314
6933
|
// Override shared store config - hooks don't need DuckDB
|
|
5315
6934
|
sharedStoreConfig: sharedStoreConfig ?? { enabled: false },
|
|
5316
6935
|
analyticsEnabled: false
|
|
@@ -5320,12 +6939,36 @@ function getMemoryServiceForProject(projectPath, sharedStoreConfig) {
|
|
|
5320
6939
|
return serviceCache.get(hash);
|
|
5321
6940
|
}
|
|
5322
6941
|
|
|
6942
|
+
// src/server/api/utils.ts
|
|
6943
|
+
function getServiceFromQuery(c) {
|
|
6944
|
+
const project = c.req.query("project");
|
|
6945
|
+
if (project) {
|
|
6946
|
+
const isHash = /^[a-f0-9]{8}$/.test(project);
|
|
6947
|
+
let storagePath;
|
|
6948
|
+
if (isHash) {
|
|
6949
|
+
storagePath = path4.join(os2.homedir(), ".claude-code", "memory", "projects", project);
|
|
6950
|
+
} else {
|
|
6951
|
+
const crypto3 = __require("crypto");
|
|
6952
|
+
const normalized = project.replace(/\/+$/, "") || "/";
|
|
6953
|
+
const hash = crypto3.createHash("sha256").update(normalized).digest("hex").slice(0, 8);
|
|
6954
|
+
storagePath = path4.join(os2.homedir(), ".claude-code", "memory", "projects", hash);
|
|
6955
|
+
}
|
|
6956
|
+
return new MemoryService({
|
|
6957
|
+
storagePath,
|
|
6958
|
+
readOnly: true,
|
|
6959
|
+
analyticsEnabled: false,
|
|
6960
|
+
sharedStoreConfig: { enabled: false }
|
|
6961
|
+
});
|
|
6962
|
+
}
|
|
6963
|
+
return getReadOnlyMemoryService();
|
|
6964
|
+
}
|
|
6965
|
+
|
|
5323
6966
|
// src/server/api/sessions.ts
|
|
5324
6967
|
var sessionsRouter = new Hono();
|
|
5325
6968
|
sessionsRouter.get("/", async (c) => {
|
|
5326
6969
|
const page = parseInt(c.req.query("page") || "1", 10);
|
|
5327
6970
|
const pageSize = parseInt(c.req.query("pageSize") || "20", 10);
|
|
5328
|
-
const memoryService =
|
|
6971
|
+
const memoryService = getServiceFromQuery(c);
|
|
5329
6972
|
try {
|
|
5330
6973
|
await memoryService.initialize();
|
|
5331
6974
|
const recentEvents = await memoryService.getRecentEvents(1e3);
|
|
@@ -5369,7 +7012,7 @@ sessionsRouter.get("/", async (c) => {
|
|
|
5369
7012
|
});
|
|
5370
7013
|
sessionsRouter.get("/:id", async (c) => {
|
|
5371
7014
|
const { id } = c.req.param();
|
|
5372
|
-
const memoryService =
|
|
7015
|
+
const memoryService = getServiceFromQuery(c);
|
|
5373
7016
|
try {
|
|
5374
7017
|
await memoryService.initialize();
|
|
5375
7018
|
const events = await memoryService.getSessionHistory(id);
|
|
@@ -5410,18 +7053,36 @@ var eventsRouter = new Hono2();
|
|
|
5410
7053
|
eventsRouter.get("/", async (c) => {
|
|
5411
7054
|
const sessionId = c.req.query("sessionId");
|
|
5412
7055
|
const eventType = c.req.query("type");
|
|
7056
|
+
const level = c.req.query("level");
|
|
7057
|
+
const sort = c.req.query("sort") || "recent";
|
|
5413
7058
|
const limit = parseInt(c.req.query("limit") || "100", 10);
|
|
5414
7059
|
const offset = parseInt(c.req.query("offset") || "0", 10);
|
|
5415
|
-
const memoryService =
|
|
7060
|
+
const memoryService = getServiceFromQuery(c);
|
|
5416
7061
|
try {
|
|
5417
7062
|
await memoryService.initialize();
|
|
5418
|
-
let events
|
|
7063
|
+
let events;
|
|
7064
|
+
if (level) {
|
|
7065
|
+
events = await memoryService.getEventsByLevel(level, { limit: limit + offset + 1e3, offset: 0 });
|
|
7066
|
+
} else {
|
|
7067
|
+
events = await memoryService.getRecentEvents(limit + offset + 1e3);
|
|
7068
|
+
}
|
|
5419
7069
|
if (sessionId) {
|
|
5420
7070
|
events = events.filter((e) => e.sessionId === sessionId);
|
|
5421
7071
|
}
|
|
5422
7072
|
if (eventType) {
|
|
5423
7073
|
events = events.filter((e) => e.eventType === eventType);
|
|
5424
7074
|
}
|
|
7075
|
+
if (sort === "accessed") {
|
|
7076
|
+
events.sort((a, b) => {
|
|
7077
|
+
const aTime = a.last_accessed_at || "";
|
|
7078
|
+
const bTime = b.last_accessed_at || "";
|
|
7079
|
+
return bTime.localeCompare(aTime);
|
|
7080
|
+
});
|
|
7081
|
+
} else if (sort === "most-accessed") {
|
|
7082
|
+
events.sort((a, b) => (b.access_count || 0) - (a.access_count || 0));
|
|
7083
|
+
} else if (sort === "oldest") {
|
|
7084
|
+
events.sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime());
|
|
7085
|
+
}
|
|
5425
7086
|
const total = events.length;
|
|
5426
7087
|
events = events.slice(offset, offset + limit);
|
|
5427
7088
|
return c.json({
|
|
@@ -5431,7 +7092,9 @@ eventsRouter.get("/", async (c) => {
|
|
|
5431
7092
|
timestamp: e.timestamp,
|
|
5432
7093
|
sessionId: e.sessionId,
|
|
5433
7094
|
preview: e.content.slice(0, 200) + (e.content.length > 200 ? "..." : ""),
|
|
5434
|
-
contentLength: e.content.length
|
|
7095
|
+
contentLength: e.content.length,
|
|
7096
|
+
accessCount: e.access_count || 0,
|
|
7097
|
+
lastAccessedAt: e.last_accessed_at || null
|
|
5435
7098
|
})),
|
|
5436
7099
|
total,
|
|
5437
7100
|
limit,
|
|
@@ -5446,7 +7109,7 @@ eventsRouter.get("/", async (c) => {
|
|
|
5446
7109
|
});
|
|
5447
7110
|
eventsRouter.get("/:id", async (c) => {
|
|
5448
7111
|
const { id } = c.req.param();
|
|
5449
|
-
const memoryService =
|
|
7112
|
+
const memoryService = getServiceFromQuery(c);
|
|
5450
7113
|
try {
|
|
5451
7114
|
await memoryService.initialize();
|
|
5452
7115
|
const recentEvents = await memoryService.getRecentEvents(1e4);
|
|
@@ -5486,7 +7149,7 @@ eventsRouter.get("/:id", async (c) => {
|
|
|
5486
7149
|
import { Hono as Hono3 } from "hono";
|
|
5487
7150
|
var searchRouter = new Hono3();
|
|
5488
7151
|
searchRouter.post("/", async (c) => {
|
|
5489
|
-
const memoryService =
|
|
7152
|
+
const memoryService = getServiceFromQuery(c);
|
|
5490
7153
|
try {
|
|
5491
7154
|
const body = await c.req.json();
|
|
5492
7155
|
if (!body.query) {
|
|
@@ -5530,7 +7193,7 @@ searchRouter.get("/", async (c) => {
|
|
|
5530
7193
|
return c.json({ error: 'Query parameter "q" is required' }, 400);
|
|
5531
7194
|
}
|
|
5532
7195
|
const topK = parseInt(c.req.query("topK") || "5", 10);
|
|
5533
|
-
const memoryService =
|
|
7196
|
+
const memoryService = getServiceFromQuery(c);
|
|
5534
7197
|
try {
|
|
5535
7198
|
await memoryService.initialize();
|
|
5536
7199
|
const result = await memoryService.retrieveMemories(query, { topK });
|
|
@@ -5558,7 +7221,7 @@ searchRouter.get("/", async (c) => {
|
|
|
5558
7221
|
import { Hono as Hono4 } from "hono";
|
|
5559
7222
|
var statsRouter = new Hono4();
|
|
5560
7223
|
statsRouter.get("/shared", async (c) => {
|
|
5561
|
-
const memoryService =
|
|
7224
|
+
const memoryService = getServiceFromQuery(c);
|
|
5562
7225
|
try {
|
|
5563
7226
|
await memoryService.initialize();
|
|
5564
7227
|
const sharedStats = await memoryService.getSharedStoreStats();
|
|
@@ -5615,7 +7278,7 @@ statsRouter.get("/levels/:level", async (c) => {
|
|
|
5615
7278
|
if (!validLevels.includes(level)) {
|
|
5616
7279
|
return c.json({ error: `Invalid level. Must be one of: ${validLevels.join(", ")}` }, 400);
|
|
5617
7280
|
}
|
|
5618
|
-
const memoryService =
|
|
7281
|
+
const memoryService = getServiceFromQuery(c);
|
|
5619
7282
|
try {
|
|
5620
7283
|
await memoryService.initialize();
|
|
5621
7284
|
let events = await memoryService.getEventsByLevel(level, { limit: limit * 2, offset });
|
|
@@ -5662,7 +7325,7 @@ statsRouter.get("/levels/:level", async (c) => {
|
|
|
5662
7325
|
}
|
|
5663
7326
|
});
|
|
5664
7327
|
statsRouter.get("/", async (c) => {
|
|
5665
|
-
const memoryService =
|
|
7328
|
+
const memoryService = getServiceFromQuery(c);
|
|
5666
7329
|
try {
|
|
5667
7330
|
await memoryService.initialize();
|
|
5668
7331
|
const stats = await memoryService.getStats();
|
|
@@ -5679,6 +7342,7 @@ statsRouter.get("/", async (c) => {
|
|
|
5679
7342
|
acc[day] = (acc[day] || 0) + 1;
|
|
5680
7343
|
return acc;
|
|
5681
7344
|
}, {});
|
|
7345
|
+
const retrievalTrace = await memoryService.getRetrievalTraceStats();
|
|
5682
7346
|
return c.json({
|
|
5683
7347
|
storage: {
|
|
5684
7348
|
eventCount: stats.totalEvents,
|
|
@@ -5696,7 +7360,8 @@ statsRouter.get("/", async (c) => {
|
|
|
5696
7360
|
heapUsed: Math.round(process.memoryUsage().heapUsed / 1024 / 1024),
|
|
5697
7361
|
heapTotal: Math.round(process.memoryUsage().heapTotal / 1024 / 1024)
|
|
5698
7362
|
},
|
|
5699
|
-
levelStats: stats.levelStats
|
|
7363
|
+
levelStats: stats.levelStats,
|
|
7364
|
+
retrievalTrace
|
|
5700
7365
|
});
|
|
5701
7366
|
} catch (error) {
|
|
5702
7367
|
return c.json({ error: error.message }, 500);
|
|
@@ -5706,7 +7371,7 @@ statsRouter.get("/", async (c) => {
|
|
|
5706
7371
|
});
|
|
5707
7372
|
statsRouter.get("/most-accessed", async (c) => {
|
|
5708
7373
|
const limit = parseInt(c.req.query("limit") || "10", 10);
|
|
5709
|
-
const memoryService =
|
|
7374
|
+
const memoryService = getServiceFromQuery(c);
|
|
5710
7375
|
try {
|
|
5711
7376
|
await memoryService.initialize();
|
|
5712
7377
|
console.log("[most-accessed] Fetching most accessed memories, limit:", limit);
|
|
@@ -5737,7 +7402,7 @@ statsRouter.get("/most-accessed", async (c) => {
|
|
|
5737
7402
|
});
|
|
5738
7403
|
statsRouter.get("/timeline", async (c) => {
|
|
5739
7404
|
const days = parseInt(c.req.query("days") || "7", 10);
|
|
5740
|
-
const memoryService =
|
|
7405
|
+
const memoryService = getServiceFromQuery(c);
|
|
5741
7406
|
try {
|
|
5742
7407
|
await memoryService.initialize();
|
|
5743
7408
|
const recentEvents = await memoryService.getRecentEvents(1e4);
|
|
@@ -5767,8 +7432,75 @@ statsRouter.get("/timeline", async (c) => {
|
|
|
5767
7432
|
await memoryService.shutdown();
|
|
5768
7433
|
}
|
|
5769
7434
|
});
|
|
7435
|
+
statsRouter.get("/helpfulness", async (c) => {
|
|
7436
|
+
const limit = parseInt(c.req.query("limit") || "10", 10);
|
|
7437
|
+
const memoryService = getServiceFromQuery(c);
|
|
7438
|
+
try {
|
|
7439
|
+
await memoryService.initialize();
|
|
7440
|
+
const stats = await memoryService.getHelpfulnessStats();
|
|
7441
|
+
const topMemories = await memoryService.getHelpfulMemories(limit);
|
|
7442
|
+
return c.json({
|
|
7443
|
+
...stats,
|
|
7444
|
+
topMemories: topMemories.map((m) => ({
|
|
7445
|
+
eventId: m.eventId,
|
|
7446
|
+
summary: m.summary,
|
|
7447
|
+
helpfulnessScore: m.helpfulnessScore,
|
|
7448
|
+
accessCount: m.accessCount,
|
|
7449
|
+
evaluationCount: m.evaluationCount
|
|
7450
|
+
}))
|
|
7451
|
+
});
|
|
7452
|
+
} catch (error) {
|
|
7453
|
+
return c.json({
|
|
7454
|
+
avgScore: 0,
|
|
7455
|
+
totalEvaluated: 0,
|
|
7456
|
+
totalRetrievals: 0,
|
|
7457
|
+
helpful: 0,
|
|
7458
|
+
neutral: 0,
|
|
7459
|
+
unhelpful: 0,
|
|
7460
|
+
topMemories: []
|
|
7461
|
+
});
|
|
7462
|
+
} finally {
|
|
7463
|
+
await memoryService.shutdown();
|
|
7464
|
+
}
|
|
7465
|
+
});
|
|
7466
|
+
statsRouter.get("/retrieval-traces", async (c) => {
|
|
7467
|
+
const limit = parseInt(c.req.query("limit") || "50", 10);
|
|
7468
|
+
const memoryService = getServiceFromQuery(c);
|
|
7469
|
+
try {
|
|
7470
|
+
await memoryService.initialize();
|
|
7471
|
+
const traces = await memoryService.getRecentRetrievalTraces(limit);
|
|
7472
|
+
const traceStats = await memoryService.getRetrievalTraceStats();
|
|
7473
|
+
return c.json({
|
|
7474
|
+
stats: traceStats,
|
|
7475
|
+
traces: traces.map((t) => ({
|
|
7476
|
+
traceId: t.traceId,
|
|
7477
|
+
sessionId: t.sessionId || null,
|
|
7478
|
+
projectHash: t.projectHash || null,
|
|
7479
|
+
queryText: t.queryText,
|
|
7480
|
+
strategy: t.strategy || null,
|
|
7481
|
+
candidateEventIds: t.candidateEventIds,
|
|
7482
|
+
selectedEventIds: t.selectedEventIds,
|
|
7483
|
+
candidateDetails: t.candidateDetails || [],
|
|
7484
|
+
selectedDetails: t.selectedDetails || [],
|
|
7485
|
+
candidateCount: t.candidateCount,
|
|
7486
|
+
selectedCount: t.selectedCount,
|
|
7487
|
+
confidence: t.confidence || null,
|
|
7488
|
+
fallbackTrace: t.fallbackTrace,
|
|
7489
|
+
createdAt: t.createdAt.toISOString()
|
|
7490
|
+
}))
|
|
7491
|
+
});
|
|
7492
|
+
} catch (error) {
|
|
7493
|
+
return c.json({
|
|
7494
|
+
stats: { totalQueries: 0, avgCandidateCount: 0, avgSelectedCount: 0, selectionRate: 0 },
|
|
7495
|
+
traces: [],
|
|
7496
|
+
error: error.message
|
|
7497
|
+
}, 500);
|
|
7498
|
+
} finally {
|
|
7499
|
+
await memoryService.shutdown();
|
|
7500
|
+
}
|
|
7501
|
+
});
|
|
5770
7502
|
statsRouter.post("/graduation/run", async (c) => {
|
|
5771
|
-
const memoryService =
|
|
7503
|
+
const memoryService = getServiceFromQuery(c);
|
|
5772
7504
|
try {
|
|
5773
7505
|
await memoryService.initialize();
|
|
5774
7506
|
const result = await memoryService.forceGraduation();
|
|
@@ -5829,7 +7561,7 @@ var citationsRouter = new Hono5();
|
|
|
5829
7561
|
citationsRouter.get("/:id", async (c) => {
|
|
5830
7562
|
const { id } = c.req.param();
|
|
5831
7563
|
const citationId = parseCitationId(id) || id;
|
|
5832
|
-
const memoryService =
|
|
7564
|
+
const memoryService = getServiceFromQuery(c);
|
|
5833
7565
|
try {
|
|
5834
7566
|
await memoryService.initialize();
|
|
5835
7567
|
const recentEvents = await memoryService.getRecentEvents(1e4);
|
|
@@ -5863,7 +7595,7 @@ citationsRouter.get("/:id", async (c) => {
|
|
|
5863
7595
|
citationsRouter.get("/:id/related", async (c) => {
|
|
5864
7596
|
const { id } = c.req.param();
|
|
5865
7597
|
const citationId = parseCitationId(id) || id;
|
|
5866
|
-
const memoryService =
|
|
7598
|
+
const memoryService = getServiceFromQuery(c);
|
|
5867
7599
|
try {
|
|
5868
7600
|
await memoryService.initialize();
|
|
5869
7601
|
const recentEvents = await memoryService.getRecentEvents(1e4);
|
|
@@ -5899,8 +7631,400 @@ citationsRouter.get("/:id/related", async (c) => {
|
|
|
5899
7631
|
}
|
|
5900
7632
|
});
|
|
5901
7633
|
|
|
7634
|
+
// src/server/api/turns.ts
|
|
7635
|
+
import { Hono as Hono6 } from "hono";
|
|
7636
|
+
var turnsRouter = new Hono6();
|
|
7637
|
+
turnsRouter.get("/", async (c) => {
|
|
7638
|
+
const sessionId = c.req.query("sessionId");
|
|
7639
|
+
const limit = parseInt(c.req.query("limit") || "20", 10);
|
|
7640
|
+
const offset = parseInt(c.req.query("offset") || "0", 10);
|
|
7641
|
+
if (!sessionId) {
|
|
7642
|
+
return c.json({ error: "sessionId is required" }, 400);
|
|
7643
|
+
}
|
|
7644
|
+
const memoryService = getServiceFromQuery(c);
|
|
7645
|
+
try {
|
|
7646
|
+
await memoryService.initialize();
|
|
7647
|
+
const turns = await memoryService.getSessionTurns(sessionId, { limit, offset });
|
|
7648
|
+
const totalTurns = await memoryService.countSessionTurns(sessionId);
|
|
7649
|
+
return c.json({
|
|
7650
|
+
turns: turns.map((t) => ({
|
|
7651
|
+
turnId: t.turnId,
|
|
7652
|
+
startedAt: t.startedAt.toISOString(),
|
|
7653
|
+
promptPreview: t.promptPreview,
|
|
7654
|
+
eventCount: t.eventCount,
|
|
7655
|
+
toolCount: t.toolCount,
|
|
7656
|
+
hasResponse: t.hasResponse,
|
|
7657
|
+
events: t.events.map((e) => ({
|
|
7658
|
+
id: e.id,
|
|
7659
|
+
eventType: e.eventType,
|
|
7660
|
+
timestamp: e.timestamp instanceof Date ? e.timestamp.toISOString() : e.timestamp,
|
|
7661
|
+
preview: e.content.slice(0, 300) + (e.content.length > 300 ? "..." : ""),
|
|
7662
|
+
contentLength: e.content.length
|
|
7663
|
+
}))
|
|
7664
|
+
})),
|
|
7665
|
+
total: totalTurns,
|
|
7666
|
+
limit,
|
|
7667
|
+
offset,
|
|
7668
|
+
hasMore: offset + limit < totalTurns
|
|
7669
|
+
});
|
|
7670
|
+
} catch (error) {
|
|
7671
|
+
return c.json({ error: error.message }, 500);
|
|
7672
|
+
} finally {
|
|
7673
|
+
await memoryService.shutdown();
|
|
7674
|
+
}
|
|
7675
|
+
});
|
|
7676
|
+
turnsRouter.get("/:turnId", async (c) => {
|
|
7677
|
+
const { turnId } = c.req.param();
|
|
7678
|
+
const memoryService = getServiceFromQuery(c);
|
|
7679
|
+
try {
|
|
7680
|
+
await memoryService.initialize();
|
|
7681
|
+
const events = await memoryService.getEventsByTurn(turnId);
|
|
7682
|
+
if (events.length === 0) {
|
|
7683
|
+
return c.json({ error: "Turn not found" }, 404);
|
|
7684
|
+
}
|
|
7685
|
+
const promptEvent = events.find((e) => e.eventType === "user_prompt");
|
|
7686
|
+
const toolEvents = events.filter((e) => e.eventType === "tool_observation");
|
|
7687
|
+
const responseEvents = events.filter((e) => e.eventType === "agent_response");
|
|
7688
|
+
return c.json({
|
|
7689
|
+
turnId,
|
|
7690
|
+
sessionId: events[0].sessionId,
|
|
7691
|
+
startedAt: events[0].timestamp instanceof Date ? events[0].timestamp.toISOString() : events[0].timestamp,
|
|
7692
|
+
prompt: promptEvent ? {
|
|
7693
|
+
id: promptEvent.id,
|
|
7694
|
+
content: promptEvent.content,
|
|
7695
|
+
timestamp: promptEvent.timestamp instanceof Date ? promptEvent.timestamp.toISOString() : promptEvent.timestamp
|
|
7696
|
+
} : null,
|
|
7697
|
+
tools: toolEvents.map((e) => {
|
|
7698
|
+
let toolName = "";
|
|
7699
|
+
let success = true;
|
|
7700
|
+
try {
|
|
7701
|
+
const parsed = JSON.parse(e.content);
|
|
7702
|
+
toolName = parsed.toolName || "";
|
|
7703
|
+
success = parsed.success !== false;
|
|
7704
|
+
} catch {
|
|
7705
|
+
}
|
|
7706
|
+
return {
|
|
7707
|
+
id: e.id,
|
|
7708
|
+
toolName,
|
|
7709
|
+
success,
|
|
7710
|
+
timestamp: e.timestamp instanceof Date ? e.timestamp.toISOString() : e.timestamp,
|
|
7711
|
+
preview: e.content.slice(0, 500) + (e.content.length > 500 ? "..." : "")
|
|
7712
|
+
};
|
|
7713
|
+
}),
|
|
7714
|
+
responses: responseEvents.map((e) => ({
|
|
7715
|
+
id: e.id,
|
|
7716
|
+
content: e.content,
|
|
7717
|
+
timestamp: e.timestamp instanceof Date ? e.timestamp.toISOString() : e.timestamp
|
|
7718
|
+
})),
|
|
7719
|
+
totalEvents: events.length
|
|
7720
|
+
});
|
|
7721
|
+
} catch (error) {
|
|
7722
|
+
return c.json({ error: error.message }, 500);
|
|
7723
|
+
} finally {
|
|
7724
|
+
await memoryService.shutdown();
|
|
7725
|
+
}
|
|
7726
|
+
});
|
|
7727
|
+
turnsRouter.post("/backfill", async (c) => {
|
|
7728
|
+
const memoryService = getServiceFromQuery(c);
|
|
7729
|
+
try {
|
|
7730
|
+
await memoryService.initialize();
|
|
7731
|
+
const updated = await memoryService.backfillTurnIds();
|
|
7732
|
+
return c.json({
|
|
7733
|
+
success: true,
|
|
7734
|
+
updated,
|
|
7735
|
+
message: `Backfilled turn_id for ${updated} events`
|
|
7736
|
+
});
|
|
7737
|
+
} catch (error) {
|
|
7738
|
+
return c.json({
|
|
7739
|
+
success: false,
|
|
7740
|
+
error: error.message
|
|
7741
|
+
}, 500);
|
|
7742
|
+
} finally {
|
|
7743
|
+
await memoryService.shutdown();
|
|
7744
|
+
}
|
|
7745
|
+
});
|
|
7746
|
+
|
|
7747
|
+
// src/server/api/projects.ts
|
|
7748
|
+
import { Hono as Hono7 } from "hono";
|
|
7749
|
+
import * as fs5 from "fs";
|
|
7750
|
+
import * as path5 from "path";
|
|
7751
|
+
import * as os3 from "os";
|
|
7752
|
+
var projectsRouter = new Hono7();
|
|
7753
|
+
projectsRouter.get("/", async (c) => {
|
|
7754
|
+
try {
|
|
7755
|
+
const projectsDir = path5.join(os3.homedir(), ".claude-code", "memory", "projects");
|
|
7756
|
+
if (!fs5.existsSync(projectsDir)) {
|
|
7757
|
+
return c.json({ projects: [] });
|
|
7758
|
+
}
|
|
7759
|
+
const projectHashes = fs5.readdirSync(projectsDir).filter((name) => {
|
|
7760
|
+
const fullPath = path5.join(projectsDir, name);
|
|
7761
|
+
return fs5.statSync(fullPath).isDirectory();
|
|
7762
|
+
});
|
|
7763
|
+
const registry = loadSessionRegistry();
|
|
7764
|
+
const hashToPath = /* @__PURE__ */ new Map();
|
|
7765
|
+
for (const entry of Object.values(registry.sessions)) {
|
|
7766
|
+
if (!hashToPath.has(entry.projectHash)) {
|
|
7767
|
+
hashToPath.set(entry.projectHash, entry.projectPath);
|
|
7768
|
+
}
|
|
7769
|
+
}
|
|
7770
|
+
const projects = projectHashes.map((hash) => {
|
|
7771
|
+
const dirPath = path5.join(projectsDir, hash);
|
|
7772
|
+
const dbPath = path5.join(dirPath, "events.sqlite");
|
|
7773
|
+
let dbSize = 0;
|
|
7774
|
+
if (fs5.existsSync(dbPath)) {
|
|
7775
|
+
dbSize = fs5.statSync(dbPath).size;
|
|
7776
|
+
}
|
|
7777
|
+
const projectPath = hashToPath.get(hash) || `unknown (${hash})`;
|
|
7778
|
+
return {
|
|
7779
|
+
hash,
|
|
7780
|
+
projectPath,
|
|
7781
|
+
projectName: path5.basename(projectPath),
|
|
7782
|
+
dbSize,
|
|
7783
|
+
dbSizeHuman: formatBytes(dbSize)
|
|
7784
|
+
};
|
|
7785
|
+
});
|
|
7786
|
+
projects.sort((a, b) => a.projectName.localeCompare(b.projectName));
|
|
7787
|
+
return c.json({ projects });
|
|
7788
|
+
} catch (error) {
|
|
7789
|
+
return c.json({ projects: [], error: error.message }, 500);
|
|
7790
|
+
}
|
|
7791
|
+
});
|
|
7792
|
+
function formatBytes(bytes) {
|
|
7793
|
+
if (bytes === 0)
|
|
7794
|
+
return "0 B";
|
|
7795
|
+
const k = 1024;
|
|
7796
|
+
const sizes = ["B", "KB", "MB", "GB"];
|
|
7797
|
+
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
|
7798
|
+
return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + " " + sizes[i];
|
|
7799
|
+
}
|
|
7800
|
+
|
|
7801
|
+
// src/server/api/chat.ts
|
|
7802
|
+
import { Hono as Hono8 } from "hono";
|
|
7803
|
+
import { streamSSE } from "hono/streaming";
|
|
7804
|
+
import { spawn } from "child_process";
|
|
7805
|
+
var chatRouter = new Hono8();
|
|
7806
|
+
var CLAUDE_TIMEOUT_MS = 12e4;
|
|
7807
|
+
chatRouter.post("/", async (c) => {
|
|
7808
|
+
let body;
|
|
7809
|
+
try {
|
|
7810
|
+
body = await c.req.json();
|
|
7811
|
+
} catch {
|
|
7812
|
+
return c.json({ error: "Invalid JSON body" }, 400);
|
|
7813
|
+
}
|
|
7814
|
+
if (!body.message?.trim()) {
|
|
7815
|
+
return c.json({ error: "Message is required" }, 400);
|
|
7816
|
+
}
|
|
7817
|
+
const memoryService = getServiceFromQuery(c);
|
|
7818
|
+
try {
|
|
7819
|
+
await memoryService.initialize();
|
|
7820
|
+
let memoryContext = "";
|
|
7821
|
+
let statsContext = "";
|
|
7822
|
+
try {
|
|
7823
|
+
const result = await memoryService.retrieveMemories(body.message, {
|
|
7824
|
+
topK: 8,
|
|
7825
|
+
minScore: 0.5
|
|
7826
|
+
});
|
|
7827
|
+
if (result.memories.length > 0) {
|
|
7828
|
+
const parts = ["## Relevant Memories\n"];
|
|
7829
|
+
for (const m of result.memories) {
|
|
7830
|
+
const date = new Date(m.event.timestamp).toISOString().split("T")[0];
|
|
7831
|
+
const content = m.event.content.slice(0, 500);
|
|
7832
|
+
parts.push(`### [${m.event.eventType}] ${date} (score: ${m.score.toFixed(2)})`);
|
|
7833
|
+
parts.push(content);
|
|
7834
|
+
if (m.sessionContext) {
|
|
7835
|
+
parts.push(`_Context: ${m.sessionContext}_`);
|
|
7836
|
+
}
|
|
7837
|
+
parts.push("");
|
|
7838
|
+
}
|
|
7839
|
+
memoryContext = parts.join("\n");
|
|
7840
|
+
}
|
|
7841
|
+
} catch {
|
|
7842
|
+
}
|
|
7843
|
+
try {
|
|
7844
|
+
const stats = await memoryService.getStats();
|
|
7845
|
+
const levels = stats.levelStats.map((l) => `${l.level}: ${l.count}`).join(", ");
|
|
7846
|
+
statsContext = [
|
|
7847
|
+
"## Memory Stats",
|
|
7848
|
+
`- Total events: ${stats.totalEvents}`,
|
|
7849
|
+
`- Vector nodes: ${stats.vectorCount}`,
|
|
7850
|
+
`- By level: ${levels}`
|
|
7851
|
+
].join("\n");
|
|
7852
|
+
} catch {
|
|
7853
|
+
}
|
|
7854
|
+
const fullPrompt = buildPrompt(
|
|
7855
|
+
statsContext,
|
|
7856
|
+
memoryContext,
|
|
7857
|
+
body.history || [],
|
|
7858
|
+
body.message
|
|
7859
|
+
);
|
|
7860
|
+
return streamSSE(c, async (stream) => {
|
|
7861
|
+
try {
|
|
7862
|
+
await streamClaudeResponse(fullPrompt, stream);
|
|
7863
|
+
} catch (err) {
|
|
7864
|
+
await stream.writeSSE({
|
|
7865
|
+
event: "error",
|
|
7866
|
+
data: JSON.stringify({ error: err.message })
|
|
7867
|
+
});
|
|
7868
|
+
}
|
|
7869
|
+
});
|
|
7870
|
+
} catch (error) {
|
|
7871
|
+
return c.json({ error: error.message }, 500);
|
|
7872
|
+
} finally {
|
|
7873
|
+
await memoryService.shutdown();
|
|
7874
|
+
}
|
|
7875
|
+
});
|
|
7876
|
+
function buildPrompt(statsContext, memoryContext, history, currentMessage) {
|
|
7877
|
+
const parts = [];
|
|
7878
|
+
parts.push("You are a helpful assistant that answers questions about the user's code memory data.");
|
|
7879
|
+
parts.push("The memory system tracks coding sessions, tool usage, prompts, and responses.");
|
|
7880
|
+
parts.push("Answer concisely based on the memory context below. If you don't have enough data, say so.");
|
|
7881
|
+
parts.push("Use markdown formatting in your responses.\n");
|
|
7882
|
+
if (statsContext) {
|
|
7883
|
+
parts.push(statsContext);
|
|
7884
|
+
parts.push("");
|
|
7885
|
+
}
|
|
7886
|
+
if (memoryContext) {
|
|
7887
|
+
parts.push(memoryContext);
|
|
7888
|
+
} else {
|
|
7889
|
+
parts.push("No directly relevant memories found for this query.");
|
|
7890
|
+
parts.push("Answer based on general knowledge or suggest the user rephrase.\n");
|
|
7891
|
+
}
|
|
7892
|
+
parts.push("---\n");
|
|
7893
|
+
const recentHistory = history.slice(-10);
|
|
7894
|
+
if (recentHistory.length > 0) {
|
|
7895
|
+
parts.push("## Conversation History\n");
|
|
7896
|
+
for (const msg of recentHistory) {
|
|
7897
|
+
const prefix = msg.role === "user" ? "User" : "Assistant";
|
|
7898
|
+
parts.push(`**${prefix}:** ${msg.content}
|
|
7899
|
+
`);
|
|
7900
|
+
}
|
|
7901
|
+
}
|
|
7902
|
+
parts.push(`**User:** ${currentMessage}`);
|
|
7903
|
+
return parts.join("\n");
|
|
7904
|
+
}
|
|
7905
|
+
function streamClaudeResponse(prompt, stream) {
|
|
7906
|
+
return new Promise((resolve2, reject) => {
|
|
7907
|
+
const proc = spawn("claude", [
|
|
7908
|
+
"-p",
|
|
7909
|
+
"--output-format",
|
|
7910
|
+
"stream-json",
|
|
7911
|
+
"--verbose"
|
|
7912
|
+
], {
|
|
7913
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
7914
|
+
env: { ...process.env }
|
|
7915
|
+
});
|
|
7916
|
+
const timeout = setTimeout(() => {
|
|
7917
|
+
proc.kill("SIGTERM");
|
|
7918
|
+
reject(new Error("Chat response timed out after 2 minutes"));
|
|
7919
|
+
}, CLAUDE_TIMEOUT_MS);
|
|
7920
|
+
proc.stdin.write(prompt);
|
|
7921
|
+
proc.stdin.end();
|
|
7922
|
+
let buffer = "";
|
|
7923
|
+
let lastSentText = "";
|
|
7924
|
+
proc.stdout.on("data", async (chunk) => {
|
|
7925
|
+
buffer += chunk.toString();
|
|
7926
|
+
const lines = buffer.split("\n");
|
|
7927
|
+
buffer = lines.pop() || "";
|
|
7928
|
+
for (const line of lines) {
|
|
7929
|
+
if (!line.trim())
|
|
7930
|
+
continue;
|
|
7931
|
+
try {
|
|
7932
|
+
const parsed = JSON.parse(line);
|
|
7933
|
+
if (parsed.type === "assistant" && parsed.message?.content) {
|
|
7934
|
+
const textBlocks = parsed.message.content.filter((b) => b.type === "text").map((b) => b.text).join("");
|
|
7935
|
+
if (textBlocks.length > lastSentText.length) {
|
|
7936
|
+
const delta = textBlocks.slice(lastSentText.length);
|
|
7937
|
+
lastSentText = textBlocks;
|
|
7938
|
+
await stream.writeSSE({
|
|
7939
|
+
event: "message",
|
|
7940
|
+
data: JSON.stringify({ content: delta })
|
|
7941
|
+
});
|
|
7942
|
+
}
|
|
7943
|
+
}
|
|
7944
|
+
if (parsed.type === "result") {
|
|
7945
|
+
await stream.writeSSE({ event: "done", data: "{}" });
|
|
7946
|
+
}
|
|
7947
|
+
} catch {
|
|
7948
|
+
}
|
|
7949
|
+
}
|
|
7950
|
+
});
|
|
7951
|
+
proc.stderr.on("data", (chunk) => {
|
|
7952
|
+
if (process.env.CLAUDE_MEMORY_DEBUG) {
|
|
7953
|
+
console.error("[chat] claude stderr:", chunk.toString());
|
|
7954
|
+
}
|
|
7955
|
+
});
|
|
7956
|
+
proc.on("error", (err) => {
|
|
7957
|
+
clearTimeout(timeout);
|
|
7958
|
+
if (err.code === "ENOENT") {
|
|
7959
|
+
reject(new Error("Claude CLI not found. Install with: npm install -g @anthropic-ai/claude-code"));
|
|
7960
|
+
} else {
|
|
7961
|
+
reject(err);
|
|
7962
|
+
}
|
|
7963
|
+
});
|
|
7964
|
+
proc.on("close", async (code) => {
|
|
7965
|
+
clearTimeout(timeout);
|
|
7966
|
+
if (buffer.trim()) {
|
|
7967
|
+
try {
|
|
7968
|
+
const parsed = JSON.parse(buffer);
|
|
7969
|
+
if (parsed.type === "result") {
|
|
7970
|
+
await stream.writeSSE({ event: "done", data: "{}" });
|
|
7971
|
+
}
|
|
7972
|
+
} catch {
|
|
7973
|
+
}
|
|
7974
|
+
}
|
|
7975
|
+
if (code !== 0 && code !== null) {
|
|
7976
|
+
reject(new Error(`Claude CLI exited with code ${code}`));
|
|
7977
|
+
} else {
|
|
7978
|
+
resolve2();
|
|
7979
|
+
}
|
|
7980
|
+
});
|
|
7981
|
+
});
|
|
7982
|
+
}
|
|
7983
|
+
|
|
7984
|
+
// src/server/api/health.ts
|
|
7985
|
+
import { Hono as Hono9 } from "hono";
|
|
7986
|
+
var healthRouter = new Hono9();
|
|
7987
|
+
healthRouter.get("/", async (c) => {
|
|
7988
|
+
const memoryService = getServiceFromQuery(c);
|
|
7989
|
+
try {
|
|
7990
|
+
await memoryService.initialize();
|
|
7991
|
+
const [stats, outbox] = await Promise.all([
|
|
7992
|
+
memoryService.getStats(),
|
|
7993
|
+
memoryService.getOutboxStats()
|
|
7994
|
+
]);
|
|
7995
|
+
const outboxPending = outbox.embedding.pending + outbox.vector.pending;
|
|
7996
|
+
const outboxFailed = outbox.embedding.failed + outbox.vector.failed;
|
|
7997
|
+
const status = outboxFailed > 0 ? "needs-attention" : "ok";
|
|
7998
|
+
return c.json({
|
|
7999
|
+
status,
|
|
8000
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
8001
|
+
storage: {
|
|
8002
|
+
totalEvents: stats.totalEvents,
|
|
8003
|
+
vectorCount: stats.vectorCount
|
|
8004
|
+
},
|
|
8005
|
+
outbox: {
|
|
8006
|
+
embedding: outbox.embedding,
|
|
8007
|
+
vector: outbox.vector,
|
|
8008
|
+
totals: {
|
|
8009
|
+
pending: outboxPending,
|
|
8010
|
+
failed: outboxFailed
|
|
8011
|
+
}
|
|
8012
|
+
},
|
|
8013
|
+
levelStats: stats.levelStats
|
|
8014
|
+
});
|
|
8015
|
+
} catch (error) {
|
|
8016
|
+
return c.json({
|
|
8017
|
+
status: "error",
|
|
8018
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
8019
|
+
error: error.message
|
|
8020
|
+
}, 500);
|
|
8021
|
+
} finally {
|
|
8022
|
+
await memoryService.shutdown();
|
|
8023
|
+
}
|
|
8024
|
+
});
|
|
8025
|
+
|
|
5902
8026
|
// src/server/api/index.ts
|
|
5903
|
-
var apiRouter = new
|
|
8027
|
+
var apiRouter = new Hono10().route("/sessions", sessionsRouter).route("/events", eventsRouter).route("/search", searchRouter).route("/stats", statsRouter).route("/citations", citationsRouter).route("/turns", turnsRouter).route("/projects", projectsRouter).route("/chat", chatRouter).route("/health", healthRouter);
|
|
5904
8028
|
export {
|
|
5905
8029
|
apiRouter
|
|
5906
8030
|
};
|