claude-memory-layer 1.0.30 → 1.0.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -5
- package/dist/cli/index.js +4 -3
- package/dist/cli/index.js.map +2 -2
- package/dist/core/index.js +3 -2
- package/dist/core/index.js.map +2 -2
- package/dist/hooks/post-tool-use.js +3 -2
- package/dist/hooks/post-tool-use.js.map +2 -2
- package/dist/hooks/semantic-daemon.js +3 -2
- package/dist/hooks/semantic-daemon.js.map +2 -2
- package/dist/hooks/session-end.js +3 -2
- package/dist/hooks/session-end.js.map +2 -2
- package/dist/hooks/session-start.js +3 -2
- package/dist/hooks/session-start.js.map +2 -2
- package/dist/hooks/stop.js +3 -2
- package/dist/hooks/stop.js.map +2 -2
- package/dist/hooks/user-prompt-submit.js +3 -2
- package/dist/hooks/user-prompt-submit.js.map +2 -2
- package/dist/index.js +3 -2
- package/dist/index.js.map +2 -2
- package/dist/mcp/index.js +3 -2
- package/dist/mcp/index.js.map +2 -2
- package/dist/server/api/index.js +3 -2
- package/dist/server/api/index.js.map +2 -2
- package/dist/server/index.js +3 -2
- package/dist/server/index.js.map +2 -2
- package/dist/services/memory-service.js +3 -2
- package/dist/services/memory-service.js.map +2 -2
- package/package.json +10 -3
- package/scripts/postinstall-embedding-backend.cjs +18 -16
- package/AGENTS.md +0 -71
- package/CLAUDE.md +0 -30
- package/HANDOFF.md +0 -92
- package/Memo.txt +0 -558
- package/benchmarks/replay/anonymized-real-sessions.json +0 -48
- package/config/kpi-thresholds.json +0 -7
- package/context.md +0 -636
- package/docs/ARCHITECTURE_COMPARISON_AND_RECOMMENDATIONS.md +0 -627
- package/docs/HERMES_MEMORY_INGESTION_ANALYSIS.md +0 -440
- package/docs/MCP_MEMORY_SERVICE_COMPARATIVE_REVIEW.md +0 -271
- package/docs/MEMORY_USEFULNESS_AUDIT.md +0 -371
- package/docs/MEMORY_USEFULNESS_AUDIT_RAW.json +0 -80
- package/docs/MEMSEARCH_PROJECT_STRUCTURE_ANALYSIS.md +0 -333
- package/docs/MEMU_ADOPTION.md +0 -40
- package/docs/OPERATIONS.md +0 -18
- package/docs/PRODUCT_VALIDATION_MATRIX.md +0 -82
- package/docs/PROJECT_STRUCTURE_ANALYSIS.md +0 -421
- package/docs/REFACTORING_MILESTONES_AND_ISSUES.md +0 -501
- package/docs/REFACTORING_PLAN_THIN_CORE.md +0 -414
- package/docs/REFERENCE_PROJECT_ANALYSES.md +0 -25
- package/docs/SUPERLOCALMEMORY_PROJECT_STRUCTURE_ANALYSIS.md +0 -452
- package/docs/TARGET_ARCHITECTURE_AND_FOLDER_STRUCTURE.md +0 -446
- package/docs/architecture/comparison-index.md +0 -47
- package/docs/reports/codex-real-data-validation-20260505T040447Z.md +0 -46
- package/plan.md +0 -1642
- package/scripts/build.ts +0 -159
- package/scripts/bump-patch-version.sh +0 -18
- package/scripts/delete-unknown-projects.js +0 -154
- package/scripts/fix-sync-gap.js +0 -32
- package/scripts/generate-session-qrels.ts +0 -126
- package/scripts/heartbeat-memory-orchestrator.sh +0 -28
- package/scripts/replay-retrieval-benchmark.ts +0 -69
- package/scripts/report-sync-gap.js +0 -26
- package/scripts/review-queue-auto-resolve.js +0 -21
- package/scripts/sync-gap-auto-heal.sh +0 -17
- package/spec.md +0 -624
- package/specs/20260207-dashboard-upgrade/context.md +0 -38
- package/specs/20260207-dashboard-upgrade/spec.md +0 -96
- package/specs/citations-system/context.md +0 -243
- package/specs/citations-system/plan.md +0 -495
- package/specs/citations-system/spec.md +0 -371
- package/specs/endless-mode/context.md +0 -305
- package/specs/endless-mode/plan.md +0 -620
- package/specs/endless-mode/spec.md +0 -455
- package/specs/entity-edge-model/context.md +0 -401
- package/specs/entity-edge-model/plan.md +0 -459
- package/specs/entity-edge-model/spec.md +0 -391
- package/specs/evidence-aligner-v2/context.md +0 -401
- package/specs/evidence-aligner-v2/plan.md +0 -303
- package/specs/evidence-aligner-v2/spec.md +0 -312
- package/specs/mcp-desktop-integration/context.md +0 -278
- package/specs/mcp-desktop-integration/plan.md +0 -550
- package/specs/mcp-desktop-integration/spec.md +0 -494
- package/specs/memory-utilization-improvements/context.md +0 -145
- package/specs/memory-utilization-improvements/plan.md +0 -361
- package/specs/memory-utilization-improvements/spec.md +0 -361
- package/specs/post-tool-use-hook/context.md +0 -319
- package/specs/post-tool-use-hook/plan.md +0 -469
- package/specs/post-tool-use-hook/spec.md +0 -364
- package/specs/private-tags/context.md +0 -288
- package/specs/private-tags/plan.md +0 -412
- package/specs/private-tags/spec.md +0 -345
- package/specs/progressive-disclosure/context.md +0 -346
- package/specs/progressive-disclosure/plan.md +0 -663
- package/specs/progressive-disclosure/spec.md +0 -415
- package/specs/selective-tool-observation/context.md +0 -100
- package/specs/selective-tool-observation/plan.md +0 -158
- package/specs/selective-tool-observation/spec.md +0 -127
- package/specs/task-entity-system/context.md +0 -297
- package/specs/task-entity-system/plan.md +0 -301
- package/specs/task-entity-system/spec.md +0 -314
- package/specs/thin-core-refactor/context.md +0 -275
- package/specs/thin-core-refactor/plan.md +0 -536
- package/specs/thin-core-refactor/spec.md +0 -465
- package/specs/vector-outbox-v2/context.md +0 -470
- package/specs/vector-outbox-v2/plan.md +0 -562
- package/specs/vector-outbox-v2/spec.md +0 -466
- package/specs/web-viewer-ui/context.md +0 -384
- package/specs/web-viewer-ui/plan.md +0 -797
- package/specs/web-viewer-ui/spec.md +0 -516
- package/src/adapters/claude/capture/index.ts +0 -3
- package/src/adapters/claude/context/index.ts +0 -3
- package/src/adapters/claude/hooks/index.ts +0 -21
- package/src/adapters/claude/hooks/post-tool-use.ts +0 -239
- package/src/adapters/claude/hooks/prompt-injection-policy.ts +0 -104
- package/src/adapters/claude/hooks/semantic-daemon-client.ts +0 -209
- package/src/adapters/claude/hooks/semantic-daemon.ts +0 -283
- package/src/adapters/claude/hooks/session-end.ts +0 -59
- package/src/adapters/claude/hooks/session-start.ts +0 -73
- package/src/adapters/claude/hooks/stop.ts +0 -128
- package/src/adapters/claude/hooks/user-prompt-submit.ts +0 -361
- package/src/adapters/claude/index.ts +0 -4
- package/src/adapters/claude/transcript/index.ts +0 -4
- package/src/adapters/claude/transcript/transcript-reader.ts +0 -57
- package/src/adapters/claude/transcript/turn-reconstructor.ts +0 -65
- package/src/apps/cli/claude-settings-hooks.ts +0 -138
- package/src/apps/cli/codex-import-runner.ts +0 -125
- package/src/apps/cli/codex-validation-output.ts +0 -95
- package/src/apps/cli/hermes-import-runner.ts +0 -130
- package/src/apps/cli/hermes-validation-output.ts +0 -91
- package/src/apps/cli/index.ts +0 -1735
- package/src/apps/cli/mcp-install.ts +0 -106
- package/src/apps/cli/retrieval-disclosure-output.ts +0 -196
- package/src/apps/dashboard/assets/js/bootstrap.js +0 -244
- package/src/apps/dashboard/assets/js/chat.js +0 -373
- package/src/apps/dashboard/assets/js/disclosure.js +0 -232
- package/src/apps/dashboard/assets/js/modals.js +0 -298
- package/src/apps/dashboard/assets/js/overview.js +0 -655
- package/src/apps/dashboard/assets/js/state.js +0 -72
- package/src/apps/dashboard/assets/js/views.js +0 -468
- package/src/apps/dashboard/index.html +0 -543
- package/src/apps/dashboard/index.ts +0 -3
- package/src/apps/dashboard/style.css +0 -1750
- package/src/apps/index.ts +0 -5
- package/src/apps/server/api/chat.ts +0 -244
- package/src/apps/server/api/citations.ts +0 -105
- package/src/apps/server/api/events.ts +0 -137
- package/src/apps/server/api/health.ts +0 -53
- package/src/apps/server/api/index.ts +0 -26
- package/src/apps/server/api/projects.ts +0 -74
- package/src/apps/server/api/search.ts +0 -184
- package/src/apps/server/api/sessions.ts +0 -115
- package/src/apps/server/api/stats.ts +0 -723
- package/src/apps/server/api/turns.ts +0 -143
- package/src/apps/server/api/utils.ts +0 -65
- package/src/apps/server/index.ts +0 -111
- package/src/cli/index.ts +0 -3
- package/src/cli/retrieval-disclosure-output.ts +0 -2
- package/src/compat/index.ts +0 -5
- package/src/core/canonical-key.ts +0 -186
- package/src/core/citation-generator.ts +0 -63
- package/src/core/consolidated-store.ts +0 -356
- package/src/core/consolidation-worker.ts +0 -493
- package/src/core/context-formatter.ts +0 -276
- package/src/core/continuity-manager.ts +0 -341
- package/src/core/db-wrapper.ts +0 -64
- package/src/core/derive/fact-deriver.ts +0 -170
- package/src/core/derive/index.ts +0 -2
- package/src/core/derive/summary-deriver.ts +0 -76
- package/src/core/edge-repo.ts +0 -333
- package/src/core/embedder.ts +0 -4
- package/src/core/engine/embedding-maintenance-service.ts +0 -187
- package/src/core/engine/endless-memory-services.ts +0 -4
- package/src/core/engine/index.ts +0 -19
- package/src/core/engine/memory-engine-services.ts +0 -170
- package/src/core/engine/memory-ingest-service.ts +0 -317
- package/src/core/engine/memory-query-service.ts +0 -173
- package/src/core/engine/memory-runtime-service.ts +0 -162
- package/src/core/engine/memory-service-composition.ts +0 -231
- package/src/core/engine/retrieval-analytics-service.ts +0 -181
- package/src/core/engine/retrieval-disclosure-service.ts +0 -420
- package/src/core/engine/retrieval-orchestrator.ts +0 -377
- package/src/core/engine/retrieval-services.ts +0 -176
- package/src/core/engine/shared-memory-services.ts +0 -4
- package/src/core/entity-repo.ts +0 -349
- package/src/core/event-store.ts +0 -779
- package/src/core/evidence-aligner.ts +0 -635
- package/src/core/external-market-context.ts +0 -582
- package/src/core/graduation-worker.ts +0 -171
- package/src/core/graduation.ts +0 -377
- package/src/core/index.ts +0 -64
- package/src/core/ingest-interceptor.ts +0 -80
- package/src/core/markdown-mirror.ts +0 -70
- package/src/core/matcher.ts +0 -208
- package/src/core/md-mirror.ts +0 -92
- package/src/core/metadata-extractor.ts +0 -203
- package/src/core/model/memory-fact.ts +0 -30
- package/src/core/model/memory-rule.ts +0 -14
- package/src/core/model/memory-summary.ts +0 -21
- package/src/core/model/raw-event.ts +0 -28
- package/src/core/model/retrieval-result.ts +0 -35
- package/src/core/mongo-sync-config.ts +0 -165
- package/src/core/mongo-sync-worker.ts +0 -381
- package/src/core/privacy/filter.ts +0 -190
- package/src/core/privacy/index.ts +0 -20
- package/src/core/privacy/tag-parser.ts +0 -145
- package/src/core/product-validation-matrix.ts +0 -314
- package/src/core/progressive-retriever.ts +0 -414
- package/src/core/registry/project-path.ts +0 -54
- package/src/core/registry/session-registry.ts +0 -69
- package/src/core/replay-evaluator.ts +0 -625
- package/src/core/retrieval-benchmark.ts +0 -117
- package/src/core/retrieval-quality.ts +0 -109
- package/src/core/retriever.ts +0 -800
- package/src/core/session-qrels.ts +0 -360
- package/src/core/shared-event-store.ts +0 -114
- package/src/core/shared-promoter.ts +0 -249
- package/src/core/shared-store.ts +0 -289
- package/src/core/shared-vector-store.ts +0 -203
- package/src/core/sqlite-event-store.ts +0 -1846
- package/src/core/sqlite-wrapper.ts +0 -116
- package/src/core/sync-worker.ts +0 -228
- package/src/core/tag-taxonomy.ts +0 -51
- package/src/core/task/blocker-resolver.ts +0 -333
- package/src/core/task/index.ts +0 -9
- package/src/core/task/task-matcher.ts +0 -240
- package/src/core/task/task-projector.ts +0 -358
- package/src/core/task/task-resolver.ts +0 -421
- package/src/core/turn-state.ts +0 -207
- package/src/core/types.ts +0 -952
- package/src/core/vector-outbox.ts +0 -299
- package/src/core/vector-store.ts +0 -231
- package/src/core/vector-worker.ts +0 -521
- package/src/core/working-set-store.ts +0 -257
- package/src/extensions/endless-memory/endless-memory-services.ts +0 -350
- package/src/extensions/endless-memory/index.ts +0 -1
- package/src/extensions/index.ts +0 -5
- package/src/extensions/mcp/handlers.ts +0 -960
- package/src/extensions/mcp/index.ts +0 -48
- package/src/extensions/mcp/tools.ts +0 -252
- package/src/extensions/shared-memory/index.ts +0 -1
- package/src/extensions/shared-memory/shared-memory-services.ts +0 -211
- package/src/extensions/vector/embedder.ts +0 -233
- package/src/extensions/vector/index.ts +0 -1
- package/src/hooks/post-tool-use.ts +0 -9
- package/src/hooks/semantic-daemon-client.ts +0 -1
- package/src/hooks/semantic-daemon.ts +0 -11
- package/src/hooks/session-end.ts +0 -9
- package/src/hooks/session-start.ts +0 -9
- package/src/hooks/stop.ts +0 -9
- package/src/hooks/user-prompt-submit.ts +0 -9
- package/src/index.ts +0 -13
- package/src/mcp/handlers.ts +0 -2
- package/src/mcp/index.ts +0 -4
- package/src/mcp/tools.ts +0 -2
- package/src/server/api/chat.ts +0 -2
- package/src/server/api/citations.ts +0 -2
- package/src/server/api/events.ts +0 -2
- package/src/server/api/health.ts +0 -2
- package/src/server/api/index.ts +0 -2
- package/src/server/api/projects.ts +0 -2
- package/src/server/api/search.ts +0 -2
- package/src/server/api/sessions.ts +0 -2
- package/src/server/api/stats.ts +0 -2
- package/src/server/api/turns.ts +0 -2
- package/src/server/api/utils.ts +0 -2
- package/src/server/index.ts +0 -2
- package/src/services/bootstrap-organizer.ts +0 -463
- package/src/services/codex-session-history-importer.ts +0 -966
- package/src/services/hermes-session-history-importer.ts +0 -733
- package/src/services/memory-service-config.ts +0 -36
- package/src/services/memory-service-registry.ts +0 -150
- package/src/services/memory-service.ts +0 -688
- package/src/services/session-history-importer.ts +0 -629
- package/tests/README.md +0 -23
- package/tests/adapters/claude/claude-semantic-daemon-adapter.test.ts +0 -54
- package/tests/adapters/claude/claude-transcript-reconstructor.test.ts +0 -98
- package/tests/adapters/claude-hook-prompt-injection-policy.test.ts +0 -99
- package/tests/apps/app-layer-boundary.test.ts +0 -48
- package/tests/apps/claude-settings-hooks.test.ts +0 -107
- package/tests/apps/cli-disclosure-output.test.ts +0 -212
- package/tests/apps/codex-import-runner.test.ts +0 -99
- package/tests/apps/codex-validation-output.test.ts +0 -100
- package/tests/apps/hermes-import-runner.test.ts +0 -99
- package/tests/apps/mcp-install-command.test.ts +0 -59
- package/tests/apps/package-build-entrypoints.test.ts +0 -30
- package/tests/apps/postinstall-embedding-backend.test.ts +0 -175
- package/tests/apps/search-api-disclosure.test.ts +0 -162
- package/tests/apps/stats-api-lightweight.test.ts +0 -67
- package/tests/apps/ui-disclosure-output.test.ts +0 -140
- package/tests/core/bootstrap-organizer.test.ts +0 -111
- package/tests/core/canonical-key.test.ts +0 -101
- package/tests/core/codex-session-history-importer-validation.test.ts +0 -185
- package/tests/core/consolidation-worker.test.ts +0 -75
- package/tests/core/embedding-maintenance-service.test.ts +0 -282
- package/tests/core/evidence-aligner.test.ts +0 -152
- package/tests/core/external-market-context.test.ts +0 -209
- package/tests/core/fact-deriver.test.ts +0 -79
- package/tests/core/hermes-session-history-importer-validation.test.ts +0 -609
- package/tests/core/ingest-interceptor.test.ts +0 -38
- package/tests/core/markdown-mirror.test.ts +0 -85
- package/tests/core/matcher.test.ts +0 -112
- package/tests/core/md-mirror.test.ts +0 -50
- package/tests/core/memory-engine-services.test.ts +0 -240
- package/tests/core/memory-ingest-service.test.ts +0 -296
- package/tests/core/memory-query-service.test.ts +0 -129
- package/tests/core/memory-runtime-service.test.ts +0 -201
- package/tests/core/memory-service-composition.test.ts +0 -192
- package/tests/core/memory-service-config.test.ts +0 -41
- package/tests/core/memory-service-facade.test.ts +0 -30
- package/tests/core/memory-service-registry.test.ts +0 -206
- package/tests/core/product-validation-matrix.test.ts +0 -61
- package/tests/core/project-registry.test.ts +0 -78
- package/tests/core/replay-evaluator.test.ts +0 -181
- package/tests/core/retrieval-analytics-service.test.ts +0 -210
- package/tests/core/retrieval-benchmark.test.ts +0 -93
- package/tests/core/retrieval-disclosure-service.test.ts +0 -264
- package/tests/core/retrieval-orchestrator.test.ts +0 -403
- package/tests/core/retrieval-quality.test.ts +0 -31
- package/tests/core/retrieval-services.test.ts +0 -185
- package/tests/core/retriever-fallback-chain.test.ts +0 -223
- package/tests/core/retriever-strategy-scope.test.ts +0 -164
- package/tests/core/retriever.memu-adoption.test.ts +0 -122
- package/tests/core/session-history-importer-filter.test.ts +0 -78
- package/tests/core/session-qrels.test.ts +0 -250
- package/tests/core/sqlite-event-store-replication.test.ts +0 -127
- package/tests/core/summary-deriver.test.ts +0 -66
- package/tests/extensions/embedder-warning-suppression.test.ts +0 -83
- package/tests/extensions/endless-memory-extension-boundary.test.ts +0 -17
- package/tests/extensions/endless-memory-services.test.ts +0 -325
- package/tests/extensions/mcp-context-tools.test.ts +0 -905
- package/tests/extensions/mcp-extension-boundary.test.ts +0 -21
- package/tests/extensions/mcp-package-build.test.ts +0 -22
- package/tests/extensions/mcp-project-aware-tools.test.ts +0 -102
- package/tests/extensions/shared-memory-extension-boundary.test.ts +0 -24
- package/tests/extensions/shared-memory-services.test.ts +0 -309
- package/tests/extensions/vector-extension-boundary.test.ts +0 -21
- package/tsconfig.json +0 -24
- package/vitest.config.ts +0 -15
|
@@ -1,1846 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* SQLite-based EventStore implementation
|
|
3
|
-
* Primary store for hooks - WAL mode enables concurrent access
|
|
4
|
-
*/
|
|
5
|
-
|
|
6
|
-
import { randomUUID } from 'crypto';
|
|
7
|
-
import {
|
|
8
|
-
MemoryEvent,
|
|
9
|
-
MemoryEventInput,
|
|
10
|
-
Session,
|
|
11
|
-
AppendResult,
|
|
12
|
-
OutboxItem
|
|
13
|
-
} from './types.js';
|
|
14
|
-
import { makeCanonicalKey, makeDedupeKey } from './canonical-key.js';
|
|
15
|
-
import {
|
|
16
|
-
createSQLiteDatabase,
|
|
17
|
-
sqliteRun,
|
|
18
|
-
sqliteAll,
|
|
19
|
-
sqliteGet,
|
|
20
|
-
sqliteClose,
|
|
21
|
-
sqliteExec,
|
|
22
|
-
toDateFromSQLite,
|
|
23
|
-
toSQLiteTimestamp,
|
|
24
|
-
type SQLiteDatabase,
|
|
25
|
-
type SQLiteOptions
|
|
26
|
-
} from './sqlite-wrapper.js';
|
|
27
|
-
import { MarkdownMirror } from './markdown-mirror.js';
|
|
28
|
-
|
|
29
|
-
export interface SQLiteEventStoreOptions extends SQLiteOptions {
|
|
30
|
-
markdownMirrorRoot?: string;
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
export class SQLiteEventStore {
|
|
34
|
-
private db: SQLiteDatabase;
|
|
35
|
-
private initialized = false;
|
|
36
|
-
private readonly readOnly: boolean;
|
|
37
|
-
private readonly markdownMirror: MarkdownMirror | null;
|
|
38
|
-
|
|
39
|
-
constructor(dbPath: string, options?: SQLiteEventStoreOptions) {
|
|
40
|
-
this.readOnly = options?.readonly ?? false;
|
|
41
|
-
this.db = createSQLiteDatabase(dbPath, {
|
|
42
|
-
readonly: this.readOnly,
|
|
43
|
-
walMode: !this.readOnly
|
|
44
|
-
});
|
|
45
|
-
this.markdownMirror = this.readOnly || !options?.markdownMirrorRoot
|
|
46
|
-
? null
|
|
47
|
-
: new MarkdownMirror(options.markdownMirrorRoot);
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
/**
|
|
51
|
-
* Initialize database schema
|
|
52
|
-
*/
|
|
53
|
-
async initialize(): Promise<void> {
|
|
54
|
-
if (this.initialized) return;
|
|
55
|
-
|
|
56
|
-
// In read-only mode, skip schema creation
|
|
57
|
-
if (this.readOnly) {
|
|
58
|
-
this.initialized = true;
|
|
59
|
-
return;
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
// Create all tables in a single exec for efficiency
|
|
63
|
-
sqliteExec(this.db, `
|
|
64
|
-
-- L0 EventStore: Single Source of Truth (immutable, append-only)
|
|
65
|
-
CREATE TABLE IF NOT EXISTS events (
|
|
66
|
-
id TEXT PRIMARY KEY,
|
|
67
|
-
event_type TEXT NOT NULL,
|
|
68
|
-
session_id TEXT NOT NULL,
|
|
69
|
-
timestamp TEXT NOT NULL,
|
|
70
|
-
content TEXT NOT NULL,
|
|
71
|
-
canonical_key TEXT NOT NULL,
|
|
72
|
-
dedupe_key TEXT UNIQUE,
|
|
73
|
-
metadata TEXT,
|
|
74
|
-
access_count INTEGER DEFAULT 0,
|
|
75
|
-
last_accessed_at TEXT
|
|
76
|
-
);
|
|
77
|
-
|
|
78
|
-
-- Dedup table for idempotency
|
|
79
|
-
CREATE TABLE IF NOT EXISTS event_dedup (
|
|
80
|
-
dedupe_key TEXT PRIMARY KEY,
|
|
81
|
-
event_id TEXT NOT NULL,
|
|
82
|
-
created_at TEXT DEFAULT (datetime('now'))
|
|
83
|
-
);
|
|
84
|
-
|
|
85
|
-
-- Session metadata
|
|
86
|
-
CREATE TABLE IF NOT EXISTS sessions (
|
|
87
|
-
id TEXT PRIMARY KEY,
|
|
88
|
-
started_at TEXT NOT NULL,
|
|
89
|
-
ended_at TEXT,
|
|
90
|
-
project_path TEXT,
|
|
91
|
-
summary TEXT,
|
|
92
|
-
tags TEXT
|
|
93
|
-
);
|
|
94
|
-
|
|
95
|
-
-- Insights (derived data, rebuildable)
|
|
96
|
-
CREATE TABLE IF NOT EXISTS insights (
|
|
97
|
-
id TEXT PRIMARY KEY,
|
|
98
|
-
insight_type TEXT NOT NULL,
|
|
99
|
-
content TEXT NOT NULL,
|
|
100
|
-
canonical_key TEXT NOT NULL,
|
|
101
|
-
confidence REAL,
|
|
102
|
-
source_events TEXT,
|
|
103
|
-
created_at TEXT,
|
|
104
|
-
last_updated TEXT
|
|
105
|
-
);
|
|
106
|
-
|
|
107
|
-
-- Embedding Outbox (Single-Writer Pattern)
|
|
108
|
-
CREATE TABLE IF NOT EXISTS embedding_outbox (
|
|
109
|
-
id TEXT PRIMARY KEY,
|
|
110
|
-
event_id TEXT NOT NULL,
|
|
111
|
-
content TEXT NOT NULL,
|
|
112
|
-
status TEXT DEFAULT 'pending',
|
|
113
|
-
retry_count INTEGER DEFAULT 0,
|
|
114
|
-
created_at TEXT DEFAULT (datetime('now')),
|
|
115
|
-
processed_at TEXT,
|
|
116
|
-
error_message TEXT
|
|
117
|
-
);
|
|
118
|
-
|
|
119
|
-
-- Projection offset tracking
|
|
120
|
-
CREATE TABLE IF NOT EXISTS projection_offsets (
|
|
121
|
-
projection_name TEXT PRIMARY KEY,
|
|
122
|
-
last_event_id TEXT,
|
|
123
|
-
last_timestamp TEXT,
|
|
124
|
-
updated_at TEXT DEFAULT (datetime('now'))
|
|
125
|
-
);
|
|
126
|
-
|
|
127
|
-
-- Memory level tracking
|
|
128
|
-
CREATE TABLE IF NOT EXISTS memory_levels (
|
|
129
|
-
event_id TEXT PRIMARY KEY,
|
|
130
|
-
level TEXT NOT NULL DEFAULT 'L0',
|
|
131
|
-
promoted_at TEXT DEFAULT (datetime('now'))
|
|
132
|
-
);
|
|
133
|
-
|
|
134
|
-
-- Entries (immutable memory units)
|
|
135
|
-
CREATE TABLE IF NOT EXISTS entries (
|
|
136
|
-
entry_id TEXT PRIMARY KEY,
|
|
137
|
-
created_ts TEXT NOT NULL,
|
|
138
|
-
entry_type TEXT NOT NULL,
|
|
139
|
-
title TEXT NOT NULL,
|
|
140
|
-
content_json TEXT NOT NULL,
|
|
141
|
-
stage TEXT NOT NULL DEFAULT 'raw',
|
|
142
|
-
status TEXT DEFAULT 'active',
|
|
143
|
-
superseded_by TEXT,
|
|
144
|
-
build_id TEXT,
|
|
145
|
-
evidence_json TEXT,
|
|
146
|
-
canonical_key TEXT,
|
|
147
|
-
created_at TEXT DEFAULT (datetime('now'))
|
|
148
|
-
);
|
|
149
|
-
|
|
150
|
-
-- Entities (task/condition/artifact)
|
|
151
|
-
CREATE TABLE IF NOT EXISTS entities (
|
|
152
|
-
entity_id TEXT PRIMARY KEY,
|
|
153
|
-
entity_type TEXT NOT NULL,
|
|
154
|
-
canonical_key TEXT NOT NULL,
|
|
155
|
-
title TEXT NOT NULL,
|
|
156
|
-
stage TEXT NOT NULL DEFAULT 'raw',
|
|
157
|
-
status TEXT NOT NULL DEFAULT 'active',
|
|
158
|
-
current_json TEXT NOT NULL,
|
|
159
|
-
title_norm TEXT,
|
|
160
|
-
search_text TEXT,
|
|
161
|
-
created_at TEXT DEFAULT (datetime('now')),
|
|
162
|
-
updated_at TEXT DEFAULT (datetime('now'))
|
|
163
|
-
);
|
|
164
|
-
|
|
165
|
-
-- Entity aliases for canonical key lookup
|
|
166
|
-
CREATE TABLE IF NOT EXISTS entity_aliases (
|
|
167
|
-
entity_type TEXT NOT NULL,
|
|
168
|
-
canonical_key TEXT NOT NULL,
|
|
169
|
-
entity_id TEXT NOT NULL,
|
|
170
|
-
is_primary INTEGER DEFAULT 0,
|
|
171
|
-
created_at TEXT DEFAULT (datetime('now')),
|
|
172
|
-
PRIMARY KEY(entity_type, canonical_key)
|
|
173
|
-
);
|
|
174
|
-
|
|
175
|
-
-- Edges (relationships between entries/entities)
|
|
176
|
-
CREATE TABLE IF NOT EXISTS edges (
|
|
177
|
-
edge_id TEXT PRIMARY KEY,
|
|
178
|
-
src_type TEXT NOT NULL,
|
|
179
|
-
src_id TEXT NOT NULL,
|
|
180
|
-
rel_type TEXT NOT NULL,
|
|
181
|
-
dst_type TEXT NOT NULL,
|
|
182
|
-
dst_id TEXT NOT NULL,
|
|
183
|
-
meta_json TEXT,
|
|
184
|
-
created_at TEXT DEFAULT (datetime('now'))
|
|
185
|
-
);
|
|
186
|
-
|
|
187
|
-
-- Vector Outbox V2 Table
|
|
188
|
-
CREATE TABLE IF NOT EXISTS vector_outbox (
|
|
189
|
-
job_id TEXT PRIMARY KEY,
|
|
190
|
-
item_kind TEXT NOT NULL,
|
|
191
|
-
item_id TEXT NOT NULL,
|
|
192
|
-
embedding_version TEXT NOT NULL,
|
|
193
|
-
status TEXT NOT NULL DEFAULT 'pending',
|
|
194
|
-
retry_count INTEGER DEFAULT 0,
|
|
195
|
-
error TEXT,
|
|
196
|
-
created_at TEXT DEFAULT (datetime('now')),
|
|
197
|
-
updated_at TEXT DEFAULT (datetime('now')),
|
|
198
|
-
UNIQUE(item_kind, item_id, embedding_version)
|
|
199
|
-
);
|
|
200
|
-
|
|
201
|
-
-- Build Runs
|
|
202
|
-
CREATE TABLE IF NOT EXISTS build_runs (
|
|
203
|
-
build_id TEXT PRIMARY KEY,
|
|
204
|
-
started_at TEXT NOT NULL,
|
|
205
|
-
finished_at TEXT,
|
|
206
|
-
extractor_model TEXT NOT NULL,
|
|
207
|
-
extractor_prompt_hash TEXT NOT NULL,
|
|
208
|
-
embedder_model TEXT NOT NULL,
|
|
209
|
-
embedding_version TEXT NOT NULL,
|
|
210
|
-
idris_version TEXT NOT NULL,
|
|
211
|
-
schema_version TEXT NOT NULL,
|
|
212
|
-
status TEXT NOT NULL DEFAULT 'running',
|
|
213
|
-
error TEXT
|
|
214
|
-
);
|
|
215
|
-
|
|
216
|
-
-- Pipeline Metrics
|
|
217
|
-
CREATE TABLE IF NOT EXISTS pipeline_metrics (
|
|
218
|
-
id TEXT PRIMARY KEY,
|
|
219
|
-
ts TEXT NOT NULL,
|
|
220
|
-
stage TEXT NOT NULL,
|
|
221
|
-
latency_ms REAL NOT NULL,
|
|
222
|
-
success INTEGER NOT NULL,
|
|
223
|
-
error TEXT,
|
|
224
|
-
session_id TEXT
|
|
225
|
-
);
|
|
226
|
-
|
|
227
|
-
-- Working Set table (active memory window)
|
|
228
|
-
CREATE TABLE IF NOT EXISTS working_set (
|
|
229
|
-
id TEXT PRIMARY KEY,
|
|
230
|
-
event_id TEXT NOT NULL,
|
|
231
|
-
added_at TEXT DEFAULT (datetime('now')),
|
|
232
|
-
relevance_score REAL DEFAULT 1.0,
|
|
233
|
-
topics TEXT,
|
|
234
|
-
expires_at TEXT
|
|
235
|
-
);
|
|
236
|
-
|
|
237
|
-
-- Consolidated Memories table (long-term integrated memories)
|
|
238
|
-
CREATE TABLE IF NOT EXISTS consolidated_memories (
|
|
239
|
-
memory_id TEXT PRIMARY KEY,
|
|
240
|
-
summary TEXT NOT NULL,
|
|
241
|
-
topics TEXT,
|
|
242
|
-
source_events TEXT,
|
|
243
|
-
confidence REAL DEFAULT 0.5,
|
|
244
|
-
created_at TEXT DEFAULT (datetime('now')),
|
|
245
|
-
accessed_at TEXT,
|
|
246
|
-
access_count INTEGER DEFAULT 0
|
|
247
|
-
);
|
|
248
|
-
|
|
249
|
-
-- Continuity Log table (tracks context transitions)
|
|
250
|
-
CREATE TABLE IF NOT EXISTS continuity_log (
|
|
251
|
-
log_id TEXT PRIMARY KEY,
|
|
252
|
-
from_context_id TEXT,
|
|
253
|
-
to_context_id TEXT,
|
|
254
|
-
continuity_score REAL,
|
|
255
|
-
transition_type TEXT,
|
|
256
|
-
created_at TEXT DEFAULT (datetime('now'))
|
|
257
|
-
);
|
|
258
|
-
|
|
259
|
-
-- Consolidated Rules table (long-term stable memory)
|
|
260
|
-
CREATE TABLE IF NOT EXISTS consolidated_rules (
|
|
261
|
-
rule_id TEXT PRIMARY KEY,
|
|
262
|
-
rule TEXT NOT NULL,
|
|
263
|
-
topics TEXT,
|
|
264
|
-
source_memory_ids TEXT,
|
|
265
|
-
source_events TEXT,
|
|
266
|
-
confidence REAL DEFAULT 0.5,
|
|
267
|
-
created_at TEXT DEFAULT (datetime('now'))
|
|
268
|
-
);
|
|
269
|
-
|
|
270
|
-
-- Endless Mode Config table
|
|
271
|
-
CREATE TABLE IF NOT EXISTS endless_config (
|
|
272
|
-
key TEXT PRIMARY KEY,
|
|
273
|
-
value TEXT,
|
|
274
|
-
updated_at TEXT DEFAULT (datetime('now'))
|
|
275
|
-
);
|
|
276
|
-
|
|
277
|
-
-- Memory Helpfulness tracking
|
|
278
|
-
CREATE TABLE IF NOT EXISTS memory_helpfulness (
|
|
279
|
-
id TEXT PRIMARY KEY,
|
|
280
|
-
event_id TEXT NOT NULL,
|
|
281
|
-
session_id TEXT NOT NULL,
|
|
282
|
-
retrieval_score REAL DEFAULT 0,
|
|
283
|
-
query_preview TEXT,
|
|
284
|
-
session_continued INTEGER DEFAULT 0,
|
|
285
|
-
prompt_count_after INTEGER DEFAULT 0,
|
|
286
|
-
tool_success_count INTEGER DEFAULT 0,
|
|
287
|
-
tool_total_count INTEGER DEFAULT 0,
|
|
288
|
-
was_reasked INTEGER DEFAULT 0,
|
|
289
|
-
helpfulness_score REAL DEFAULT 0.5,
|
|
290
|
-
created_at TEXT DEFAULT (datetime('now')),
|
|
291
|
-
measured_at TEXT
|
|
292
|
-
);
|
|
293
|
-
|
|
294
|
-
-- Retrieval trace log (query -> candidates -> selected for context)
|
|
295
|
-
CREATE TABLE IF NOT EXISTS retrieval_traces (
|
|
296
|
-
trace_id TEXT PRIMARY KEY,
|
|
297
|
-
session_id TEXT,
|
|
298
|
-
project_hash TEXT,
|
|
299
|
-
query_text TEXT NOT NULL,
|
|
300
|
-
strategy TEXT,
|
|
301
|
-
candidate_event_ids TEXT,
|
|
302
|
-
selected_event_ids TEXT,
|
|
303
|
-
candidate_details_json TEXT,
|
|
304
|
-
selected_details_json TEXT,
|
|
305
|
-
candidate_count INTEGER DEFAULT 0,
|
|
306
|
-
selected_count INTEGER DEFAULT 0,
|
|
307
|
-
confidence TEXT,
|
|
308
|
-
fallback_trace TEXT,
|
|
309
|
-
created_at TEXT DEFAULT (datetime('now'))
|
|
310
|
-
);
|
|
311
|
-
|
|
312
|
-
-- Sync position tracking (for SQLite -> DuckDB sync)
|
|
313
|
-
CREATE TABLE IF NOT EXISTS sync_positions (
|
|
314
|
-
target_name TEXT PRIMARY KEY,
|
|
315
|
-
last_event_id TEXT,
|
|
316
|
-
last_timestamp TEXT,
|
|
317
|
-
updated_at TEXT DEFAULT (datetime('now'))
|
|
318
|
-
);
|
|
319
|
-
|
|
320
|
-
-- Create indexes
|
|
321
|
-
CREATE INDEX IF NOT EXISTS idx_events_session ON events(session_id);
|
|
322
|
-
CREATE INDEX IF NOT EXISTS idx_events_timestamp ON events(timestamp);
|
|
323
|
-
CREATE INDEX IF NOT EXISTS idx_entries_type ON entries(entry_type);
|
|
324
|
-
CREATE INDEX IF NOT EXISTS idx_entries_stage ON entries(stage);
|
|
325
|
-
CREATE INDEX IF NOT EXISTS idx_entries_canonical ON entries(canonical_key);
|
|
326
|
-
CREATE INDEX IF NOT EXISTS idx_entities_type_key ON entities(entity_type, canonical_key);
|
|
327
|
-
CREATE INDEX IF NOT EXISTS idx_entities_status ON entities(status);
|
|
328
|
-
CREATE INDEX IF NOT EXISTS idx_edges_src ON edges(src_id, rel_type);
|
|
329
|
-
CREATE INDEX IF NOT EXISTS idx_edges_dst ON edges(dst_id, rel_type);
|
|
330
|
-
CREATE INDEX IF NOT EXISTS idx_edges_rel ON edges(rel_type);
|
|
331
|
-
CREATE INDEX IF NOT EXISTS idx_outbox_status ON vector_outbox(status);
|
|
332
|
-
CREATE INDEX IF NOT EXISTS idx_working_set_expires ON working_set(expires_at);
|
|
333
|
-
CREATE INDEX IF NOT EXISTS idx_working_set_relevance ON working_set(relevance_score);
|
|
334
|
-
CREATE INDEX IF NOT EXISTS idx_consolidated_confidence ON consolidated_memories(confidence);
|
|
335
|
-
CREATE INDEX IF NOT EXISTS idx_continuity_created ON continuity_log(created_at);
|
|
336
|
-
CREATE INDEX IF NOT EXISTS idx_consolidated_rules_confidence ON consolidated_rules(confidence);
|
|
337
|
-
CREATE INDEX IF NOT EXISTS idx_embedding_outbox_status ON embedding_outbox(status);
|
|
338
|
-
CREATE INDEX IF NOT EXISTS idx_helpfulness_event ON memory_helpfulness(event_id);
|
|
339
|
-
CREATE INDEX IF NOT EXISTS idx_helpfulness_session ON memory_helpfulness(session_id);
|
|
340
|
-
CREATE INDEX IF NOT EXISTS idx_helpfulness_score ON memory_helpfulness(helpfulness_score DESC);
|
|
341
|
-
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_created_at ON retrieval_traces(created_at DESC);
|
|
342
|
-
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_project_hash ON retrieval_traces(project_hash);
|
|
343
|
-
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_session_id ON retrieval_traces(session_id);
|
|
344
|
-
|
|
345
|
-
-- FTS5 Full-Text Search for fast keyword search
|
|
346
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS events_fts USING fts5(
|
|
347
|
-
content,
|
|
348
|
-
event_id UNINDEXED,
|
|
349
|
-
tokenize='porter unicode61'
|
|
350
|
-
);
|
|
351
|
-
|
|
352
|
-
-- Triggers to keep FTS in sync with events table
|
|
353
|
-
CREATE TRIGGER IF NOT EXISTS events_fts_insert AFTER INSERT ON events BEGIN
|
|
354
|
-
INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
|
|
355
|
-
END;
|
|
356
|
-
|
|
357
|
-
CREATE TRIGGER IF NOT EXISTS events_fts_delete AFTER DELETE ON events BEGIN
|
|
358
|
-
DELETE FROM events_fts WHERE rowid = OLD.rowid;
|
|
359
|
-
END;
|
|
360
|
-
|
|
361
|
-
CREATE TRIGGER IF NOT EXISTS events_fts_update AFTER UPDATE ON events BEGIN
|
|
362
|
-
DELETE FROM events_fts WHERE rowid = OLD.rowid;
|
|
363
|
-
INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
|
|
364
|
-
END;
|
|
365
|
-
`);
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
// Best-effort forward migration for retrieval trace detail column
|
|
369
|
-
try {
|
|
370
|
-
sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN selected_details_json TEXT;`);
|
|
371
|
-
} catch {
|
|
372
|
-
// column may already exist
|
|
373
|
-
}
|
|
374
|
-
try {
|
|
375
|
-
sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN candidate_details_json TEXT;`);
|
|
376
|
-
} catch {
|
|
377
|
-
// column may already exist
|
|
378
|
-
}
|
|
379
|
-
|
|
380
|
-
// Migrate existing events table to add new columns if they don't exist
|
|
381
|
-
// Check if columns exist before trying to add them
|
|
382
|
-
const tableInfo = sqliteAll(this.db, "PRAGMA table_info(events)", []);
|
|
383
|
-
const columnNames = tableInfo.map((col: any) => col.name);
|
|
384
|
-
|
|
385
|
-
if (!columnNames.includes('access_count')) {
|
|
386
|
-
try {
|
|
387
|
-
sqliteExec(this.db, `
|
|
388
|
-
ALTER TABLE events ADD COLUMN access_count INTEGER DEFAULT 0;
|
|
389
|
-
`);
|
|
390
|
-
} catch (err: any) {
|
|
391
|
-
console.error('Error adding access_count column:', err);
|
|
392
|
-
}
|
|
393
|
-
}
|
|
394
|
-
|
|
395
|
-
if (!columnNames.includes('last_accessed_at')) {
|
|
396
|
-
try {
|
|
397
|
-
sqliteExec(this.db, `
|
|
398
|
-
ALTER TABLE events ADD COLUMN last_accessed_at TEXT;
|
|
399
|
-
`);
|
|
400
|
-
} catch (err: any) {
|
|
401
|
-
console.error('Error adding last_accessed_at column:', err);
|
|
402
|
-
}
|
|
403
|
-
}
|
|
404
|
-
|
|
405
|
-
// Add turn_id column for grouping events within a conversation turn
|
|
406
|
-
if (!columnNames.includes('turn_id')) {
|
|
407
|
-
try {
|
|
408
|
-
sqliteExec(this.db, `
|
|
409
|
-
ALTER TABLE events ADD COLUMN turn_id TEXT;
|
|
410
|
-
`);
|
|
411
|
-
} catch (err: any) {
|
|
412
|
-
console.error('Error adding turn_id column:', err);
|
|
413
|
-
}
|
|
414
|
-
}
|
|
415
|
-
|
|
416
|
-
// Create indexes for new columns if they don't exist
|
|
417
|
-
try {
|
|
418
|
-
sqliteExec(this.db, `
|
|
419
|
-
CREATE INDEX IF NOT EXISTS idx_events_access_count ON events(access_count DESC);
|
|
420
|
-
`);
|
|
421
|
-
} catch (err: any) {
|
|
422
|
-
// Index may already exist, ignore
|
|
423
|
-
}
|
|
424
|
-
|
|
425
|
-
try {
|
|
426
|
-
sqliteExec(this.db, `
|
|
427
|
-
CREATE INDEX IF NOT EXISTS idx_events_last_accessed ON events(last_accessed_at DESC);
|
|
428
|
-
`);
|
|
429
|
-
} catch (err: any) {
|
|
430
|
-
// Index may already exist, ignore
|
|
431
|
-
}
|
|
432
|
-
|
|
433
|
-
try {
|
|
434
|
-
sqliteExec(this.db, `
|
|
435
|
-
CREATE INDEX IF NOT EXISTS idx_events_turn_id ON events(turn_id);
|
|
436
|
-
`);
|
|
437
|
-
} catch (err: any) {
|
|
438
|
-
// Index may already exist, ignore
|
|
439
|
-
}
|
|
440
|
-
|
|
441
|
-
this.initialized = true;
|
|
442
|
-
}
|
|
443
|
-
|
|
444
|
-
/**
|
|
445
|
-
* Append event to store (Append-only, Idempotent)
|
|
446
|
-
*/
|
|
447
|
-
async append(input: MemoryEventInput): Promise<AppendResult> {
|
|
448
|
-
await this.initialize();
|
|
449
|
-
|
|
450
|
-
const canonicalKey = makeCanonicalKey(input.content);
|
|
451
|
-
const dedupeKey = makeDedupeKey(input.content, input.sessionId);
|
|
452
|
-
|
|
453
|
-
// Check for duplicate
|
|
454
|
-
const existing = sqliteGet<{ event_id: string }>(
|
|
455
|
-
this.db,
|
|
456
|
-
`SELECT event_id FROM event_dedup WHERE dedupe_key = ?`,
|
|
457
|
-
[dedupeKey]
|
|
458
|
-
);
|
|
459
|
-
|
|
460
|
-
if (existing) {
|
|
461
|
-
return {
|
|
462
|
-
success: true,
|
|
463
|
-
eventId: existing.event_id,
|
|
464
|
-
isDuplicate: true
|
|
465
|
-
};
|
|
466
|
-
}
|
|
467
|
-
|
|
468
|
-
const id = randomUUID();
|
|
469
|
-
const timestamp = toSQLiteTimestamp(input.timestamp);
|
|
470
|
-
|
|
471
|
-
try {
|
|
472
|
-
// Extract turnId from metadata if present
|
|
473
|
-
const metadata = input.metadata || {};
|
|
474
|
-
const turnId = (metadata.turnId as string) || null;
|
|
475
|
-
|
|
476
|
-
// Use transaction for atomicity
|
|
477
|
-
const insertEvent = this.db.prepare(`
|
|
478
|
-
INSERT INTO events (id, event_type, session_id, timestamp, content, canonical_key, dedupe_key, metadata, turn_id)
|
|
479
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
480
|
-
`);
|
|
481
|
-
|
|
482
|
-
const insertDedup = this.db.prepare(`
|
|
483
|
-
INSERT INTO event_dedup (dedupe_key, event_id) VALUES (?, ?)
|
|
484
|
-
`);
|
|
485
|
-
|
|
486
|
-
const insertLevel = this.db.prepare(`
|
|
487
|
-
INSERT INTO memory_levels (event_id, level) VALUES (?, 'L0')
|
|
488
|
-
`);
|
|
489
|
-
|
|
490
|
-
const transaction = this.db.transaction(() => {
|
|
491
|
-
insertEvent.run(
|
|
492
|
-
id,
|
|
493
|
-
input.eventType,
|
|
494
|
-
input.sessionId,
|
|
495
|
-
timestamp,
|
|
496
|
-
input.content,
|
|
497
|
-
canonicalKey,
|
|
498
|
-
dedupeKey,
|
|
499
|
-
JSON.stringify(metadata),
|
|
500
|
-
turnId
|
|
501
|
-
);
|
|
502
|
-
insertDedup.run(dedupeKey, id);
|
|
503
|
-
insertLevel.run(id);
|
|
504
|
-
});
|
|
505
|
-
|
|
506
|
-
transaction();
|
|
507
|
-
|
|
508
|
-
if (this.markdownMirror) {
|
|
509
|
-
const event: MemoryEvent = {
|
|
510
|
-
id,
|
|
511
|
-
eventType: input.eventType,
|
|
512
|
-
sessionId: input.sessionId,
|
|
513
|
-
timestamp: input.timestamp,
|
|
514
|
-
content: input.content,
|
|
515
|
-
canonicalKey,
|
|
516
|
-
dedupeKey,
|
|
517
|
-
metadata
|
|
518
|
-
};
|
|
519
|
-
this.markdownMirror.append(event).catch((err) => {
|
|
520
|
-
console.warn('[SQLiteEventStore] markdown mirror append failed:', err);
|
|
521
|
-
});
|
|
522
|
-
}
|
|
523
|
-
|
|
524
|
-
return { success: true, eventId: id, isDuplicate: false };
|
|
525
|
-
} catch (error) {
|
|
526
|
-
return {
|
|
527
|
-
success: false,
|
|
528
|
-
error: error instanceof Error ? error.message : String(error)
|
|
529
|
-
};
|
|
530
|
-
}
|
|
531
|
-
}
|
|
532
|
-
|
|
533
|
-
/**
|
|
534
|
-
* Get session IDs that have events but no session_summary event.
|
|
535
|
-
* Used to backfill summaries for sessions that ended without Stop hook.
|
|
536
|
-
*/
|
|
537
|
-
async getSessionsWithoutSummary(currentSessionId: string, limit = 5): Promise<string[]> {
|
|
538
|
-
await this.initialize();
|
|
539
|
-
const rows = sqliteAll<{ session_id: string }>(
|
|
540
|
-
this.db,
|
|
541
|
-
`SELECT DISTINCT e.session_id
|
|
542
|
-
FROM events e
|
|
543
|
-
WHERE e.session_id != ?
|
|
544
|
-
AND e.event_type != 'session_summary'
|
|
545
|
-
AND e.session_id NOT IN (
|
|
546
|
-
SELECT DISTINCT session_id FROM events WHERE event_type = 'session_summary'
|
|
547
|
-
)
|
|
548
|
-
GROUP BY e.session_id
|
|
549
|
-
HAVING COUNT(*) >= 3
|
|
550
|
-
ORDER BY MAX(e.timestamp) DESC
|
|
551
|
-
LIMIT ?`,
|
|
552
|
-
[currentSessionId, limit]
|
|
553
|
-
);
|
|
554
|
-
return rows.map((r) => r.session_id);
|
|
555
|
-
}
|
|
556
|
-
|
|
557
|
-
/**
|
|
558
|
-
* Get events by session ID
|
|
559
|
-
*/
|
|
560
|
-
async getSessionEvents(sessionId: string): Promise<MemoryEvent[]> {
|
|
561
|
-
await this.initialize();
|
|
562
|
-
|
|
563
|
-
const rows = sqliteAll<Record<string, unknown>>(
|
|
564
|
-
this.db,
|
|
565
|
-
`SELECT * FROM events WHERE session_id = ? ORDER BY timestamp ASC`,
|
|
566
|
-
[sessionId]
|
|
567
|
-
);
|
|
568
|
-
|
|
569
|
-
return rows.map(this.rowToEvent);
|
|
570
|
-
}
|
|
571
|
-
|
|
572
|
-
/**
|
|
573
|
-
* Get recent events
|
|
574
|
-
*/
|
|
575
|
-
async getRecentEvents(limit: number = 100): Promise<MemoryEvent[]> {
|
|
576
|
-
await this.initialize();
|
|
577
|
-
|
|
578
|
-
const rows = sqliteAll<Record<string, unknown>>(
|
|
579
|
-
this.db,
|
|
580
|
-
`SELECT * FROM events ORDER BY timestamp DESC LIMIT ?`,
|
|
581
|
-
[limit]
|
|
582
|
-
);
|
|
583
|
-
|
|
584
|
-
return rows.map(this.rowToEvent);
|
|
585
|
-
}
|
|
586
|
-
|
|
587
|
-
/**
|
|
588
|
-
* Get event by ID
|
|
589
|
-
*/
|
|
590
|
-
async getEvent(id: string): Promise<MemoryEvent | null> {
|
|
591
|
-
await this.initialize();
|
|
592
|
-
|
|
593
|
-
const row = sqliteGet<Record<string, unknown>>(
|
|
594
|
-
this.db,
|
|
595
|
-
`SELECT * FROM events WHERE id = ?`,
|
|
596
|
-
[id]
|
|
597
|
-
);
|
|
598
|
-
|
|
599
|
-
if (!row) return null;
|
|
600
|
-
return this.rowToEvent(row);
|
|
601
|
-
}
|
|
602
|
-
|
|
603
|
-
/**
|
|
604
|
-
* Get events since a timestamp (for sync)
|
|
605
|
-
*/
|
|
606
|
-
async getEventsSince(timestamp: string, limit: number = 1000): Promise<MemoryEvent[]> {
|
|
607
|
-
await this.initialize();
|
|
608
|
-
|
|
609
|
-
const rows = sqliteAll<Record<string, unknown>>(
|
|
610
|
-
this.db,
|
|
611
|
-
`SELECT * FROM events WHERE timestamp > ? ORDER BY timestamp ASC LIMIT ?`,
|
|
612
|
-
[timestamp, limit]
|
|
613
|
-
);
|
|
614
|
-
|
|
615
|
-
return rows.map(this.rowToEvent);
|
|
616
|
-
}
|
|
617
|
-
|
|
618
|
-
/**
|
|
619
|
-
* Get events since a SQLite rowid (for robust incremental replication).
|
|
620
|
-
* Rowid is monotonic for append-only tables, independent of client timestamps.
|
|
621
|
-
*/
|
|
622
|
-
async getEventsSinceRowid(
|
|
623
|
-
lastRowid: number,
|
|
624
|
-
limit: number = 1000
|
|
625
|
-
): Promise<Array<{ rowid: number; event: MemoryEvent }>> {
|
|
626
|
-
await this.initialize();
|
|
627
|
-
|
|
628
|
-
const rows = sqliteAll<Record<string, unknown>>(
|
|
629
|
-
this.db,
|
|
630
|
-
`SELECT rowid as _rowid, * FROM events WHERE rowid > ? ORDER BY rowid ASC LIMIT ?`,
|
|
631
|
-
[lastRowid, limit]
|
|
632
|
-
);
|
|
633
|
-
|
|
634
|
-
return rows.map(row => ({
|
|
635
|
-
rowid: row._rowid as number,
|
|
636
|
-
event: this.rowToEvent(row)
|
|
637
|
-
}));
|
|
638
|
-
}
|
|
639
|
-
|
|
640
|
-
/**
|
|
641
|
-
* Import events with fixed IDs (used for cross-machine replication).
|
|
642
|
-
* Idempotent: skips if event id or dedupeKey already exists.
|
|
643
|
-
*
|
|
644
|
-
* NOTE: This bypasses the append() id generation to preserve stable IDs.
|
|
645
|
-
*/
|
|
646
|
-
async importEvents(events: MemoryEvent[]): Promise<{ inserted: number; skipped: number }> {
|
|
647
|
-
if (events.length === 0) return { inserted: 0, skipped: 0 };
|
|
648
|
-
if (this.readOnly) return { inserted: 0, skipped: events.length };
|
|
649
|
-
|
|
650
|
-
await this.initialize();
|
|
651
|
-
|
|
652
|
-
const getById = this.db.prepare(`SELECT id FROM events WHERE id = ?`);
|
|
653
|
-
const getByDedupe = this.db.prepare(`SELECT event_id FROM event_dedup WHERE dedupe_key = ?`);
|
|
654
|
-
|
|
655
|
-
const insertEvent = this.db.prepare(`
|
|
656
|
-
INSERT INTO events (id, event_type, session_id, timestamp, content, canonical_key, dedupe_key, metadata, turn_id)
|
|
657
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
658
|
-
`);
|
|
659
|
-
|
|
660
|
-
const insertDedup = this.db.prepare(`
|
|
661
|
-
INSERT INTO event_dedup (dedupe_key, event_id) VALUES (?, ?)
|
|
662
|
-
`);
|
|
663
|
-
|
|
664
|
-
const insertLevel = this.db.prepare(`
|
|
665
|
-
INSERT INTO memory_levels (event_id, level) VALUES (?, 'L0')
|
|
666
|
-
`);
|
|
667
|
-
|
|
668
|
-
let inserted = 0;
|
|
669
|
-
let skipped = 0;
|
|
670
|
-
const insertedEvents: MemoryEvent[] = [];
|
|
671
|
-
|
|
672
|
-
const tx = this.db.transaction((batch: MemoryEvent[]) => {
|
|
673
|
-
for (const ev of batch) {
|
|
674
|
-
// Skip if already present by id
|
|
675
|
-
const existingById = getById.get(ev.id) as { id: string } | undefined;
|
|
676
|
-
if (existingById) {
|
|
677
|
-
skipped++;
|
|
678
|
-
continue;
|
|
679
|
-
}
|
|
680
|
-
|
|
681
|
-
const canonicalKey = ev.canonicalKey || makeCanonicalKey(ev.content);
|
|
682
|
-
const dedupeKey = ev.dedupeKey || makeDedupeKey(ev.content, ev.sessionId);
|
|
683
|
-
|
|
684
|
-
// Skip if already present by dedupe key
|
|
685
|
-
const existingByDedupe = getByDedupe.get(dedupeKey) as { event_id: string } | undefined;
|
|
686
|
-
if (existingByDedupe) {
|
|
687
|
-
skipped++;
|
|
688
|
-
continue;
|
|
689
|
-
}
|
|
690
|
-
|
|
691
|
-
const metadata = ev.metadata || {};
|
|
692
|
-
const turnId = (metadata as any).turnId as string | undefined;
|
|
693
|
-
|
|
694
|
-
insertEvent.run(
|
|
695
|
-
ev.id,
|
|
696
|
-
ev.eventType,
|
|
697
|
-
ev.sessionId,
|
|
698
|
-
toSQLiteTimestamp(ev.timestamp),
|
|
699
|
-
ev.content,
|
|
700
|
-
canonicalKey,
|
|
701
|
-
dedupeKey,
|
|
702
|
-
JSON.stringify(metadata),
|
|
703
|
-
turnId ?? null
|
|
704
|
-
);
|
|
705
|
-
|
|
706
|
-
insertDedup.run(dedupeKey, ev.id);
|
|
707
|
-
insertLevel.run(ev.id);
|
|
708
|
-
inserted++;
|
|
709
|
-
insertedEvents.push(ev);
|
|
710
|
-
}
|
|
711
|
-
});
|
|
712
|
-
|
|
713
|
-
tx(events);
|
|
714
|
-
|
|
715
|
-
if (this.markdownMirror && insertedEvents.length > 0) {
|
|
716
|
-
for (const ev of insertedEvents) {
|
|
717
|
-
this.markdownMirror.append(ev).catch((err) => {
|
|
718
|
-
console.warn('[SQLiteEventStore] markdown mirror append failed:', err);
|
|
719
|
-
});
|
|
720
|
-
}
|
|
721
|
-
}
|
|
722
|
-
|
|
723
|
-
return { inserted, skipped };
|
|
724
|
-
}
|
|
725
|
-
|
|
726
|
-
/**
|
|
727
|
-
* Create or update session
|
|
728
|
-
*/
|
|
729
|
-
async upsertSession(session: Partial<Session> & { id: string }): Promise<void> {
|
|
730
|
-
await this.initialize();
|
|
731
|
-
|
|
732
|
-
const existing = sqliteGet<{ id: string }>(
|
|
733
|
-
this.db,
|
|
734
|
-
`SELECT id FROM sessions WHERE id = ?`,
|
|
735
|
-
[session.id]
|
|
736
|
-
);
|
|
737
|
-
|
|
738
|
-
if (!existing) {
|
|
739
|
-
sqliteRun(
|
|
740
|
-
this.db,
|
|
741
|
-
`INSERT INTO sessions (id, started_at, project_path, tags)
|
|
742
|
-
VALUES (?, ?, ?, ?)`,
|
|
743
|
-
[
|
|
744
|
-
session.id,
|
|
745
|
-
toSQLiteTimestamp(session.startedAt || new Date()),
|
|
746
|
-
session.projectPath || null,
|
|
747
|
-
JSON.stringify(session.tags || [])
|
|
748
|
-
]
|
|
749
|
-
);
|
|
750
|
-
} else {
|
|
751
|
-
const updates: string[] = [];
|
|
752
|
-
const values: unknown[] = [];
|
|
753
|
-
|
|
754
|
-
if (session.endedAt) {
|
|
755
|
-
updates.push('ended_at = ?');
|
|
756
|
-
values.push(toSQLiteTimestamp(session.endedAt));
|
|
757
|
-
}
|
|
758
|
-
if (session.summary) {
|
|
759
|
-
updates.push('summary = ?');
|
|
760
|
-
values.push(session.summary);
|
|
761
|
-
}
|
|
762
|
-
if (session.tags) {
|
|
763
|
-
updates.push('tags = ?');
|
|
764
|
-
values.push(JSON.stringify(session.tags));
|
|
765
|
-
}
|
|
766
|
-
|
|
767
|
-
if (updates.length > 0) {
|
|
768
|
-
values.push(session.id);
|
|
769
|
-
sqliteRun(
|
|
770
|
-
this.db,
|
|
771
|
-
`UPDATE sessions SET ${updates.join(', ')} WHERE id = ?`,
|
|
772
|
-
values
|
|
773
|
-
);
|
|
774
|
-
}
|
|
775
|
-
}
|
|
776
|
-
}
|
|
777
|
-
|
|
778
|
-
/**
|
|
779
|
-
* Get session by ID
|
|
780
|
-
*/
|
|
781
|
-
async getSession(id: string): Promise<Session | null> {
|
|
782
|
-
await this.initialize();
|
|
783
|
-
|
|
784
|
-
const row = sqliteGet<Record<string, unknown>>(
|
|
785
|
-
this.db,
|
|
786
|
-
`SELECT * FROM sessions WHERE id = ?`,
|
|
787
|
-
[id]
|
|
788
|
-
);
|
|
789
|
-
|
|
790
|
-
if (!row) return null;
|
|
791
|
-
|
|
792
|
-
return {
|
|
793
|
-
id: row.id as string,
|
|
794
|
-
startedAt: toDateFromSQLite(row.started_at),
|
|
795
|
-
endedAt: row.ended_at ? toDateFromSQLite(row.ended_at) : undefined,
|
|
796
|
-
projectPath: row.project_path as string | undefined,
|
|
797
|
-
summary: row.summary as string | undefined,
|
|
798
|
-
tags: row.tags ? JSON.parse(row.tags as string) : undefined
|
|
799
|
-
};
|
|
800
|
-
}
|
|
801
|
-
|
|
802
|
-
/**
|
|
803
|
-
* Get all sessions
|
|
804
|
-
*/
|
|
805
|
-
async getAllSessions(): Promise<Session[]> {
|
|
806
|
-
await this.initialize();
|
|
807
|
-
|
|
808
|
-
const rows = sqliteAll<Record<string, unknown>>(
|
|
809
|
-
this.db,
|
|
810
|
-
`SELECT * FROM sessions ORDER BY started_at DESC`
|
|
811
|
-
);
|
|
812
|
-
|
|
813
|
-
return rows.map(row => ({
|
|
814
|
-
id: row.id as string,
|
|
815
|
-
startedAt: toDateFromSQLite(row.started_at),
|
|
816
|
-
endedAt: row.ended_at ? toDateFromSQLite(row.ended_at) : undefined,
|
|
817
|
-
projectPath: row.project_path as string | undefined,
|
|
818
|
-
summary: row.summary as string | undefined,
|
|
819
|
-
tags: row.tags ? JSON.parse(row.tags as string) : undefined
|
|
820
|
-
}));
|
|
821
|
-
}
|
|
822
|
-
|
|
823
|
-
/**
|
|
824
|
-
* Add to embedding outbox
|
|
825
|
-
*/
|
|
826
|
-
async enqueueForEmbedding(eventId: string, content: string): Promise<string> {
|
|
827
|
-
await this.initialize();
|
|
828
|
-
|
|
829
|
-
const id = randomUUID();
|
|
830
|
-
sqliteRun(
|
|
831
|
-
this.db,
|
|
832
|
-
`INSERT INTO embedding_outbox (id, event_id, content, status, retry_count)
|
|
833
|
-
VALUES (?, ?, ?, 'pending', 0)`,
|
|
834
|
-
[id, eventId, content]
|
|
835
|
-
);
|
|
836
|
-
|
|
837
|
-
return id;
|
|
838
|
-
}
|
|
839
|
-
|
|
840
|
-
/**
|
|
841
|
-
* Get pending outbox items
|
|
842
|
-
*/
|
|
843
|
-
async getPendingOutboxItems(limit: number = 32): Promise<OutboxItem[]> {
|
|
844
|
-
await this.initialize();
|
|
845
|
-
|
|
846
|
-
const pending = sqliteAll<Record<string, unknown>>(
|
|
847
|
-
this.db,
|
|
848
|
-
`SELECT * FROM embedding_outbox
|
|
849
|
-
WHERE status = 'pending'
|
|
850
|
-
ORDER BY created_at
|
|
851
|
-
LIMIT ?`,
|
|
852
|
-
[limit]
|
|
853
|
-
);
|
|
854
|
-
|
|
855
|
-
if (pending.length === 0) return [];
|
|
856
|
-
|
|
857
|
-
// Update status to processing
|
|
858
|
-
const ids = pending.map(r => r.id as string);
|
|
859
|
-
const placeholders = ids.map(() => '?').join(',');
|
|
860
|
-
sqliteRun(
|
|
861
|
-
this.db,
|
|
862
|
-
`UPDATE embedding_outbox SET status = 'processing' WHERE id IN (${placeholders})`,
|
|
863
|
-
ids
|
|
864
|
-
);
|
|
865
|
-
|
|
866
|
-
return pending.map(row => ({
|
|
867
|
-
id: row.id as string,
|
|
868
|
-
eventId: row.event_id as string,
|
|
869
|
-
content: row.content as string,
|
|
870
|
-
status: 'processing' as const,
|
|
871
|
-
retryCount: row.retry_count as number,
|
|
872
|
-
createdAt: toDateFromSQLite(row.created_at),
|
|
873
|
-
errorMessage: row.error_message as string | undefined
|
|
874
|
-
}));
|
|
875
|
-
}
|
|
876
|
-
|
|
877
|
-
/**
|
|
878
|
-
* Mark outbox items as done
|
|
879
|
-
*/
|
|
880
|
-
async completeOutboxItems(ids: string[]): Promise<void> {
|
|
881
|
-
if (ids.length === 0) return;
|
|
882
|
-
|
|
883
|
-
const placeholders = ids.map(() => '?').join(',');
|
|
884
|
-
sqliteRun(
|
|
885
|
-
this.db,
|
|
886
|
-
`DELETE FROM embedding_outbox WHERE id IN (${placeholders})`,
|
|
887
|
-
ids
|
|
888
|
-
);
|
|
889
|
-
}
|
|
890
|
-
|
|
891
|
-
/**
|
|
892
|
-
* Clear embedding outbox (used for embedding model migration)
|
|
893
|
-
*/
|
|
894
|
-
async clearEmbeddingOutbox(): Promise<void> {
|
|
895
|
-
await this.initialize();
|
|
896
|
-
sqliteRun(this.db, `DELETE FROM embedding_outbox`);
|
|
897
|
-
}
|
|
898
|
-
|
|
899
|
-
/**
|
|
900
|
-
* Count total events
|
|
901
|
-
*/
|
|
902
|
-
async countEvents(): Promise<number> {
|
|
903
|
-
await this.initialize();
|
|
904
|
-
const row = sqliteGet<{ count: number }>(this.db, `SELECT COUNT(*) as count FROM events`);
|
|
905
|
-
return row?.count || 0;
|
|
906
|
-
}
|
|
907
|
-
|
|
908
|
-
/**
|
|
909
|
-
* Get events page in timestamp ascending order (stable migration/reindex scans)
|
|
910
|
-
*/
|
|
911
|
-
async getEventsPage(limit: number = 1000, offset: number = 0): Promise<MemoryEvent[]> {
|
|
912
|
-
await this.initialize();
|
|
913
|
-
|
|
914
|
-
const rows = sqliteAll<Record<string, unknown>>(
|
|
915
|
-
this.db,
|
|
916
|
-
`SELECT * FROM events ORDER BY timestamp ASC LIMIT ? OFFSET ?`,
|
|
917
|
-
[limit, offset]
|
|
918
|
-
);
|
|
919
|
-
|
|
920
|
-
return rows.map(this.rowToEvent);
|
|
921
|
-
}
|
|
922
|
-
|
|
923
|
-
/**
|
|
924
|
-
* Mark outbox items as failed
|
|
925
|
-
*/
|
|
926
|
-
async failOutboxItems(ids: string[], error: string): Promise<void> {
|
|
927
|
-
if (ids.length === 0) return;
|
|
928
|
-
|
|
929
|
-
const placeholders = ids.map(() => '?').join(',');
|
|
930
|
-
sqliteRun(
|
|
931
|
-
this.db,
|
|
932
|
-
`UPDATE embedding_outbox
|
|
933
|
-
SET status = CASE WHEN retry_count >= 3 THEN 'failed' ELSE 'pending' END,
|
|
934
|
-
retry_count = retry_count + 1,
|
|
935
|
-
error_message = ?
|
|
936
|
-
WHERE id IN (${placeholders})`,
|
|
937
|
-
[error, ...ids]
|
|
938
|
-
);
|
|
939
|
-
}
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
/**
|
|
943
|
-
* Get embedding/vector outbox health statistics
|
|
944
|
-
*/
|
|
945
|
-
async getOutboxStats(): Promise<{
|
|
946
|
-
embedding: { pending: number; processing: number; failed: number; total: number };
|
|
947
|
-
vector: { pending: number; processing: number; failed: number; total: number };
|
|
948
|
-
}> {
|
|
949
|
-
await this.initialize();
|
|
950
|
-
|
|
951
|
-
const embeddingRows = sqliteAll<{ status: string; count: number }>(
|
|
952
|
-
this.db,
|
|
953
|
-
`SELECT status, COUNT(*) as count FROM embedding_outbox GROUP BY status`
|
|
954
|
-
);
|
|
955
|
-
const vectorRows = sqliteAll<{ status: string; count: number }>(
|
|
956
|
-
this.db,
|
|
957
|
-
`SELECT status, COUNT(*) as count FROM vector_outbox GROUP BY status`
|
|
958
|
-
);
|
|
959
|
-
|
|
960
|
-
const fromRows = (rows: Array<{ status: string; count: number }>) => {
|
|
961
|
-
const out = { pending: 0, processing: 0, failed: 0, total: 0 };
|
|
962
|
-
for (const row of rows) {
|
|
963
|
-
const key = row.status as 'pending' | 'processing' | 'failed' | 'done';
|
|
964
|
-
if (key === 'pending' || key === 'processing' || key === 'failed') {
|
|
965
|
-
out[key] += row.count;
|
|
966
|
-
}
|
|
967
|
-
out.total += row.count;
|
|
968
|
-
}
|
|
969
|
-
return out;
|
|
970
|
-
};
|
|
971
|
-
|
|
972
|
-
return {
|
|
973
|
-
embedding: fromRows(embeddingRows),
|
|
974
|
-
vector: fromRows(vectorRows)
|
|
975
|
-
};
|
|
976
|
-
}
|
|
977
|
-
|
|
978
|
-
/**
|
|
979
|
-
* Update memory level
|
|
980
|
-
*/
|
|
981
|
-
async updateMemoryLevel(eventId: string, level: string): Promise<void> {
|
|
982
|
-
await this.initialize();
|
|
983
|
-
|
|
984
|
-
sqliteRun(
|
|
985
|
-
this.db,
|
|
986
|
-
`UPDATE memory_levels SET level = ?, promoted_at = datetime('now') WHERE event_id = ?`,
|
|
987
|
-
[level, eventId]
|
|
988
|
-
);
|
|
989
|
-
}
|
|
990
|
-
|
|
991
|
-
/**
|
|
992
|
-
* Get memory level statistics
|
|
993
|
-
*/
|
|
994
|
-
async getLevelStats(): Promise<Array<{ level: string; count: number }>> {
|
|
995
|
-
await this.initialize();
|
|
996
|
-
|
|
997
|
-
const rows = sqliteAll<{ level: string; count: number }>(
|
|
998
|
-
this.db,
|
|
999
|
-
`SELECT level, COUNT(*) as count FROM memory_levels GROUP BY level`
|
|
1000
|
-
);
|
|
1001
|
-
|
|
1002
|
-
return rows;
|
|
1003
|
-
}
|
|
1004
|
-
|
|
1005
|
-
/**
|
|
1006
|
-
* Get events by memory level
|
|
1007
|
-
*/
|
|
1008
|
-
async getEventsByLevel(level: string, options?: { limit?: number; offset?: number }): Promise<MemoryEvent[]> {
|
|
1009
|
-
await this.initialize();
|
|
1010
|
-
|
|
1011
|
-
const limit = options?.limit || 50;
|
|
1012
|
-
const offset = options?.offset || 0;
|
|
1013
|
-
|
|
1014
|
-
const rows = sqliteAll<Record<string, unknown>>(
|
|
1015
|
-
this.db,
|
|
1016
|
-
`SELECT e.* FROM events e
|
|
1017
|
-
INNER JOIN memory_levels ml ON e.id = ml.event_id
|
|
1018
|
-
WHERE ml.level = ?
|
|
1019
|
-
ORDER BY e.timestamp DESC
|
|
1020
|
-
LIMIT ? OFFSET ?`,
|
|
1021
|
-
[level, limit, offset]
|
|
1022
|
-
);
|
|
1023
|
-
|
|
1024
|
-
return rows.map(row => this.rowToEvent(row));
|
|
1025
|
-
}
|
|
1026
|
-
|
|
1027
|
-
/**
|
|
1028
|
-
* Get memory level for a specific event
|
|
1029
|
-
*/
|
|
1030
|
-
async getEventLevel(eventId: string): Promise<string | null> {
|
|
1031
|
-
await this.initialize();
|
|
1032
|
-
|
|
1033
|
-
const row = sqliteGet<{ level: string }>(
|
|
1034
|
-
this.db,
|
|
1035
|
-
`SELECT level FROM memory_levels WHERE event_id = ?`,
|
|
1036
|
-
[eventId]
|
|
1037
|
-
);
|
|
1038
|
-
|
|
1039
|
-
return row ? row.level : null;
|
|
1040
|
-
}
|
|
1041
|
-
|
|
1042
|
-
/**
|
|
1043
|
-
* Get sync position for a target
|
|
1044
|
-
*/
|
|
1045
|
-
async getSyncPosition(targetName: string): Promise<{ lastEventId: string | null; lastTimestamp: string | null }> {
|
|
1046
|
-
await this.initialize();
|
|
1047
|
-
|
|
1048
|
-
const row = sqliteGet<{ last_event_id: string | null; last_timestamp: string | null }>(
|
|
1049
|
-
this.db,
|
|
1050
|
-
`SELECT last_event_id, last_timestamp FROM sync_positions WHERE target_name = ?`,
|
|
1051
|
-
[targetName]
|
|
1052
|
-
);
|
|
1053
|
-
|
|
1054
|
-
return {
|
|
1055
|
-
lastEventId: row?.last_event_id ?? null,
|
|
1056
|
-
lastTimestamp: row?.last_timestamp ?? null
|
|
1057
|
-
};
|
|
1058
|
-
}
|
|
1059
|
-
|
|
1060
|
-
/**
|
|
1061
|
-
* Update sync position for a target
|
|
1062
|
-
*/
|
|
1063
|
-
async updateSyncPosition(targetName: string, lastEventId: string, lastTimestamp: string): Promise<void> {
|
|
1064
|
-
await this.initialize();
|
|
1065
|
-
|
|
1066
|
-
sqliteRun(
|
|
1067
|
-
this.db,
|
|
1068
|
-
`INSERT OR REPLACE INTO sync_positions (target_name, last_event_id, last_timestamp, updated_at)
|
|
1069
|
-
VALUES (?, ?, ?, datetime('now'))`,
|
|
1070
|
-
[targetName, lastEventId, lastTimestamp]
|
|
1071
|
-
);
|
|
1072
|
-
}
|
|
1073
|
-
|
|
1074
|
-
/**
|
|
1075
|
-
* Get config value for endless mode
|
|
1076
|
-
*/
|
|
1077
|
-
async getEndlessConfig(key: string): Promise<unknown | null> {
|
|
1078
|
-
await this.initialize();
|
|
1079
|
-
|
|
1080
|
-
const row = sqliteGet<{ value: string }>(
|
|
1081
|
-
this.db,
|
|
1082
|
-
`SELECT value FROM endless_config WHERE key = ?`,
|
|
1083
|
-
[key]
|
|
1084
|
-
);
|
|
1085
|
-
|
|
1086
|
-
if (!row) return null;
|
|
1087
|
-
return JSON.parse(row.value);
|
|
1088
|
-
}
|
|
1089
|
-
|
|
1090
|
-
/**
|
|
1091
|
-
* Set config value for endless mode
|
|
1092
|
-
*/
|
|
1093
|
-
async setEndlessConfig(key: string, value: unknown): Promise<void> {
|
|
1094
|
-
await this.initialize();
|
|
1095
|
-
|
|
1096
|
-
sqliteRun(
|
|
1097
|
-
this.db,
|
|
1098
|
-
`INSERT OR REPLACE INTO endless_config (key, value, updated_at)
|
|
1099
|
-
VALUES (?, ?, datetime('now'))`,
|
|
1100
|
-
[key, JSON.stringify(value)]
|
|
1101
|
-
);
|
|
1102
|
-
}
|
|
1103
|
-
|
|
1104
|
-
/**
|
|
1105
|
-
* Increment access count for events
|
|
1106
|
-
*/
|
|
1107
|
-
async incrementAccessCount(eventIds: string[]): Promise<void> {
|
|
1108
|
-
if (eventIds.length === 0 || this.readOnly) return;
|
|
1109
|
-
|
|
1110
|
-
await this.initialize();
|
|
1111
|
-
|
|
1112
|
-
const placeholders = eventIds.map(() => '?').join(',');
|
|
1113
|
-
const currentTime = toSQLiteTimestamp(new Date());
|
|
1114
|
-
|
|
1115
|
-
sqliteRun(
|
|
1116
|
-
this.db,
|
|
1117
|
-
`UPDATE events
|
|
1118
|
-
SET access_count = access_count + 1,
|
|
1119
|
-
last_accessed_at = ?
|
|
1120
|
-
WHERE id IN (${placeholders})`,
|
|
1121
|
-
[currentTime, ...eventIds]
|
|
1122
|
-
);
|
|
1123
|
-
}
|
|
1124
|
-
|
|
1125
|
-
/**
|
|
1126
|
-
* Get most accessed memories (falls back to recent events if none accessed)
|
|
1127
|
-
*/
|
|
1128
|
-
async getMostAccessed(limit: number = 10): Promise<MemoryEvent[]> {
|
|
1129
|
-
await this.initialize();
|
|
1130
|
-
|
|
1131
|
-
// First try events with access_count > 0
|
|
1132
|
-
let rows = sqliteAll<Record<string, unknown>>(
|
|
1133
|
-
this.db,
|
|
1134
|
-
`SELECT * FROM events
|
|
1135
|
-
WHERE access_count > 0
|
|
1136
|
-
ORDER BY access_count DESC, last_accessed_at DESC
|
|
1137
|
-
LIMIT ?`,
|
|
1138
|
-
[limit]
|
|
1139
|
-
);
|
|
1140
|
-
|
|
1141
|
-
// Fallback: if no accessed events, show recent events
|
|
1142
|
-
if (rows.length === 0) {
|
|
1143
|
-
rows = sqliteAll<Record<string, unknown>>(
|
|
1144
|
-
this.db,
|
|
1145
|
-
`SELECT * FROM events
|
|
1146
|
-
ORDER BY timestamp DESC
|
|
1147
|
-
LIMIT ?`,
|
|
1148
|
-
[limit]
|
|
1149
|
-
);
|
|
1150
|
-
}
|
|
1151
|
-
|
|
1152
|
-
return rows.map(row => this.rowToEvent(row));
|
|
1153
|
-
}
|
|
1154
|
-
|
|
1155
|
-
/**
|
|
1156
|
-
* Record a memory retrieval for helpfulness tracking
|
|
1157
|
-
*/
|
|
1158
|
-
async recordRetrieval(eventId: string, sessionId: string, score: number, query: string): Promise<void> {
|
|
1159
|
-
if (this.readOnly) return;
|
|
1160
|
-
await this.initialize();
|
|
1161
|
-
|
|
1162
|
-
const id = randomUUID();
|
|
1163
|
-
sqliteRun(
|
|
1164
|
-
this.db,
|
|
1165
|
-
`INSERT INTO memory_helpfulness (id, event_id, session_id, retrieval_score, query_preview, created_at)
|
|
1166
|
-
VALUES (?, ?, ?, ?, ?, datetime('now'))`,
|
|
1167
|
-
[id, eventId, sessionId, score, query.slice(0, 100)]
|
|
1168
|
-
);
|
|
1169
|
-
}
|
|
1170
|
-
|
|
1171
|
-
/**
|
|
1172
|
-
* Get session IDs that have unevaluated retrievals (measured_at IS NULL).
|
|
1173
|
-
* Excludes the current session. Used to backfill sessions that ended without Stop hook.
|
|
1174
|
-
*/
|
|
1175
|
-
async getUnevaluatedSessions(currentSessionId: string, limit = 5): Promise<string[]> {
|
|
1176
|
-
await this.initialize();
|
|
1177
|
-
const rows = sqliteAll<{ session_id: string }>(
|
|
1178
|
-
this.db,
|
|
1179
|
-
`SELECT DISTINCT session_id FROM memory_helpfulness
|
|
1180
|
-
WHERE measured_at IS NULL AND session_id != ?
|
|
1181
|
-
ORDER BY created_at DESC LIMIT ?`,
|
|
1182
|
-
[currentSessionId, limit]
|
|
1183
|
-
);
|
|
1184
|
-
return rows.map((r) => r.session_id);
|
|
1185
|
-
}
|
|
1186
|
-
|
|
1187
|
-
/**
|
|
1188
|
-
* Evaluate helpfulness for all retrievals in a session
|
|
1189
|
-
* Called at session end - uses behavioral signals to compute score
|
|
1190
|
-
*/
|
|
1191
|
-
async evaluateSessionHelpfulness(sessionId: string): Promise<void> {
|
|
1192
|
-
if (this.readOnly) return;
|
|
1193
|
-
await this.initialize();
|
|
1194
|
-
|
|
1195
|
-
// Get all retrieval records for this session
|
|
1196
|
-
const retrievals = sqliteAll<Record<string, unknown>>(
|
|
1197
|
-
this.db,
|
|
1198
|
-
`SELECT * FROM memory_helpfulness WHERE session_id = ? AND measured_at IS NULL`,
|
|
1199
|
-
[sessionId]
|
|
1200
|
-
);
|
|
1201
|
-
|
|
1202
|
-
if (retrievals.length === 0) return;
|
|
1203
|
-
|
|
1204
|
-
// Get session events to analyze behavior after retrieval
|
|
1205
|
-
const sessionEvents = sqliteAll<Record<string, unknown>>(
|
|
1206
|
-
this.db,
|
|
1207
|
-
`SELECT * FROM events WHERE session_id = ? ORDER BY timestamp ASC`,
|
|
1208
|
-
[sessionId]
|
|
1209
|
-
);
|
|
1210
|
-
|
|
1211
|
-
const promptEvents = sessionEvents.filter((e: any) => e.event_type === 'user_prompt');
|
|
1212
|
-
const toolEvents = sessionEvents.filter((e: any) => e.event_type === 'tool_observation');
|
|
1213
|
-
|
|
1214
|
-
// Count successful vs failed tools
|
|
1215
|
-
let toolSuccessCount = 0;
|
|
1216
|
-
let toolTotalCount = toolEvents.length;
|
|
1217
|
-
for (const t of toolEvents) {
|
|
1218
|
-
try {
|
|
1219
|
-
const content = JSON.parse(t.content as string);
|
|
1220
|
-
if (content.success !== false) toolSuccessCount++;
|
|
1221
|
-
} catch {
|
|
1222
|
-
toolSuccessCount++; // Assume success if can't parse
|
|
1223
|
-
}
|
|
1224
|
-
}
|
|
1225
|
-
const toolSuccessRatio = toolTotalCount > 0 ? toolSuccessCount / toolTotalCount : 0.5;
|
|
1226
|
-
|
|
1227
|
-
for (const retrieval of retrievals) {
|
|
1228
|
-
const retrievalTime = retrieval.created_at as string;
|
|
1229
|
-
|
|
1230
|
-
// 1. Session continued after retrieval?
|
|
1231
|
-
const eventsAfter = sessionEvents.filter((e: any) => e.timestamp > retrievalTime);
|
|
1232
|
-
const sessionContinued = eventsAfter.length > 0 ? 1 : 0;
|
|
1233
|
-
|
|
1234
|
-
// 2. How many prompts came after?
|
|
1235
|
-
const promptsAfter = promptEvents.filter((e: any) => e.timestamp > retrievalTime);
|
|
1236
|
-
const promptCountAfter = promptsAfter.length;
|
|
1237
|
-
|
|
1238
|
-
// 3. Was a similar query asked again? (simple word overlap check)
|
|
1239
|
-
const queryWords = new Set((retrieval.query_preview as string || '').toLowerCase().split(/\s+/).filter(w => w.length > 2));
|
|
1240
|
-
let wasReasked = 0;
|
|
1241
|
-
for (const p of promptsAfter) {
|
|
1242
|
-
const pWords = new Set((p.content as string).toLowerCase().split(/\s+/).filter((w: string) => w.length > 2));
|
|
1243
|
-
let overlap = 0;
|
|
1244
|
-
for (const w of queryWords) {
|
|
1245
|
-
if (pWords.has(w)) overlap++;
|
|
1246
|
-
}
|
|
1247
|
-
if (queryWords.size > 0 && overlap / queryWords.size > 0.5) {
|
|
1248
|
-
wasReasked = 1;
|
|
1249
|
-
break;
|
|
1250
|
-
}
|
|
1251
|
-
}
|
|
1252
|
-
|
|
1253
|
-
// Calculate helpfulness score
|
|
1254
|
-
// Weights tuned for shopping-assistant-like corpora where sessions
|
|
1255
|
-
// continue on the same topic (was_reasked was over-penalising normal conversation flow)
|
|
1256
|
-
const retrievalScore = retrieval.retrieval_score as number || 0;
|
|
1257
|
-
// More prompts after retrieval = memory was actually useful to the conversation
|
|
1258
|
-
const promptNorm = Math.min(promptCountAfter / 2, 1.0);
|
|
1259
|
-
const helpfulnessScore = (
|
|
1260
|
-
0.40 * Math.min(retrievalScore, 1.0) +
|
|
1261
|
-
0.30 * promptNorm +
|
|
1262
|
-
0.20 * toolSuccessRatio +
|
|
1263
|
-
0.10 * (sessionContinued ? 1.0 : 0.0)
|
|
1264
|
-
);
|
|
1265
|
-
|
|
1266
|
-
sqliteRun(
|
|
1267
|
-
this.db,
|
|
1268
|
-
`UPDATE memory_helpfulness
|
|
1269
|
-
SET session_continued = ?, prompt_count_after = ?,
|
|
1270
|
-
tool_success_count = ?, tool_total_count = ?,
|
|
1271
|
-
was_reasked = ?, helpfulness_score = ?,
|
|
1272
|
-
measured_at = datetime('now')
|
|
1273
|
-
WHERE id = ?`,
|
|
1274
|
-
[sessionContinued, promptCountAfter, toolSuccessCount, toolTotalCount,
|
|
1275
|
-
wasReasked, helpfulnessScore, retrieval.id]
|
|
1276
|
-
);
|
|
1277
|
-
}
|
|
1278
|
-
}
|
|
1279
|
-
|
|
1280
|
-
/**
|
|
1281
|
-
* Get most helpful memories ranked by helpfulness score
|
|
1282
|
-
*/
|
|
1283
|
-
async getHelpfulMemories(limit: number = 10): Promise<Array<{
|
|
1284
|
-
eventId: string;
|
|
1285
|
-
summary: string;
|
|
1286
|
-
helpfulnessScore: number;
|
|
1287
|
-
accessCount: number;
|
|
1288
|
-
evaluationCount: number;
|
|
1289
|
-
}>> {
|
|
1290
|
-
await this.initialize();
|
|
1291
|
-
|
|
1292
|
-
const rows = sqliteAll<Record<string, unknown>>(
|
|
1293
|
-
this.db,
|
|
1294
|
-
`SELECT
|
|
1295
|
-
mh.event_id,
|
|
1296
|
-
AVG(mh.helpfulness_score) as avg_score,
|
|
1297
|
-
COUNT(*) as eval_count,
|
|
1298
|
-
e.content,
|
|
1299
|
-
e.access_count
|
|
1300
|
-
FROM memory_helpfulness mh
|
|
1301
|
-
JOIN events e ON e.id = mh.event_id
|
|
1302
|
-
WHERE mh.measured_at IS NOT NULL
|
|
1303
|
-
GROUP BY mh.event_id
|
|
1304
|
-
ORDER BY avg_score DESC
|
|
1305
|
-
LIMIT ?`,
|
|
1306
|
-
[limit]
|
|
1307
|
-
);
|
|
1308
|
-
|
|
1309
|
-
return rows.map(r => ({
|
|
1310
|
-
eventId: r.event_id as string,
|
|
1311
|
-
summary: (r.content as string).substring(0, 200) + ((r.content as string).length > 200 ? '...' : ''),
|
|
1312
|
-
helpfulnessScore: Math.round((r.avg_score as number) * 100) / 100,
|
|
1313
|
-
accessCount: (r.access_count as number) || 0,
|
|
1314
|
-
evaluationCount: r.eval_count as number
|
|
1315
|
-
}));
|
|
1316
|
-
}
|
|
1317
|
-
|
|
1318
|
-
/**
|
|
1319
|
-
* Get helpfulness statistics for dashboard
|
|
1320
|
-
*/
|
|
1321
|
-
async getHelpfulnessStats(): Promise<{
|
|
1322
|
-
avgScore: number;
|
|
1323
|
-
totalEvaluated: number;
|
|
1324
|
-
totalRetrievals: number;
|
|
1325
|
-
helpful: number;
|
|
1326
|
-
neutral: number;
|
|
1327
|
-
unhelpful: number;
|
|
1328
|
-
}> {
|
|
1329
|
-
await this.initialize();
|
|
1330
|
-
|
|
1331
|
-
const stats = sqliteGet<Record<string, unknown>>(
|
|
1332
|
-
this.db,
|
|
1333
|
-
`SELECT
|
|
1334
|
-
AVG(helpfulness_score) as avg_score,
|
|
1335
|
-
COUNT(*) as total_evaluated,
|
|
1336
|
-
SUM(CASE WHEN helpfulness_score >= 0.7 THEN 1 ELSE 0 END) as helpful,
|
|
1337
|
-
SUM(CASE WHEN helpfulness_score >= 0.4 AND helpfulness_score < 0.7 THEN 1 ELSE 0 END) as neutral,
|
|
1338
|
-
SUM(CASE WHEN helpfulness_score < 0.4 THEN 1 ELSE 0 END) as unhelpful
|
|
1339
|
-
FROM memory_helpfulness
|
|
1340
|
-
WHERE measured_at IS NOT NULL`
|
|
1341
|
-
);
|
|
1342
|
-
|
|
1343
|
-
const totalRow = sqliteGet<Record<string, unknown>>(
|
|
1344
|
-
this.db,
|
|
1345
|
-
`SELECT COUNT(*) as total FROM memory_helpfulness`
|
|
1346
|
-
);
|
|
1347
|
-
|
|
1348
|
-
return {
|
|
1349
|
-
avgScore: Math.round(((stats?.avg_score as number) || 0) * 100) / 100,
|
|
1350
|
-
totalEvaluated: (stats?.total_evaluated as number) || 0,
|
|
1351
|
-
totalRetrievals: (totalRow?.total as number) || 0,
|
|
1352
|
-
helpful: (stats?.helpful as number) || 0,
|
|
1353
|
-
neutral: (stats?.neutral as number) || 0,
|
|
1354
|
-
unhelpful: (stats?.unhelpful as number) || 0
|
|
1355
|
-
};
|
|
1356
|
-
}
|
|
1357
|
-
|
|
1358
|
-
/**
|
|
1359
|
-
* Fast keyword search using FTS5
|
|
1360
|
-
* Returns events matching the search query, ranked by relevance
|
|
1361
|
-
*/
|
|
1362
|
-
async keywordSearch(query: string, limit: number = 10): Promise<Array<{event: MemoryEvent; rank: number}>> {
|
|
1363
|
-
await this.initialize();
|
|
1364
|
-
|
|
1365
|
-
// Escape special FTS5 characters and prepare search terms
|
|
1366
|
-
const searchTerms = query
|
|
1367
|
-
.replace(/['"(){}[\]^~*?:\\/-]/g, ' ') // Remove special chars
|
|
1368
|
-
.split(/\s+/)
|
|
1369
|
-
.filter(term => term.length > 1) // Filter short terms
|
|
1370
|
-
.map(term => `"${term}"*`) // Prefix matching
|
|
1371
|
-
.join(' OR ');
|
|
1372
|
-
|
|
1373
|
-
if (!searchTerms) {
|
|
1374
|
-
return [];
|
|
1375
|
-
}
|
|
1376
|
-
|
|
1377
|
-
try {
|
|
1378
|
-
const rows = sqliteAll<Record<string, unknown>>(
|
|
1379
|
-
this.db,
|
|
1380
|
-
`SELECT e.*, fts.rank
|
|
1381
|
-
FROM events_fts fts
|
|
1382
|
-
JOIN events e ON e.id = fts.event_id
|
|
1383
|
-
WHERE events_fts MATCH ?
|
|
1384
|
-
ORDER BY fts.rank
|
|
1385
|
-
LIMIT ?`,
|
|
1386
|
-
[searchTerms, limit]
|
|
1387
|
-
);
|
|
1388
|
-
|
|
1389
|
-
return rows.map(row => ({
|
|
1390
|
-
event: this.rowToEvent(row),
|
|
1391
|
-
rank: row.rank as number
|
|
1392
|
-
}));
|
|
1393
|
-
} catch (error: any) {
|
|
1394
|
-
// FTS table might not exist yet (old database)
|
|
1395
|
-
// Fallback to LIKE search
|
|
1396
|
-
const likePattern = `%${query}%`;
|
|
1397
|
-
const rows = sqliteAll<Record<string, unknown>>(
|
|
1398
|
-
this.db,
|
|
1399
|
-
`SELECT *, 0 as rank FROM events
|
|
1400
|
-
WHERE content LIKE ?
|
|
1401
|
-
ORDER BY timestamp DESC
|
|
1402
|
-
LIMIT ?`,
|
|
1403
|
-
[likePattern, limit]
|
|
1404
|
-
);
|
|
1405
|
-
|
|
1406
|
-
return rows.map(row => ({
|
|
1407
|
-
event: this.rowToEvent(row),
|
|
1408
|
-
rank: 0
|
|
1409
|
-
}));
|
|
1410
|
-
}
|
|
1411
|
-
}
|
|
1412
|
-
|
|
1413
|
-
/**
|
|
1414
|
-
* Rebuild FTS index from existing events
|
|
1415
|
-
* Call this once after upgrading to FTS5
|
|
1416
|
-
*/
|
|
1417
|
-
async rebuildFtsIndex(): Promise<number> {
|
|
1418
|
-
await this.initialize();
|
|
1419
|
-
|
|
1420
|
-
// Get count of events to index
|
|
1421
|
-
const countRow = sqliteGet<{count: number}>(this.db, 'SELECT COUNT(*) as count FROM events', []);
|
|
1422
|
-
const totalEvents = countRow?.count ?? 0;
|
|
1423
|
-
|
|
1424
|
-
// Clear and rebuild FTS index. Recreate the virtual table instead of
|
|
1425
|
-
// issuing DELETE against it: older migrated FTS5 tables/triggers can fail
|
|
1426
|
-
// with `no such column: T.event_id` when processing synthetic deletes.
|
|
1427
|
-
sqliteExec(this.db, `
|
|
1428
|
-
DROP TRIGGER IF EXISTS events_fts_insert;
|
|
1429
|
-
DROP TRIGGER IF EXISTS events_fts_delete;
|
|
1430
|
-
DROP TRIGGER IF EXISTS events_fts_update;
|
|
1431
|
-
DROP TABLE IF EXISTS events_fts;
|
|
1432
|
-
|
|
1433
|
-
CREATE VIRTUAL TABLE events_fts USING fts5(
|
|
1434
|
-
content,
|
|
1435
|
-
event_id UNINDEXED,
|
|
1436
|
-
tokenize='porter unicode61'
|
|
1437
|
-
);
|
|
1438
|
-
|
|
1439
|
-
INSERT INTO events_fts(rowid, content, event_id)
|
|
1440
|
-
SELECT rowid, content, id FROM events;
|
|
1441
|
-
|
|
1442
|
-
CREATE TRIGGER events_fts_insert AFTER INSERT ON events BEGIN
|
|
1443
|
-
INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
|
|
1444
|
-
END;
|
|
1445
|
-
|
|
1446
|
-
CREATE TRIGGER events_fts_delete AFTER DELETE ON events BEGIN
|
|
1447
|
-
DELETE FROM events_fts WHERE rowid = OLD.rowid;
|
|
1448
|
-
END;
|
|
1449
|
-
|
|
1450
|
-
CREATE TRIGGER events_fts_update AFTER UPDATE ON events BEGIN
|
|
1451
|
-
DELETE FROM events_fts WHERE rowid = OLD.rowid;
|
|
1452
|
-
INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
|
|
1453
|
-
END;
|
|
1454
|
-
`);
|
|
1455
|
-
|
|
1456
|
-
return totalEvents;
|
|
1457
|
-
}
|
|
1458
|
-
|
|
1459
|
-
/**
|
|
1460
|
-
* Get database instance for direct access
|
|
1461
|
-
*/
|
|
1462
|
-
getDatabase(): SQLiteDatabase {
|
|
1463
|
-
return this.db;
|
|
1464
|
-
}
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
async recordRetrievalTrace(input: {
|
|
1468
|
-
sessionId?: string;
|
|
1469
|
-
projectHash?: string;
|
|
1470
|
-
queryText: string;
|
|
1471
|
-
strategy?: string;
|
|
1472
|
-
candidateEventIds: string[];
|
|
1473
|
-
selectedEventIds: string[];
|
|
1474
|
-
candidateDetails?: Array<{
|
|
1475
|
-
eventId: string;
|
|
1476
|
-
score: number;
|
|
1477
|
-
semanticScore?: number;
|
|
1478
|
-
lexicalScore?: number;
|
|
1479
|
-
recencyScore?: number;
|
|
1480
|
-
}>;
|
|
1481
|
-
selectedDetails?: Array<{
|
|
1482
|
-
eventId: string;
|
|
1483
|
-
score: number;
|
|
1484
|
-
semanticScore?: number;
|
|
1485
|
-
lexicalScore?: number;
|
|
1486
|
-
recencyScore?: number;
|
|
1487
|
-
}>;
|
|
1488
|
-
confidence?: string;
|
|
1489
|
-
fallbackTrace?: string[];
|
|
1490
|
-
}): Promise<void> {
|
|
1491
|
-
await this.initialize();
|
|
1492
|
-
|
|
1493
|
-
const traceId = randomUUID();
|
|
1494
|
-
sqliteRun(
|
|
1495
|
-
this.db,
|
|
1496
|
-
`INSERT INTO retrieval_traces (
|
|
1497
|
-
trace_id, session_id, project_hash, query_text, strategy,
|
|
1498
|
-
candidate_event_ids, selected_event_ids, candidate_details_json, selected_details_json,
|
|
1499
|
-
candidate_count, selected_count, confidence, fallback_trace
|
|
1500
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
1501
|
-
[
|
|
1502
|
-
traceId,
|
|
1503
|
-
input.sessionId || null,
|
|
1504
|
-
input.projectHash || null,
|
|
1505
|
-
input.queryText,
|
|
1506
|
-
input.strategy || null,
|
|
1507
|
-
JSON.stringify(input.candidateEventIds || []),
|
|
1508
|
-
JSON.stringify(input.selectedEventIds || []),
|
|
1509
|
-
JSON.stringify(input.candidateDetails || []),
|
|
1510
|
-
JSON.stringify(input.selectedDetails || []),
|
|
1511
|
-
(input.candidateEventIds || []).length,
|
|
1512
|
-
(input.selectedEventIds || []).length,
|
|
1513
|
-
input.confidence || null,
|
|
1514
|
-
JSON.stringify(input.fallbackTrace || [])
|
|
1515
|
-
]
|
|
1516
|
-
);
|
|
1517
|
-
}
|
|
1518
|
-
|
|
1519
|
-
async getRecentRetrievalTraces(limit: number = 50): Promise<Array<{
|
|
1520
|
-
traceId: string;
|
|
1521
|
-
sessionId?: string;
|
|
1522
|
-
projectHash?: string;
|
|
1523
|
-
queryText: string;
|
|
1524
|
-
strategy?: string;
|
|
1525
|
-
candidateEventIds: string[];
|
|
1526
|
-
selectedEventIds: string[];
|
|
1527
|
-
candidateDetails: Array<{
|
|
1528
|
-
eventId: string;
|
|
1529
|
-
score: number;
|
|
1530
|
-
semanticScore?: number;
|
|
1531
|
-
lexicalScore?: number;
|
|
1532
|
-
recencyScore?: number;
|
|
1533
|
-
}>;
|
|
1534
|
-
selectedDetails: Array<{
|
|
1535
|
-
eventId: string;
|
|
1536
|
-
score: number;
|
|
1537
|
-
semanticScore?: number;
|
|
1538
|
-
lexicalScore?: number;
|
|
1539
|
-
recencyScore?: number;
|
|
1540
|
-
}>;
|
|
1541
|
-
candidateCount: number;
|
|
1542
|
-
selectedCount: number;
|
|
1543
|
-
confidence?: string;
|
|
1544
|
-
fallbackTrace: string[];
|
|
1545
|
-
createdAt: Date;
|
|
1546
|
-
}>> {
|
|
1547
|
-
await this.initialize();
|
|
1548
|
-
|
|
1549
|
-
try {
|
|
1550
|
-
const rows = sqliteAll<Record<string, unknown>>(
|
|
1551
|
-
this.db,
|
|
1552
|
-
`SELECT * FROM retrieval_traces ORDER BY created_at DESC LIMIT ?`,
|
|
1553
|
-
[limit]
|
|
1554
|
-
);
|
|
1555
|
-
|
|
1556
|
-
return rows.map((row) => ({
|
|
1557
|
-
traceId: row.trace_id as string,
|
|
1558
|
-
sessionId: (row.session_id as string) || undefined,
|
|
1559
|
-
projectHash: (row.project_hash as string) || undefined,
|
|
1560
|
-
queryText: row.query_text as string,
|
|
1561
|
-
strategy: (row.strategy as string) || undefined,
|
|
1562
|
-
candidateEventIds: row.candidate_event_ids ? JSON.parse(row.candidate_event_ids as string) : [],
|
|
1563
|
-
selectedEventIds: row.selected_event_ids ? JSON.parse(row.selected_event_ids as string) : [],
|
|
1564
|
-
candidateDetails: row.candidate_details_json ? JSON.parse(row.candidate_details_json as string) : [],
|
|
1565
|
-
selectedDetails: row.selected_details_json ? JSON.parse(row.selected_details_json as string) : [],
|
|
1566
|
-
candidateCount: Number(row.candidate_count || 0),
|
|
1567
|
-
selectedCount: Number(row.selected_count || 0),
|
|
1568
|
-
confidence: (row.confidence as string) || undefined,
|
|
1569
|
-
fallbackTrace: row.fallback_trace ? JSON.parse(row.fallback_trace as string) : [],
|
|
1570
|
-
createdAt: toDateFromSQLite(row.created_at),
|
|
1571
|
-
}));
|
|
1572
|
-
} catch (err: any) {
|
|
1573
|
-
if (err?.message?.includes('no such table')) return [];
|
|
1574
|
-
throw err;
|
|
1575
|
-
}
|
|
1576
|
-
}
|
|
1577
|
-
|
|
1578
|
-
async getRetrievalTraceStats(): Promise<{
|
|
1579
|
-
totalQueries: number;
|
|
1580
|
-
avgCandidateCount: number;
|
|
1581
|
-
avgSelectedCount: number;
|
|
1582
|
-
selectionRate: number;
|
|
1583
|
-
}> {
|
|
1584
|
-
await this.initialize();
|
|
1585
|
-
|
|
1586
|
-
try {
|
|
1587
|
-
const row = sqliteGet<Record<string, unknown>>(
|
|
1588
|
-
this.db,
|
|
1589
|
-
`SELECT
|
|
1590
|
-
COUNT(*) as total_queries,
|
|
1591
|
-
AVG(candidate_count) as avg_candidate_count,
|
|
1592
|
-
AVG(selected_count) as avg_selected_count,
|
|
1593
|
-
CASE
|
|
1594
|
-
WHEN SUM(candidate_count) > 0 THEN (SUM(selected_count) * 1.0 / SUM(candidate_count))
|
|
1595
|
-
ELSE 0
|
|
1596
|
-
END as selection_rate
|
|
1597
|
-
FROM retrieval_traces`,
|
|
1598
|
-
[]
|
|
1599
|
-
);
|
|
1600
|
-
|
|
1601
|
-
return {
|
|
1602
|
-
totalQueries: Number(row?.total_queries || 0),
|
|
1603
|
-
avgCandidateCount: Number(row?.avg_candidate_count || 0),
|
|
1604
|
-
avgSelectedCount: Number(row?.avg_selected_count || 0),
|
|
1605
|
-
selectionRate: Number(row?.selection_rate || 0),
|
|
1606
|
-
};
|
|
1607
|
-
} catch (err: any) {
|
|
1608
|
-
if (err?.message?.includes('no such table')) {
|
|
1609
|
-
return { totalQueries: 0, avgCandidateCount: 0, avgSelectedCount: 0, selectionRate: 0 };
|
|
1610
|
-
}
|
|
1611
|
-
throw err;
|
|
1612
|
-
}
|
|
1613
|
-
}
|
|
1614
|
-
|
|
1615
|
-
/**
|
|
1616
|
-
* Close database connection
|
|
1617
|
-
*/
|
|
1618
|
-
async close(): Promise<void> {
|
|
1619
|
-
sqliteClose(this.db);
|
|
1620
|
-
}
|
|
1621
|
-
|
|
1622
|
-
/**
|
|
1623
|
-
* Get events grouped by turn_id for a session
|
|
1624
|
-
* Returns turns ordered by first event timestamp (newest first)
|
|
1625
|
-
*/
|
|
1626
|
-
async getSessionTurns(sessionId: string, options?: { limit?: number; offset?: number }): Promise<Array<{
|
|
1627
|
-
turnId: string;
|
|
1628
|
-
events: MemoryEvent[];
|
|
1629
|
-
startedAt: Date;
|
|
1630
|
-
promptPreview: string;
|
|
1631
|
-
eventCount: number;
|
|
1632
|
-
toolCount: number;
|
|
1633
|
-
hasResponse: boolean;
|
|
1634
|
-
}>> {
|
|
1635
|
-
await this.initialize();
|
|
1636
|
-
|
|
1637
|
-
const limit = options?.limit || 20;
|
|
1638
|
-
const offset = options?.offset || 0;
|
|
1639
|
-
|
|
1640
|
-
// Get distinct turn_ids for this session, ordered by first event timestamp
|
|
1641
|
-
const turnRows = sqliteAll<{ turn_id: string; min_ts: string }>(
|
|
1642
|
-
this.db,
|
|
1643
|
-
`SELECT turn_id, MIN(timestamp) as min_ts
|
|
1644
|
-
FROM events
|
|
1645
|
-
WHERE session_id = ? AND turn_id IS NOT NULL
|
|
1646
|
-
GROUP BY turn_id
|
|
1647
|
-
ORDER BY min_ts DESC
|
|
1648
|
-
LIMIT ? OFFSET ?`,
|
|
1649
|
-
[sessionId, limit, offset]
|
|
1650
|
-
);
|
|
1651
|
-
|
|
1652
|
-
const turns: Array<{
|
|
1653
|
-
turnId: string;
|
|
1654
|
-
events: MemoryEvent[];
|
|
1655
|
-
startedAt: Date;
|
|
1656
|
-
promptPreview: string;
|
|
1657
|
-
eventCount: number;
|
|
1658
|
-
toolCount: number;
|
|
1659
|
-
hasResponse: boolean;
|
|
1660
|
-
}> = [];
|
|
1661
|
-
|
|
1662
|
-
for (const turnRow of turnRows) {
|
|
1663
|
-
const events = await this.getEventsByTurn(turnRow.turn_id);
|
|
1664
|
-
|
|
1665
|
-
const promptEvent = events.find(e => e.eventType === 'user_prompt');
|
|
1666
|
-
const toolEvents = events.filter(e => e.eventType === 'tool_observation');
|
|
1667
|
-
const hasResponse = events.some(e => e.eventType === 'agent_response');
|
|
1668
|
-
|
|
1669
|
-
turns.push({
|
|
1670
|
-
turnId: turnRow.turn_id,
|
|
1671
|
-
events,
|
|
1672
|
-
startedAt: toDateFromSQLite(turnRow.min_ts),
|
|
1673
|
-
promptPreview: promptEvent
|
|
1674
|
-
? promptEvent.content.slice(0, 200) + (promptEvent.content.length > 200 ? '...' : '')
|
|
1675
|
-
: '(no prompt)',
|
|
1676
|
-
eventCount: events.length,
|
|
1677
|
-
toolCount: toolEvents.length,
|
|
1678
|
-
hasResponse
|
|
1679
|
-
});
|
|
1680
|
-
}
|
|
1681
|
-
|
|
1682
|
-
return turns;
|
|
1683
|
-
}
|
|
1684
|
-
|
|
1685
|
-
/**
|
|
1686
|
-
* Get all events for a specific turn_id
|
|
1687
|
-
*/
|
|
1688
|
-
async getEventsByTurn(turnId: string): Promise<MemoryEvent[]> {
|
|
1689
|
-
await this.initialize();
|
|
1690
|
-
|
|
1691
|
-
const rows = sqliteAll<Record<string, unknown>>(
|
|
1692
|
-
this.db,
|
|
1693
|
-
`SELECT * FROM events WHERE turn_id = ? ORDER BY timestamp ASC`,
|
|
1694
|
-
[turnId]
|
|
1695
|
-
);
|
|
1696
|
-
|
|
1697
|
-
return rows.map(this.rowToEvent);
|
|
1698
|
-
}
|
|
1699
|
-
|
|
1700
|
-
/**
|
|
1701
|
-
* Count total turns for a session
|
|
1702
|
-
*/
|
|
1703
|
-
async countSessionTurns(sessionId: string): Promise<number> {
|
|
1704
|
-
await this.initialize();
|
|
1705
|
-
|
|
1706
|
-
const row = sqliteGet<{ count: number }>(
|
|
1707
|
-
this.db,
|
|
1708
|
-
`SELECT COUNT(DISTINCT turn_id) as count
|
|
1709
|
-
FROM events
|
|
1710
|
-
WHERE session_id = ? AND turn_id IS NOT NULL`,
|
|
1711
|
-
[sessionId]
|
|
1712
|
-
);
|
|
1713
|
-
|
|
1714
|
-
return row?.count || 0;
|
|
1715
|
-
}
|
|
1716
|
-
|
|
1717
|
-
/**
|
|
1718
|
-
* Migrate existing events: backfill turn_id for events that have turnId in metadata
|
|
1719
|
-
* but no turn_id column value (for events stored before this migration)
|
|
1720
|
-
*/
|
|
1721
|
-
async backfillTurnIds(): Promise<number> {
|
|
1722
|
-
await this.initialize();
|
|
1723
|
-
|
|
1724
|
-
// Find events with turnId in metadata JSON but no turn_id column value
|
|
1725
|
-
const rows = sqliteAll<{ id: string; metadata: string }>(
|
|
1726
|
-
this.db,
|
|
1727
|
-
`SELECT id, metadata FROM events
|
|
1728
|
-
WHERE turn_id IS NULL AND metadata IS NOT NULL AND metadata LIKE '%turnId%'`
|
|
1729
|
-
);
|
|
1730
|
-
|
|
1731
|
-
let updated = 0;
|
|
1732
|
-
for (const row of rows) {
|
|
1733
|
-
try {
|
|
1734
|
-
const metadata = JSON.parse(row.metadata);
|
|
1735
|
-
if (metadata.turnId) {
|
|
1736
|
-
sqliteRun(
|
|
1737
|
-
this.db,
|
|
1738
|
-
`UPDATE events SET turn_id = ? WHERE id = ?`,
|
|
1739
|
-
[metadata.turnId, row.id]
|
|
1740
|
-
);
|
|
1741
|
-
updated++;
|
|
1742
|
-
}
|
|
1743
|
-
} catch {
|
|
1744
|
-
// Skip rows with invalid JSON
|
|
1745
|
-
}
|
|
1746
|
-
}
|
|
1747
|
-
|
|
1748
|
-
return updated;
|
|
1749
|
-
}
|
|
1750
|
-
|
|
1751
|
-
/**
|
|
1752
|
-
* Delete all events for a session (for force reimport)
|
|
1753
|
-
*/
|
|
1754
|
-
async deleteSessionEvents(sessionId: string): Promise<number> {
|
|
1755
|
-
await this.initialize();
|
|
1756
|
-
|
|
1757
|
-
// Get event IDs first for cascading deletes
|
|
1758
|
-
const events = sqliteAll<{ id: string }>(
|
|
1759
|
-
this.db,
|
|
1760
|
-
`SELECT id FROM events WHERE session_id = ?`,
|
|
1761
|
-
[sessionId]
|
|
1762
|
-
);
|
|
1763
|
-
|
|
1764
|
-
if (events.length === 0) return 0;
|
|
1765
|
-
|
|
1766
|
-
const eventIds = events.map(e => e.id);
|
|
1767
|
-
const placeholders = eventIds.map(() => '?').join(',');
|
|
1768
|
-
|
|
1769
|
-
// Drop FTS triggers to prevent SQLITE_CORRUPT_VTAB during bulk delete
|
|
1770
|
-
const ftsTriggersDropped: string[] = [];
|
|
1771
|
-
for (const triggerName of ['events_fts_delete', 'events_fts_update', 'events_fts_insert']) {
|
|
1772
|
-
try {
|
|
1773
|
-
sqliteRun(this.db, `DROP TRIGGER IF EXISTS ${triggerName}`);
|
|
1774
|
-
ftsTriggersDropped.push(triggerName);
|
|
1775
|
-
} catch {
|
|
1776
|
-
// Trigger may not exist
|
|
1777
|
-
}
|
|
1778
|
-
}
|
|
1779
|
-
|
|
1780
|
-
// Delete from related tables first (some may not exist depending on DB version)
|
|
1781
|
-
for (const table of ['event_dedup', 'memory_levels', 'embedding_queue', 'embedding_outbox', 'vector_outbox']) {
|
|
1782
|
-
try {
|
|
1783
|
-
sqliteRun(this.db, `DELETE FROM ${table} WHERE event_id IN (${placeholders})`, eventIds);
|
|
1784
|
-
} catch {
|
|
1785
|
-
// Table may not exist
|
|
1786
|
-
}
|
|
1787
|
-
}
|
|
1788
|
-
|
|
1789
|
-
// Delete events
|
|
1790
|
-
const result = sqliteRun(this.db, `DELETE FROM events WHERE session_id = ?`, [sessionId]);
|
|
1791
|
-
|
|
1792
|
-
// Rebuild FTS index if we dropped triggers
|
|
1793
|
-
if (ftsTriggersDropped.length > 0) {
|
|
1794
|
-
try {
|
|
1795
|
-
// Rebuild FTS from remaining events
|
|
1796
|
-
sqliteRun(this.db, `INSERT INTO events_fts(events_fts) VALUES('rebuild')`);
|
|
1797
|
-
|
|
1798
|
-
// Recreate triggers
|
|
1799
|
-
sqliteRun(this.db, `CREATE TRIGGER IF NOT EXISTS events_fts_insert AFTER INSERT ON events BEGIN
|
|
1800
|
-
INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
|
|
1801
|
-
END`);
|
|
1802
|
-
sqliteRun(this.db, `CREATE TRIGGER IF NOT EXISTS events_fts_delete AFTER DELETE ON events BEGIN
|
|
1803
|
-
DELETE FROM events_fts WHERE rowid = OLD.rowid;
|
|
1804
|
-
END`);
|
|
1805
|
-
sqliteRun(this.db, `CREATE TRIGGER IF NOT EXISTS events_fts_update AFTER UPDATE ON events BEGIN
|
|
1806
|
-
DELETE FROM events_fts WHERE rowid = OLD.rowid;
|
|
1807
|
-
INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
|
|
1808
|
-
END`);
|
|
1809
|
-
} catch {
|
|
1810
|
-
// FTS rebuild failed - non-critical, will be rebuilt on next initialize
|
|
1811
|
-
}
|
|
1812
|
-
}
|
|
1813
|
-
|
|
1814
|
-
return result.changes || 0;
|
|
1815
|
-
}
|
|
1816
|
-
|
|
1817
|
-
/**
|
|
1818
|
-
* Convert database row to MemoryEvent
|
|
1819
|
-
*/
|
|
1820
|
-
private rowToEvent(row: Record<string, unknown>): MemoryEvent {
|
|
1821
|
-
const event: any = {
|
|
1822
|
-
id: row.id as string,
|
|
1823
|
-
eventType: row.event_type as 'user_prompt' | 'agent_response' | 'session_summary',
|
|
1824
|
-
sessionId: row.session_id as string,
|
|
1825
|
-
timestamp: toDateFromSQLite(row.timestamp),
|
|
1826
|
-
content: row.content as string,
|
|
1827
|
-
canonicalKey: row.canonical_key as string,
|
|
1828
|
-
dedupeKey: row.dedupe_key as string,
|
|
1829
|
-
metadata: row.metadata ? JSON.parse(row.metadata as string) : undefined
|
|
1830
|
-
};
|
|
1831
|
-
|
|
1832
|
-
// Include access tracking fields if present
|
|
1833
|
-
if (row.access_count !== undefined) {
|
|
1834
|
-
event.access_count = row.access_count;
|
|
1835
|
-
}
|
|
1836
|
-
if (row.last_accessed_at !== undefined) {
|
|
1837
|
-
event.last_accessed_at = row.last_accessed_at;
|
|
1838
|
-
}
|
|
1839
|
-
// Include turn_id if present
|
|
1840
|
-
if (row.turn_id !== undefined && row.turn_id !== null) {
|
|
1841
|
-
event.turn_id = row.turn_id;
|
|
1842
|
-
}
|
|
1843
|
-
|
|
1844
|
-
return event;
|
|
1845
|
-
}
|
|
1846
|
-
}
|