@devflow-tools/memory-engine 0.17.8 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/consolidator.d.ts.map +1 -1
- package/dist/consolidator.js +2 -0
- package/dist/consolidator.js.map +1 -1
- package/dist/embedding-model-store.d.ts +1 -4
- package/dist/embedding-model-store.d.ts.map +1 -1
- package/dist/embedding-model-store.js +3 -75
- package/dist/embedding-model-store.js.map +1 -1
- package/dist/embedding-provider.d.ts +3 -2
- package/dist/embedding-provider.d.ts.map +1 -1
- package/dist/embedding-provider.js +151 -43
- package/dist/embedding-provider.js.map +1 -1
- package/dist/hybrid-search.d.ts +5 -1
- package/dist/hybrid-search.d.ts.map +1 -1
- package/dist/hybrid-search.js +92 -56
- package/dist/hybrid-search.js.map +1 -1
- package/dist/index.d.ts +16 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -2
- package/dist/index.js.map +1 -1
- package/dist/lease-store.d.ts.map +1 -1
- package/dist/lease-store.js +30 -0
- package/dist/lease-store.js.map +1 -1
- package/dist/memory-command-service.d.ts +73 -0
- package/dist/memory-command-service.d.ts.map +1 -0
- package/dist/memory-command-service.js +283 -0
- package/dist/memory-command-service.js.map +1 -0
- package/dist/memory-engine.d.ts +19 -15
- package/dist/memory-engine.d.ts.map +1 -1
- package/dist/memory-engine.js +207 -158
- package/dist/memory-engine.js.map +1 -1
- package/dist/memory-episode-repository.d.ts +77 -0
- package/dist/memory-episode-repository.d.ts.map +1 -0
- package/dist/memory-episode-repository.js +262 -0
- package/dist/memory-episode-repository.js.map +1 -0
- package/dist/memory-fact-provider-port.d.ts +83 -0
- package/dist/memory-fact-provider-port.d.ts.map +1 -0
- package/dist/memory-fact-provider-port.js +22 -0
- package/dist/memory-fact-provider-port.js.map +1 -0
- package/dist/memory-fact-validator.d.ts +16 -0
- package/dist/memory-fact-validator.d.ts.map +1 -0
- package/dist/memory-fact-validator.js +52 -0
- package/dist/memory-fact-validator.js.map +1 -0
- package/dist/memory-gate.d.ts +14 -17
- package/dist/memory-gate.d.ts.map +1 -1
- package/dist/memory-gate.js +369 -329
- package/dist/memory-gate.js.map +1 -1
- package/dist/memory-identity-compiler.d.ts +18 -0
- package/dist/memory-identity-compiler.d.ts.map +1 -0
- package/dist/memory-identity-compiler.js +26 -0
- package/dist/memory-identity-compiler.js.map +1 -0
- package/dist/memory-index-projector.d.ts +22 -0
- package/dist/memory-index-projector.d.ts.map +1 -0
- package/dist/memory-index-projector.js +11 -0
- package/dist/memory-index-projector.js.map +1 -0
- package/dist/memory-lifecycle-reducer.d.ts +21 -0
- package/dist/memory-lifecycle-reducer.d.ts.map +1 -0
- package/dist/memory-lifecycle-reducer.js +34 -0
- package/dist/memory-lifecycle-reducer.js.map +1 -0
- package/dist/memory-maintenance.d.ts +1 -4
- package/dist/memory-maintenance.d.ts.map +1 -1
- package/dist/memory-maintenance.js +3 -20
- package/dist/memory-maintenance.js.map +1 -1
- package/dist/memory-query-service.d.ts +24 -0
- package/dist/memory-query-service.d.ts.map +1 -0
- package/dist/memory-query-service.js +180 -0
- package/dist/memory-query-service.js.map +1 -0
- package/dist/memory-relevance.d.ts +16 -1
- package/dist/memory-relevance.d.ts.map +1 -1
- package/dist/memory-relevance.js +83 -139
- package/dist/memory-relevance.js.map +1 -1
- package/dist/memory-store.d.ts +52 -36
- package/dist/memory-store.d.ts.map +1 -1
- package/dist/memory-store.js +406 -704
- package/dist/memory-store.js.map +1 -1
- package/dist/semantic-memory-compiler.d.ts +0 -2
- package/dist/semantic-memory-compiler.d.ts.map +1 -1
- package/dist/semantic-memory-compiler.js +4 -4
- package/dist/semantic-memory-compiler.js.map +1 -1
- package/dist/session-summarizer.d.ts +3 -0
- package/dist/session-summarizer.d.ts.map +1 -1
- package/dist/session-summarizer.js.map +1 -1
- package/dist/signal-analyzer.d.ts +2 -2
- package/dist/signal-analyzer.d.ts.map +1 -1
- package/dist/signal-analyzer.js +10 -16
- package/dist/signal-analyzer.js.map +1 -1
- package/package.json +14 -5
- package/dist/lease-store.test.d.ts +0 -2
- package/dist/lease-store.test.d.ts.map +0 -1
- package/dist/lease-store.test.js +0 -182
- package/dist/lease-store.test.js.map +0 -1
- package/dist/migrations/v10-to-v11.d.ts +0 -3
- package/dist/migrations/v10-to-v11.d.ts.map +0 -1
- package/dist/migrations/v10-to-v11.js +0 -21
- package/dist/migrations/v10-to-v11.js.map +0 -1
- package/dist/migrations/v11-to-v12.d.ts +0 -66
- package/dist/migrations/v11-to-v12.d.ts.map +0 -1
- package/dist/migrations/v11-to-v12.js +0 -577
- package/dist/migrations/v11-to-v12.js.map +0 -1
- package/dist/migrations/v12-to-v13.d.ts +0 -3
- package/dist/migrations/v12-to-v13.d.ts.map +0 -1
- package/dist/migrations/v12-to-v13.js +0 -50
- package/dist/migrations/v12-to-v13.js.map +0 -1
- package/dist/migrations/v2-to-v3.d.ts +0 -3
- package/dist/migrations/v2-to-v3.d.ts.map +0 -1
- package/dist/migrations/v2-to-v3.js +0 -122
- package/dist/migrations/v2-to-v3.js.map +0 -1
- package/dist/migrations/v2-to-v3.test.d.ts +0 -2
- package/dist/migrations/v2-to-v3.test.d.ts.map +0 -1
- package/dist/migrations/v2-to-v3.test.js +0 -88
- package/dist/migrations/v2-to-v3.test.js.map +0 -1
- package/dist/migrations/v3-to-v4.d.ts +0 -3
- package/dist/migrations/v3-to-v4.d.ts.map +0 -1
- package/dist/migrations/v3-to-v4.js +0 -97
- package/dist/migrations/v3-to-v4.js.map +0 -1
- package/dist/migrations/v4-to-v5.d.ts +0 -3
- package/dist/migrations/v4-to-v5.d.ts.map +0 -1
- package/dist/migrations/v4-to-v5.js +0 -34
- package/dist/migrations/v4-to-v5.js.map +0 -1
- package/dist/migrations/v4-to-v5.test.d.ts +0 -2
- package/dist/migrations/v4-to-v5.test.d.ts.map +0 -1
- package/dist/migrations/v4-to-v5.test.js +0 -32
- package/dist/migrations/v4-to-v5.test.js.map +0 -1
- package/dist/migrations/v6-to-v7.d.ts +0 -3
- package/dist/migrations/v6-to-v7.d.ts.map +0 -1
- package/dist/migrations/v6-to-v7.js +0 -18
- package/dist/migrations/v6-to-v7.js.map +0 -1
- package/dist/migrations/v6-to-v7.test.d.ts +0 -2
- package/dist/migrations/v6-to-v7.test.d.ts.map +0 -1
- package/dist/migrations/v6-to-v7.test.js +0 -33
- package/dist/migrations/v6-to-v7.test.js.map +0 -1
- package/dist/migrations/v7-to-v8.d.ts +0 -3
- package/dist/migrations/v7-to-v8.d.ts.map +0 -1
- package/dist/migrations/v7-to-v8.js +0 -20
- package/dist/migrations/v7-to-v8.js.map +0 -1
- package/dist/migrations/v8-to-v9.d.ts +0 -3
- package/dist/migrations/v8-to-v9.d.ts.map +0 -1
- package/dist/migrations/v8-to-v9.js +0 -18
- package/dist/migrations/v8-to-v9.js.map +0 -1
- package/dist/migrations/v9-to-v10.d.ts +0 -3
- package/dist/migrations/v9-to-v10.d.ts.map +0 -1
- package/dist/migrations/v9-to-v10.js +0 -45
- package/dist/migrations/v9-to-v10.js.map +0 -1
- package/dist/migrations/v9-to-v10.test.d.ts +0 -2
- package/dist/migrations/v9-to-v10.test.d.ts.map +0 -1
- package/dist/migrations/v9-to-v10.test.js +0 -40
- package/dist/migrations/v9-to-v10.test.js.map +0 -1
- package/dist/preference-conflicts.test.d.ts +0 -2
- package/dist/preference-conflicts.test.d.ts.map +0 -1
- package/dist/preference-conflicts.test.js +0 -80
- package/dist/preference-conflicts.test.js.map +0 -1
- package/dist/semantic-memory.test.d.ts +0 -2
- package/dist/semantic-memory.test.d.ts.map +0 -1
- package/dist/semantic-memory.test.js +0 -61
- package/dist/semantic-memory.test.js.map +0 -1
package/dist/memory-store.js
CHANGED
|
@@ -3,28 +3,20 @@ import * as sqliteVec from 'sqlite-vec';
|
|
|
3
3
|
import { dirname, join } from 'node:path';
|
|
4
4
|
import { existsSync, mkdirSync } from 'node:fs';
|
|
5
5
|
import { createHash, randomUUID } from 'node:crypto';
|
|
6
|
-
import {
|
|
7
|
-
import { applyV3ToV4Migration } from './migrations/v3-to-v4.js';
|
|
8
|
-
import { applyV4ToV5Migration } from './migrations/v4-to-v5.js';
|
|
9
|
-
import { applyV6ToV7Migration } from './migrations/v6-to-v7.js';
|
|
10
|
-
import { applyV7ToV8Migration } from './migrations/v7-to-v8.js';
|
|
11
|
-
import { applyV8ToV9Migration } from './migrations/v8-to-v9.js';
|
|
12
|
-
import { applyV9ToV10Migration } from './migrations/v9-to-v10.js';
|
|
13
|
-
import { applyV10ToV11Migration } from './migrations/v10-to-v11.js';
|
|
14
|
-
import { applyV11ToV12Migration } from './migrations/v11-to-v12.js';
|
|
15
|
-
import { applyV12ToV13Migration } from './migrations/v12-to-v13.js';
|
|
6
|
+
import { createMiniLmEmbeddingProfile, } from '@devflow-tools/sdk';
|
|
16
7
|
import { observationDedupKey, } from './observation-types.js';
|
|
17
|
-
import {
|
|
18
|
-
import { mergeSemanticEntities, normalizeEntityKey, normalizeSemanticMemory, } from './semantic-memory.js';
|
|
8
|
+
import { mergeSemanticEntities, normalizeEntityKey, } from './semantic-memory.js';
|
|
19
9
|
import { emitSemanticControlLog } from '@devflow-tools/telemetry';
|
|
20
|
-
import { sanitizeDurableMemoryNarrative } from './durable-memory-text.js';
|
|
21
10
|
// ── Schema ─────────────────────────────────────────────
|
|
22
|
-
const
|
|
11
|
+
const MEMORY_SCHEMA = `
|
|
23
12
|
-- ============================================================
|
|
24
13
|
-- Core: memories (expanded from v1)
|
|
25
14
|
-- ============================================================
|
|
26
15
|
CREATE TABLE IF NOT EXISTS memories (
|
|
27
16
|
id TEXT PRIMARY KEY,
|
|
17
|
+
project_id TEXT NOT NULL,
|
|
18
|
+
memory_namespace TEXT NOT NULL,
|
|
19
|
+
actor_id TEXT NOT NULL,
|
|
28
20
|
type TEXT NOT NULL DEFAULT 'fact'
|
|
29
21
|
CHECK(type IN ('constraint','pattern','preference','lesson','fact',
|
|
30
22
|
'semantic','episodic','procedural','failure_lesson',
|
|
@@ -43,7 +35,7 @@ CREATE TABLE IF NOT EXISTS memories (
|
|
|
43
35
|
confidence REAL NOT NULL DEFAULT 0.5,
|
|
44
36
|
tags TEXT NOT NULL DEFAULT '[]',
|
|
45
37
|
scope TEXT NOT NULL DEFAULT '',
|
|
46
|
-
session_id TEXT,
|
|
38
|
+
session_id TEXT NOT NULL,
|
|
47
39
|
created_at INTEGER NOT NULL,
|
|
48
40
|
updated_at INTEGER NOT NULL,
|
|
49
41
|
last_reinforced_at INTEGER,
|
|
@@ -151,17 +143,26 @@ CREATE TABLE IF NOT EXISTS session_events (
|
|
|
151
143
|
dialog_score REAL NOT NULL DEFAULT 0,
|
|
152
144
|
user_message TEXT,
|
|
153
145
|
kind TEXT,
|
|
154
|
-
payload TEXT
|
|
146
|
+
payload TEXT,
|
|
147
|
+
maintenance_status TEXT NOT NULL DEFAULT 'active',
|
|
148
|
+
quarantine_reason TEXT
|
|
155
149
|
);
|
|
156
150
|
CREATE INDEX IF NOT EXISTS idx_session_events_session ON session_events(session_id);
|
|
157
151
|
CREATE INDEX IF NOT EXISTS idx_session_events_project ON session_events(project);
|
|
158
152
|
CREATE INDEX IF NOT EXISTS idx_session_events_processed ON session_events(processed);
|
|
153
|
+
CREATE INDEX IF NOT EXISTS idx_session_events_maintenance
|
|
154
|
+
ON session_events(maintenance_status, processed, created_at);
|
|
155
|
+
CREATE INDEX IF NOT EXISTS idx_session_events_reconciliation
|
|
156
|
+
ON session_events(session_id, processed, maintenance_status, created_at, id);
|
|
159
157
|
|
|
160
158
|
-- ============================================================
|
|
161
159
|
-- Observations (Summarizer output — intermediate layer)
|
|
162
160
|
-- ============================================================
|
|
163
161
|
CREATE TABLE IF NOT EXISTS observations (
|
|
164
162
|
id TEXT PRIMARY KEY,
|
|
163
|
+
project_id TEXT NOT NULL,
|
|
164
|
+
memory_namespace TEXT NOT NULL,
|
|
165
|
+
actor_id TEXT NOT NULL,
|
|
165
166
|
session_id TEXT NOT NULL,
|
|
166
167
|
project TEXT NOT NULL DEFAULT '',
|
|
167
168
|
type TEXT NOT NULL,
|
|
@@ -193,12 +194,35 @@ CREATE TABLE IF NOT EXISTS observations (
|
|
|
193
194
|
entities TEXT NOT NULL DEFAULT '[]',
|
|
194
195
|
temporal TEXT,
|
|
195
196
|
reinforce_count INTEGER NOT NULL DEFAULT 0,
|
|
196
|
-
last_reinforced_at INTEGER
|
|
197
|
+
last_reinforced_at INTEGER,
|
|
198
|
+
quality_status TEXT NOT NULL DEFAULT 'accepted',
|
|
199
|
+
quarantine_reason TEXT,
|
|
200
|
+
quality_details TEXT,
|
|
201
|
+
quarantined_at INTEGER,
|
|
202
|
+
resolution_state TEXT NOT NULL DEFAULT 'current',
|
|
203
|
+
superseded_by_observation TEXT,
|
|
204
|
+
failure_category TEXT,
|
|
205
|
+
capsule TEXT NOT NULL DEFAULT '{}',
|
|
206
|
+
projection TEXT NOT NULL DEFAULT '{}',
|
|
207
|
+
extraction_version TEXT NOT NULL DEFAULT 'canonical.v1'
|
|
197
208
|
);
|
|
198
209
|
CREATE INDEX IF NOT EXISTS idx_observations_session ON observations(session_id);
|
|
199
210
|
CREATE INDEX IF NOT EXISTS idx_observations_project ON observations(project);
|
|
200
211
|
CREATE INDEX IF NOT EXISTS idx_observations_memory ON observations(memory_id);
|
|
201
212
|
CREATE INDEX IF NOT EXISTS idx_observations_type ON observations(type);
|
|
213
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_observations_dedup ON observations(dedup_key);
|
|
214
|
+
CREATE INDEX IF NOT EXISTS idx_observations_quality
|
|
215
|
+
ON observations(quality_status, resolution_state, timestamp);
|
|
216
|
+
CREATE INDEX IF NOT EXISTS idx_observations_failure_category
|
|
217
|
+
ON observations(failure_category, timestamp);
|
|
218
|
+
CREATE INDEX IF NOT EXISTS idx_observations_explicit_acceptance
|
|
219
|
+
ON observations(project, type, source, explicit_accepted_at);
|
|
220
|
+
CREATE INDEX IF NOT EXISTS idx_observations_evidence
|
|
221
|
+
ON observations(project, source, evidence_status, resolution_state);
|
|
222
|
+
CREATE INDEX IF NOT EXISTS idx_observations_semantic_key
|
|
223
|
+
ON observations(project, semantic_key, resolution_state, quality_status);
|
|
224
|
+
CREATE INDEX IF NOT EXISTS idx_observations_facet_key
|
|
225
|
+
ON observations(project, facet_key, resolution_state, quality_status);
|
|
202
226
|
|
|
203
227
|
CREATE TABLE IF NOT EXISTS turn_observation_lineage (
|
|
204
228
|
turn_event_id TEXT NOT NULL,
|
|
@@ -240,6 +264,17 @@ CREATE TABLE IF NOT EXISTS embeddings (
|
|
|
240
264
|
);
|
|
241
265
|
CREATE INDEX IF NOT EXISTS idx_embeddings_source ON embeddings(source_type, source_id);
|
|
242
266
|
|
|
267
|
+
CREATE TABLE IF NOT EXISTS embedding_profiles (
|
|
268
|
+
source_type TEXT NOT NULL CHECK(source_type IN ('memory','observation')),
|
|
269
|
+
source_id TEXT NOT NULL,
|
|
270
|
+
profile_hash TEXT NOT NULL,
|
|
271
|
+
generation INTEGER NOT NULL CHECK(generation > 0),
|
|
272
|
+
source_content_hash TEXT NOT NULL,
|
|
273
|
+
PRIMARY KEY (source_type, source_id)
|
|
274
|
+
);
|
|
275
|
+
CREATE INDEX IF NOT EXISTS idx_embedding_profiles_generation
|
|
276
|
+
ON embedding_profiles(source_type, profile_hash, generation);
|
|
277
|
+
|
|
243
278
|
-- ============================================================
|
|
244
279
|
-- Access log (drives reinforcement boost)
|
|
245
280
|
-- ============================================================
|
|
@@ -248,6 +283,10 @@ CREATE TABLE IF NOT EXISTS access_log (
|
|
|
248
283
|
memory_id TEXT NOT NULL REFERENCES memories(id) ON DELETE CASCADE,
|
|
249
284
|
access_type TEXT NOT NULL CHECK(access_type IN ('search_hit','user_view','agent_use','manual_ref')),
|
|
250
285
|
session_id TEXT,
|
|
286
|
+
query TEXT,
|
|
287
|
+
purpose TEXT,
|
|
288
|
+
rank INTEGER,
|
|
289
|
+
score REAL,
|
|
251
290
|
timestamp INTEGER NOT NULL
|
|
252
291
|
);
|
|
253
292
|
CREATE INDEX IF NOT EXISTS idx_access_log_memory ON access_log(memory_id);
|
|
@@ -282,6 +321,126 @@ CREATE TABLE IF NOT EXISTS signals (
|
|
|
282
321
|
);
|
|
283
322
|
CREATE INDEX IF NOT EXISTS idx_signals_session ON signals(session_id);
|
|
284
323
|
CREATE INDEX IF NOT EXISTS idx_signals_type ON signals(type);
|
|
324
|
+
|
|
325
|
+
CREATE TABLE IF NOT EXISTS distill_leases (
|
|
326
|
+
batch_id TEXT PRIMARY KEY,
|
|
327
|
+
event_ids TEXT NOT NULL,
|
|
328
|
+
session_ids TEXT NOT NULL,
|
|
329
|
+
event_count INTEGER NOT NULL,
|
|
330
|
+
created_at INTEGER NOT NULL,
|
|
331
|
+
expires_at INTEGER NOT NULL,
|
|
332
|
+
status TEXT NOT NULL CHECK(status IN ('pending','committed','expired'))
|
|
333
|
+
);
|
|
334
|
+
CREATE INDEX IF NOT EXISTS idx_leases_expires ON distill_leases(status, expires_at);
|
|
335
|
+
|
|
336
|
+
CREATE TABLE IF NOT EXISTS distill_lease_pages (
|
|
337
|
+
batch_id TEXT NOT NULL,
|
|
338
|
+
page INTEGER NOT NULL,
|
|
339
|
+
total_pages INTEGER NOT NULL,
|
|
340
|
+
read_at INTEGER NOT NULL,
|
|
341
|
+
PRIMARY KEY (batch_id, page),
|
|
342
|
+
FOREIGN KEY (batch_id) REFERENCES distill_leases(batch_id) ON DELETE CASCADE
|
|
343
|
+
);
|
|
344
|
+
CREATE INDEX IF NOT EXISTS idx_distill_pages_batch ON distill_lease_pages(batch_id, page);
|
|
345
|
+
|
|
346
|
+
CREATE TABLE IF NOT EXISTS distill_receipts (
|
|
347
|
+
receipt_id TEXT PRIMARY KEY,
|
|
348
|
+
batch_id TEXT NOT NULL UNIQUE,
|
|
349
|
+
status TEXT NOT NULL CHECK(status IN ('committed', 'committed_empty')),
|
|
350
|
+
observation_count INTEGER NOT NULL,
|
|
351
|
+
event_count INTEGER NOT NULL,
|
|
352
|
+
created_at INTEGER NOT NULL,
|
|
353
|
+
FOREIGN KEY (batch_id) REFERENCES distill_leases(batch_id) ON DELETE CASCADE
|
|
354
|
+
);
|
|
355
|
+
|
|
356
|
+
CREATE TABLE IF NOT EXISTS retrieval_access_log (
|
|
357
|
+
id TEXT PRIMARY KEY,
|
|
358
|
+
source_type TEXT NOT NULL CHECK(source_type IN ('memory','observation')),
|
|
359
|
+
source_id TEXT NOT NULL,
|
|
360
|
+
access_type TEXT NOT NULL CHECK(access_type IN ('selected','consumed','verified','rejected','positive_outcome','negative_outcome','unknown_outcome')),
|
|
361
|
+
session_id TEXT,
|
|
362
|
+
query TEXT,
|
|
363
|
+
purpose TEXT,
|
|
364
|
+
rank INTEGER,
|
|
365
|
+
score REAL,
|
|
366
|
+
timestamp INTEGER NOT NULL,
|
|
367
|
+
project TEXT NOT NULL DEFAULT '',
|
|
368
|
+
request_id TEXT,
|
|
369
|
+
execution_id TEXT,
|
|
370
|
+
workflow_run_id TEXT,
|
|
371
|
+
context_receipt TEXT,
|
|
372
|
+
tool_evidence TEXT,
|
|
373
|
+
outcome_reason TEXT,
|
|
374
|
+
verification_receipt TEXT,
|
|
375
|
+
idempotency_key TEXT
|
|
376
|
+
);
|
|
377
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_access_source
|
|
378
|
+
ON retrieval_access_log(source_type, source_id, timestamp);
|
|
379
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_access_timestamp ON retrieval_access_log(timestamp);
|
|
380
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_access_request
|
|
381
|
+
ON retrieval_access_log(project, request_id, execution_id, timestamp);
|
|
382
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_retrieval_access_idempotency
|
|
383
|
+
ON retrieval_access_log(idempotency_key) WHERE idempotency_key IS NOT NULL;
|
|
384
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_access_outcome
|
|
385
|
+
ON retrieval_access_log(source_type, source_id, access_type, timestamp DESC);
|
|
386
|
+
|
|
387
|
+
CREATE TABLE IF NOT EXISTS maintenance_checkpoints (
|
|
388
|
+
action TEXT PRIMARY KEY,
|
|
389
|
+
cursor TEXT,
|
|
390
|
+
details TEXT NOT NULL DEFAULT '{}',
|
|
391
|
+
updated_at INTEGER NOT NULL
|
|
392
|
+
);
|
|
393
|
+
|
|
394
|
+
CREATE TABLE IF NOT EXISTS memory_maintenance_runs (
|
|
395
|
+
id TEXT PRIMARY KEY,
|
|
396
|
+
mode TEXT NOT NULL CHECK(mode IN ('dry-run','apply')),
|
|
397
|
+
actions TEXT NOT NULL,
|
|
398
|
+
manifest TEXT NOT NULL,
|
|
399
|
+
created_at INTEGER NOT NULL
|
|
400
|
+
);
|
|
401
|
+
CREATE INDEX IF NOT EXISTS idx_memory_maintenance_runs_created
|
|
402
|
+
ON memory_maintenance_runs(created_at);
|
|
403
|
+
|
|
404
|
+
CREATE TABLE IF NOT EXISTS event_processing_audit (
|
|
405
|
+
id TEXT PRIMARY KEY,
|
|
406
|
+
event_id TEXT NOT NULL,
|
|
407
|
+
disposition TEXT NOT NULL,
|
|
408
|
+
reason TEXT NOT NULL,
|
|
409
|
+
run_id TEXT,
|
|
410
|
+
timestamp INTEGER NOT NULL
|
|
411
|
+
);
|
|
412
|
+
CREATE INDEX IF NOT EXISTS idx_event_processing_event
|
|
413
|
+
ON event_processing_audit(event_id, timestamp);
|
|
414
|
+
CREATE INDEX IF NOT EXISTS idx_event_processing_run_disposition
|
|
415
|
+
ON event_processing_audit(run_id, disposition);
|
|
416
|
+
|
|
417
|
+
CREATE TABLE IF NOT EXISTS event_duplicate_quarantine (
|
|
418
|
+
group_key TEXT PRIMARY KEY,
|
|
419
|
+
event_ids TEXT NOT NULL,
|
|
420
|
+
reason TEXT NOT NULL,
|
|
421
|
+
created_at INTEGER NOT NULL,
|
|
422
|
+
resolved_at INTEGER
|
|
423
|
+
);
|
|
424
|
+
|
|
425
|
+
CREATE TABLE IF NOT EXISTS observation_entities (
|
|
426
|
+
observation_id TEXT NOT NULL REFERENCES observations(id) ON DELETE CASCADE,
|
|
427
|
+
project TEXT NOT NULL,
|
|
428
|
+
normalized_entity TEXT NOT NULL,
|
|
429
|
+
display_entity TEXT NOT NULL,
|
|
430
|
+
PRIMARY KEY (observation_id, normalized_entity)
|
|
431
|
+
);
|
|
432
|
+
CREATE INDEX IF NOT EXISTS idx_observation_entities_lookup
|
|
433
|
+
ON observation_entities(project, normalized_entity, observation_id);
|
|
434
|
+
|
|
435
|
+
CREATE TABLE IF NOT EXISTS observation_relations (
|
|
436
|
+
source_observation_id TEXT NOT NULL REFERENCES observations(id) ON DELETE CASCADE,
|
|
437
|
+
target_observation_id TEXT NOT NULL REFERENCES observations(id) ON DELETE CASCADE,
|
|
438
|
+
type TEXT NOT NULL CHECK(type IN ('related_to', 'supersedes', 'references', 'contradicts', 'continues')),
|
|
439
|
+
created_at INTEGER NOT NULL,
|
|
440
|
+
PRIMARY KEY (source_observation_id, target_observation_id, type)
|
|
441
|
+
);
|
|
442
|
+
CREATE INDEX IF NOT EXISTS idx_observation_relations_target
|
|
443
|
+
ON observation_relations(target_observation_id, type);
|
|
285
444
|
`;
|
|
286
445
|
const FTS_TRIGGERS = `
|
|
287
446
|
CREATE TRIGGER IF NOT EXISTS memories_ai AFTER INSERT ON memories BEGIN
|
|
@@ -352,7 +511,6 @@ END;
|
|
|
352
511
|
const MAX_OBSERVATION_FTS_QUERY_LENGTH = 512;
|
|
353
512
|
const MAX_OBSERVATION_FTS_TERMS = 16;
|
|
354
513
|
const MAX_OBSERVATION_FTS_TERM_LENGTH = 64;
|
|
355
|
-
const OBSERVATION_FTS_SCHEMA_VERSION = 6;
|
|
356
514
|
const OBSERVATION_EXCLUSION_CTE = `
|
|
357
515
|
WITH excluded_observation_ids(id) AS (
|
|
358
516
|
SELECT CAST(value AS TEXT) FROM json_each(?)
|
|
@@ -403,30 +561,18 @@ function normalizeObservationLexicalQuery(query) {
|
|
|
403
561
|
: null,
|
|
404
562
|
};
|
|
405
563
|
}
|
|
406
|
-
export function isRecoverableObservationFtsError(error) {
|
|
407
|
-
const candidate = error;
|
|
408
|
-
const code = typeof candidate?.code === 'string' ? candidate.code.toUpperCase() : '';
|
|
409
|
-
const message = typeof candidate?.message === 'string' ? candidate.message : '';
|
|
410
|
-
if (/^SQLITE_(?:BUSY|LOCKED|READONLY|IOERR|FULL|CANTOPEN|PERM)/.test(code))
|
|
411
|
-
return false;
|
|
412
|
-
if (code === 'SQLITE_CORRUPT_VTAB')
|
|
413
|
-
return true;
|
|
414
|
-
const referencesObservationFts = /observations_(?:terms_)?fts(?:_(?:data|idx|content|docsize|config))?/i
|
|
415
|
-
.test(message);
|
|
416
|
-
if (!referencesObservationFts)
|
|
417
|
-
return false;
|
|
418
|
-
if (/^SQLITE_CORRUPT(?:_|$)/.test(code))
|
|
419
|
-
return true;
|
|
420
|
-
return code === 'SQLITE_ERROR' && /no such table|malformed schema|vtable constructor failed/i.test(message);
|
|
421
|
-
}
|
|
422
564
|
// ── MemoryStore ─────────────────────────────────────────
|
|
423
565
|
export class MemoryStore {
|
|
566
|
+
embeddingProfile;
|
|
567
|
+
projectIdentity;
|
|
424
568
|
db;
|
|
425
569
|
dbPath;
|
|
426
570
|
defaultProjectRoot;
|
|
427
571
|
leaseStore = null;
|
|
428
572
|
retrievalRevision = 0;
|
|
429
|
-
constructor(rootPath) {
|
|
573
|
+
constructor(rootPath, embeddingProfile = createMiniLmEmbeddingProfile('memory_retrieval'), projectIdentity) {
|
|
574
|
+
this.embeddingProfile = embeddingProfile;
|
|
575
|
+
this.projectIdentity = projectIdentity;
|
|
430
576
|
this.defaultProjectRoot = rootPath;
|
|
431
577
|
const memDir = join(rootPath, '.devflow', 'memory');
|
|
432
578
|
if (!existsSync(memDir))
|
|
@@ -437,22 +583,10 @@ export class MemoryStore {
|
|
|
437
583
|
this.db.pragma('busy_timeout = 5000');
|
|
438
584
|
this.db.pragma('foreign_keys = ON');
|
|
439
585
|
sqliteVec.load(this.db);
|
|
440
|
-
this.
|
|
586
|
+
this.assertCanonicalSchemaOrEmpty();
|
|
587
|
+
this.db.exec(MEMORY_SCHEMA);
|
|
588
|
+
this.db.exec(OBSERVATION_FTS_SCHEMA);
|
|
441
589
|
this.db.exec(FTS_TRIGGERS);
|
|
442
|
-
this.dropOrphanedObservationFtsTriggers();
|
|
443
|
-
this.migrateV1ToV2();
|
|
444
|
-
applyV2ToV3Migration(this.db);
|
|
445
|
-
applyV3ToV4Migration(this.db);
|
|
446
|
-
applyV4ToV5Migration(this.db);
|
|
447
|
-
this.ensureObservationFts();
|
|
448
|
-
applyV6ToV7Migration(this.db);
|
|
449
|
-
applyV7ToV8Migration(this.db);
|
|
450
|
-
applyV8ToV9Migration(this.db);
|
|
451
|
-
applyV9ToV10Migration(this.db);
|
|
452
|
-
applyV10ToV11Migration(this.db);
|
|
453
|
-
applyV11ToV12Migration(this.db, rootPath);
|
|
454
|
-
applyV12ToV13Migration(this.db);
|
|
455
|
-
this.repairOrphanSessions(rootPath);
|
|
456
590
|
emitSemanticControlLog({
|
|
457
591
|
event: 'memory.engine.transitioned', identity: { projectRoot: this.defaultProjectRoot }, level: 'debug', timestamp: Date.now(),
|
|
458
592
|
data: { stage: 'store_initialized', database: 'memories.db', tableCount: this.countTables() },
|
|
@@ -465,152 +599,42 @@ export class MemoryStore {
|
|
|
465
599
|
const row = this.db.prepare("SELECT COUNT(*) as c FROM sqlite_master WHERE type='table'").get();
|
|
466
600
|
return row.c;
|
|
467
601
|
}
|
|
468
|
-
|
|
469
|
-
const
|
|
470
|
-
if (
|
|
602
|
+
assertCanonicalSchemaOrEmpty() {
|
|
603
|
+
const memories = this.getTableColumns('memories');
|
|
604
|
+
if (memories.length === 0)
|
|
471
605
|
return;
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
if (needsRebuild) {
|
|
494
|
-
this.db.exec(`
|
|
495
|
-
DROP TRIGGER IF EXISTS observations_fts_ai;
|
|
496
|
-
DROP TRIGGER IF EXISTS observations_fts_ad;
|
|
497
|
-
DROP TRIGGER IF EXISTS observations_fts_au;
|
|
498
|
-
`);
|
|
499
|
-
}
|
|
500
|
-
if (forceRebuild) {
|
|
501
|
-
this.db.exec(`
|
|
502
|
-
DROP TABLE IF EXISTS observations_fts;
|
|
503
|
-
DROP TABLE IF EXISTS observations_terms_fts;
|
|
504
|
-
`);
|
|
505
|
-
}
|
|
506
|
-
this.db.exec(OBSERVATION_FTS_SCHEMA);
|
|
507
|
-
if (!needsRebuild)
|
|
508
|
-
return;
|
|
509
|
-
this.db.exec(`
|
|
510
|
-
DELETE FROM observations_fts;
|
|
511
|
-
DELETE FROM observations_terms_fts;
|
|
512
|
-
|
|
513
|
-
INSERT INTO observations_fts(rowid, id, title, narrative, facts, concepts)
|
|
514
|
-
SELECT o.rowid, o.id, o.title, o.narrative, o.facts, o.concepts
|
|
515
|
-
FROM observations o
|
|
516
|
-
WHERE o.quality_status = 'accepted'
|
|
517
|
-
AND o.resolution_state = 'current';
|
|
518
|
-
|
|
519
|
-
INSERT INTO observations_terms_fts(rowid, id, title, narrative, facts, concepts)
|
|
520
|
-
SELECT o.rowid, o.id, o.title, o.narrative, o.facts, o.concepts
|
|
521
|
-
FROM observations o
|
|
522
|
-
WHERE o.quality_status = 'accepted'
|
|
523
|
-
AND o.resolution_state = 'current';
|
|
524
|
-
`);
|
|
525
|
-
this.db.prepare('INSERT OR REPLACE INTO schema_versions(version, applied_at) VALUES (?, ?)').run(OBSERVATION_FTS_SCHEMA_VERSION, Date.now());
|
|
526
|
-
})();
|
|
527
|
-
}
|
|
528
|
-
// ── V1 → V2 Migration ────────────────────────────
|
|
529
|
-
migrateV1ToV2() {
|
|
530
|
-
const tables = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table'").all().map(r => r.name);
|
|
531
|
-
// If old 'events' table exists, migrate data to session_events
|
|
532
|
-
if (tables.includes('events') && tables.includes('session_events')) {
|
|
533
|
-
const count = this.db.prepare('SELECT COUNT(*) as c FROM events').get();
|
|
534
|
-
if (count.c > 0) {
|
|
535
|
-
try {
|
|
536
|
-
this.db.exec(`
|
|
537
|
-
INSERT OR IGNORE INTO session_events
|
|
538
|
-
(id, session_id, project, tool, command, exit_code, stderr,
|
|
539
|
-
duration_ms, created_at, processed, l1_score, dialog_score, user_message)
|
|
540
|
-
SELECT id, session_id, '', tool, command, exit_code, stderr,
|
|
541
|
-
duration_ms, created_at, processed,
|
|
542
|
-
COALESCE(l1_score, 0), COALESCE(dialog_score, 0), user_message
|
|
543
|
-
FROM events
|
|
544
|
-
`);
|
|
545
|
-
emitSemanticControlLog({
|
|
546
|
-
event: 'memory.write.completed', identity: { projectRoot: this.defaultProjectRoot }, level: 'info', timestamp: Date.now(),
|
|
547
|
-
data: { stage: 'event_migration', migrated: count.c },
|
|
548
|
-
});
|
|
549
|
-
}
|
|
550
|
-
catch (err) {
|
|
551
|
-
emitSemanticControlLog({
|
|
552
|
-
event: 'memory.write.degraded', identity: { projectRoot: this.defaultProjectRoot }, level: 'warn', timestamp: Date.now(),
|
|
553
|
-
data: { stage: 'event_migration', reason: memoryStoreErrorReason(err), fallback: 'existing_schema' },
|
|
554
|
-
});
|
|
555
|
-
}
|
|
556
|
-
}
|
|
557
|
-
}
|
|
558
|
-
// Add new columns to memories table if missing
|
|
559
|
-
const memCols = this.getTableColumns('memories');
|
|
560
|
-
const newCols = [
|
|
561
|
-
{ name: 'title', sql: "ALTER TABLE memories ADD COLUMN title TEXT NOT NULL DEFAULT ''" },
|
|
562
|
-
{ name: 'type', sql: "ALTER TABLE memories ADD COLUMN type TEXT" },
|
|
563
|
-
{ name: 'version', sql: 'ALTER TABLE memories ADD COLUMN version INTEGER NOT NULL DEFAULT 1' },
|
|
564
|
-
{ name: 'supersedes', sql: 'ALTER TABLE memories ADD COLUMN supersedes TEXT' },
|
|
565
|
-
{ name: 'superseded_by', sql: 'ALTER TABLE memories ADD COLUMN superseded_by TEXT' },
|
|
566
|
-
{ name: 'tags', sql: "ALTER TABLE memories ADD COLUMN tags TEXT NOT NULL DEFAULT '[]'" },
|
|
567
|
-
{ name: 'context', sql: "ALTER TABLE memories ADD COLUMN context TEXT NOT NULL DEFAULT ''" },
|
|
568
|
-
{ name: 'strength', sql: 'ALTER TABLE memories ADD COLUMN strength REAL NOT NULL DEFAULT 0.5' },
|
|
569
|
-
{ name: 'last_reinforced_at', sql: 'ALTER TABLE memories ADD COLUMN last_reinforced_at INTEGER' },
|
|
570
|
-
{ name: 'reinforce_count', sql: 'ALTER TABLE memories ADD COLUMN reinforce_count INTEGER NOT NULL DEFAULT 0' },
|
|
571
|
-
{ name: 'updated_at', sql: 'ALTER TABLE memories ADD COLUMN updated_at INTEGER NOT NULL DEFAULT 0' },
|
|
606
|
+
const requiredColumns = new Map([
|
|
607
|
+
['memories', [
|
|
608
|
+
'project_id', 'memory_namespace', 'actor_id', 'session_id',
|
|
609
|
+
'type', 'title', 'status', 'updated_at', 'last_accessed_at',
|
|
610
|
+
]],
|
|
611
|
+
['session_events', ['project', 'kind', 'payload', 'maintenance_status', 'quarantine_reason']],
|
|
612
|
+
['observations', [
|
|
613
|
+
'project_id', 'memory_namespace', 'actor_id', 'session_id',
|
|
614
|
+
'source', 'dedup_key', 'quality_status', 'resolution_state', 'evidence_status',
|
|
615
|
+
'semantic_key', 'facet_key', 'entities', 'capsule', 'projection', 'extraction_version',
|
|
616
|
+
]],
|
|
617
|
+
['embedding_profiles', ['profile_hash', 'generation', 'source_content_hash']],
|
|
618
|
+
['retrieval_access_log', [
|
|
619
|
+
'project', 'request_id', 'execution_id', 'context_receipt', 'tool_evidence',
|
|
620
|
+
'verification_receipt', 'idempotency_key',
|
|
621
|
+
]],
|
|
622
|
+
]);
|
|
623
|
+
const requiredTables = [
|
|
624
|
+
'distill_leases', 'distill_lease_pages', 'distill_receipts', 'maintenance_checkpoints',
|
|
625
|
+
'memory_maintenance_runs', 'event_processing_audit', 'event_duplicate_quarantine',
|
|
626
|
+
'observation_entities', 'observation_relations', 'observations_fts', 'observations_terms_fts',
|
|
572
627
|
];
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
}
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
try {
|
|
584
|
-
this.db.exec("UPDATE memories SET type = category WHERE type IS NULL");
|
|
585
|
-
}
|
|
586
|
-
catch { /* ignore */ }
|
|
587
|
-
}
|
|
588
|
-
if (!memCols.includes('title')) {
|
|
589
|
-
try {
|
|
590
|
-
this.db.exec("UPDATE memories SET title = COALESCE(title, '') WHERE title IS NULL OR title = ''");
|
|
591
|
-
}
|
|
592
|
-
catch { /* ignore */ }
|
|
593
|
-
}
|
|
594
|
-
if (!memCols.includes('updated_at')) {
|
|
595
|
-
try {
|
|
596
|
-
this.db.exec('UPDATE memories SET updated_at = created_at WHERE updated_at = 0');
|
|
597
|
-
}
|
|
598
|
-
catch { /* ignore */ }
|
|
599
|
-
}
|
|
600
|
-
// Add kind and payload columns to session_events if missing
|
|
601
|
-
const evtCols = this.getTableColumns('session_events');
|
|
602
|
-
if (!evtCols.includes('kind')) {
|
|
603
|
-
try {
|
|
604
|
-
this.db.exec('ALTER TABLE session_events ADD COLUMN kind TEXT');
|
|
605
|
-
}
|
|
606
|
-
catch { }
|
|
607
|
-
}
|
|
608
|
-
if (!evtCols.includes('payload')) {
|
|
609
|
-
try {
|
|
610
|
-
this.db.exec('ALTER TABLE session_events ADD COLUMN payload TEXT');
|
|
611
|
-
}
|
|
612
|
-
catch { }
|
|
613
|
-
}
|
|
628
|
+
const tables = new Set(this.db.prepare("SELECT name FROM sqlite_master WHERE type IN ('table', 'view')").all().map(row => row.name));
|
|
629
|
+
const invalid = [
|
|
630
|
+
...requiredTables.filter(table => !tables.has(table)).map(table => `table:${table}`),
|
|
631
|
+
...[...requiredColumns].flatMap(([table, columns]) => {
|
|
632
|
+
const actual = new Set(this.getTableColumns(table));
|
|
633
|
+
return columns.filter(column => !actual.has(column)).map(column => `column:${table}.${column}`);
|
|
634
|
+
}),
|
|
635
|
+
];
|
|
636
|
+
if (invalid.length > 0)
|
|
637
|
+
throw new Error(`MEMORY_SCHEMA_UNSUPPORTED:${invalid.join(',')}`);
|
|
614
638
|
}
|
|
615
639
|
getTableColumns(table) {
|
|
616
640
|
try {
|
|
@@ -623,18 +647,17 @@ export class MemoryStore {
|
|
|
623
647
|
}
|
|
624
648
|
// ── Memory CRUD ──────────────────────────────────
|
|
625
649
|
add(entry) {
|
|
626
|
-
|
|
627
|
-
this.ensureSession(entry.sessionId, this.defaultProjectRoot);
|
|
628
|
-
}
|
|
650
|
+
this.ensureSession(entry.sessionId, this.projectIdentity.memoryNamespace);
|
|
629
651
|
const id = entry.id ?? `mem:${Date.now()}:${randomUUID().slice(0, 8)}`;
|
|
630
652
|
const now = Date.now();
|
|
631
|
-
const memType = entry.type ??
|
|
653
|
+
const memType = entry.type ?? 'fact';
|
|
632
654
|
const title = entry.title ?? entry.content.slice(0, 80);
|
|
633
655
|
this.db.prepare(`
|
|
634
|
-
INSERT INTO memories (id,
|
|
656
|
+
INSERT INTO memories (id, project_id, memory_namespace, actor_id,
|
|
657
|
+
type, title, content, original_text, source, status,
|
|
635
658
|
confidence, strength, scope, session_id, created_at, updated_at)
|
|
636
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
637
|
-
`).run(id, memType, title, entry.content, entry.originalText ?? null, entry.source ?? 'auto', entry.status ?? 'active', entry.confidence ?? 0.5, entry.strength ?? 0.5, entry.scope ?? '', entry.sessionId
|
|
659
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
660
|
+
`).run(id, this.projectIdentity.projectId, this.projectIdentity.memoryNamespace, entry.actorId, memType, title, entry.content, entry.originalText ?? null, entry.source ?? 'auto', entry.status ?? 'active', entry.confidence ?? 0.5, entry.strength ?? 0.5, entry.scope ?? '', entry.sessionId, now, now);
|
|
638
661
|
emitSemanticControlLog({
|
|
639
662
|
event: 'memory.write.completed', identity: { projectRoot: this.defaultProjectRoot, sessionId: entry.sessionId }, level: 'info', timestamp: now,
|
|
640
663
|
data: { stage: 'memory_add', memoryId: id, type: memType, status: entry.status ?? 'active' },
|
|
@@ -644,7 +667,10 @@ export class MemoryStore {
|
|
|
644
667
|
get(id) {
|
|
645
668
|
return this.db.prepare('SELECT * FROM memories WHERE id = ?').get(id) ?? null;
|
|
646
669
|
}
|
|
647
|
-
|
|
670
|
+
updateScoped(scope, updates) {
|
|
671
|
+
const invalidatesEmbedding = updates.title !== undefined
|
|
672
|
+
|| updates.content !== undefined
|
|
673
|
+
|| updates.context !== undefined;
|
|
648
674
|
const sets = [];
|
|
649
675
|
const vals = [];
|
|
650
676
|
for (const [key, value] of Object.entries(updates)) {
|
|
@@ -658,20 +684,42 @@ export class MemoryStore {
|
|
|
658
684
|
vals.push(Date.now());
|
|
659
685
|
if (sets.length === 0)
|
|
660
686
|
return;
|
|
661
|
-
vals.push(id);
|
|
662
|
-
this.db.
|
|
687
|
+
vals.push(scope.id, scope.projectId, scope.memoryNamespace, scope.actorId, scope.sessionId);
|
|
688
|
+
this.db.transaction(() => {
|
|
689
|
+
const updated = this.db.prepare(`UPDATE memories SET ${sets.join(', ')}
|
|
690
|
+
WHERE id = ? AND project_id = ? AND memory_namespace = ? AND actor_id = ? AND session_id = ?`).run(...vals);
|
|
691
|
+
if (updated.changes !== 1)
|
|
692
|
+
throw new Error('MEMORY_SCOPE_MISMATCH');
|
|
693
|
+
if (invalidatesEmbedding) {
|
|
694
|
+
this.db.prepare('DELETE FROM memories_vec WHERE id = ?').run(scope.id);
|
|
695
|
+
this.db.prepare("DELETE FROM embeddings WHERE source_type = 'memory' AND source_id = ?").run(scope.id);
|
|
696
|
+
this.db.prepare("DELETE FROM embedding_profiles WHERE source_type = 'memory' AND source_id = ?").run(scope.id);
|
|
697
|
+
}
|
|
698
|
+
})();
|
|
663
699
|
emitSemanticControlLog({
|
|
664
700
|
event: 'memory.write.completed', identity: { projectRoot: this.defaultProjectRoot }, level: 'debug', timestamp: Date.now(),
|
|
665
|
-
data: { stage: 'memory_update', memoryId: id, fields: Object.keys(updates) },
|
|
701
|
+
data: { stage: 'memory_update', memoryId: scope.id, fields: Object.keys(updates) },
|
|
666
702
|
});
|
|
667
703
|
}
|
|
668
|
-
|
|
669
|
-
this.db.
|
|
704
|
+
deleteScoped(scope) {
|
|
705
|
+
this.db.transaction(() => {
|
|
706
|
+
const exists = this.db.prepare(`SELECT id FROM memories
|
|
707
|
+
WHERE id = ? AND project_id = ? AND memory_namespace = ? AND actor_id = ? AND session_id = ?`)
|
|
708
|
+
.get(scope.id, scope.projectId, scope.memoryNamespace, scope.actorId, scope.sessionId);
|
|
709
|
+
if (!exists)
|
|
710
|
+
throw new Error('MEMORY_SCOPE_MISMATCH');
|
|
711
|
+
this.db.prepare('DELETE FROM memories_vec WHERE id = ?').run(scope.id);
|
|
712
|
+
this.db.prepare("DELETE FROM embeddings WHERE source_type = 'memory' AND source_id = ?").run(scope.id);
|
|
713
|
+
this.db.prepare("DELETE FROM embedding_profiles WHERE source_type = 'memory' AND source_id = ?").run(scope.id);
|
|
714
|
+
this.db.prepare(`DELETE FROM memories
|
|
715
|
+
WHERE id = ? AND project_id = ? AND memory_namespace = ? AND actor_id = ? AND session_id = ?`)
|
|
716
|
+
.run(scope.id, scope.projectId, scope.memoryNamespace, scope.actorId, scope.sessionId);
|
|
717
|
+
})();
|
|
670
718
|
}
|
|
671
719
|
list(filter) {
|
|
672
720
|
let sql = 'SELECT * FROM memories WHERE superseded_by IS NULL';
|
|
673
721
|
const vals = [];
|
|
674
|
-
const typeFilter = filter?.type
|
|
722
|
+
const typeFilter = filter?.type;
|
|
675
723
|
if (typeFilter) {
|
|
676
724
|
sql += ' AND type = ?';
|
|
677
725
|
vals.push(typeFilter);
|
|
@@ -703,23 +751,27 @@ export class MemoryStore {
|
|
|
703
751
|
const placeholders = types.map(() => '?').join(',');
|
|
704
752
|
return this.db.prepare(`SELECT * FROM memories WHERE type IN (${placeholders}) AND superseded_by IS NULL AND status IN ('active','approved') ORDER BY updated_at DESC LIMIT ?`).all(...types, limit);
|
|
705
753
|
}
|
|
706
|
-
listByCategory(category, limit = 100) {
|
|
707
|
-
return this.listByType(category, limit);
|
|
708
|
-
}
|
|
709
754
|
countByStatus(status) {
|
|
710
755
|
const row = this.db.prepare('SELECT COUNT(*) as c FROM memories WHERE status = ?').get(status);
|
|
711
756
|
return row?.c ?? 0;
|
|
712
757
|
}
|
|
713
758
|
// ── Embedding ────────────────────────────────────
|
|
714
759
|
insertEmbedding(id, embedding) {
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
this.db.
|
|
760
|
+
this.assertEmbeddingProfile(embedding);
|
|
761
|
+
const buffer = Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength);
|
|
762
|
+
this.db.transaction(() => {
|
|
763
|
+
this.db.prepare('INSERT OR REPLACE INTO memories_vec(id, embedding) VALUES (?, ?)').run(id, buffer);
|
|
764
|
+
this.db.prepare('INSERT OR REPLACE INTO embeddings(source_type, source_id, vector) VALUES (?, ?, ?)')
|
|
765
|
+
.run('memory', id, buffer);
|
|
766
|
+
this.putEmbeddingProfile('memory', id);
|
|
767
|
+
})();
|
|
718
768
|
}
|
|
719
769
|
insertObservationEmbedding(obsId, embedding) {
|
|
770
|
+
this.assertEmbeddingProfile(embedding);
|
|
720
771
|
const buffer = Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength);
|
|
721
772
|
this.db.prepare('INSERT OR REPLACE INTO embeddings(source_type, source_id, vector) VALUES (?, ?, ?)').run('observation', obsId, buffer);
|
|
722
773
|
this.db.prepare('UPDATE observations SET embedding = ? WHERE id = ?').run(buffer, obsId);
|
|
774
|
+
this.putEmbeddingProfile('observation', obsId);
|
|
723
775
|
this.bumpRetrievalRevision();
|
|
724
776
|
}
|
|
725
777
|
insertObservationEmbeddings(entries) {
|
|
@@ -729,9 +781,11 @@ export class MemoryStore {
|
|
|
729
781
|
const update = this.db.prepare('UPDATE observations SET embedding = ? WHERE id = ?');
|
|
730
782
|
this.db.transaction(() => {
|
|
731
783
|
for (const { id, embedding } of entries) {
|
|
784
|
+
this.assertEmbeddingProfile(embedding);
|
|
732
785
|
const buffer = Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength);
|
|
733
786
|
insert.run('observation', id, buffer);
|
|
734
787
|
update.run(buffer, id);
|
|
788
|
+
this.putEmbeddingProfile('observation', id);
|
|
735
789
|
}
|
|
736
790
|
})();
|
|
737
791
|
this.bumpRetrievalRevision();
|
|
@@ -756,12 +810,14 @@ export class MemoryStore {
|
|
|
756
810
|
const rows = this.db.prepare(`${OBSERVATION_EXCLUSION_CTE}
|
|
757
811
|
SELECT o.*, vec_distance_cosine(o.embedding, ?) AS distance
|
|
758
812
|
FROM observations o
|
|
813
|
+
JOIN embedding_profiles p ON p.source_type = 'observation' AND p.source_id = o.id
|
|
759
814
|
WHERE o.embedding IS NOT NULL
|
|
815
|
+
AND p.profile_hash = ? AND p.generation = ?
|
|
760
816
|
AND o.quality_status = 'accepted'
|
|
761
817
|
AND o.resolution_state <> 'superseded'
|
|
762
818
|
AND o.id NOT IN (SELECT id FROM excluded_observation_ids)
|
|
763
819
|
ORDER BY distance LIMIT ?
|
|
764
|
-
`).all(excludedJson, buffer, limit);
|
|
820
|
+
`).all(excludedJson, buffer, this.embeddingProfile.profileHash, this.embeddingProfile.generation, limit);
|
|
765
821
|
return rows.map(row => ({ observation: row, distance: row.distance }));
|
|
766
822
|
}
|
|
767
823
|
// ── Search ───────────────────────────────────────
|
|
@@ -790,27 +846,16 @@ export class MemoryStore {
|
|
|
790
846
|
}
|
|
791
847
|
};
|
|
792
848
|
if (normalizedQuery.trigramQuery) {
|
|
793
|
-
addRanked(this.
|
|
849
|
+
addRanked(this.searchObservationFts('observations_fts', normalizedQuery.trigramQuery, boundedLimit, excludedJson));
|
|
794
850
|
}
|
|
795
851
|
if (normalizedQuery.termQuery) {
|
|
796
|
-
addRanked(this.
|
|
852
|
+
addRanked(this.searchObservationFts('observations_terms_fts', normalizedQuery.termQuery, boundedLimit, excludedJson));
|
|
797
853
|
}
|
|
798
854
|
return [...scores]
|
|
799
855
|
.map(([id, score]) => ({ id, rank: -score }))
|
|
800
856
|
.sort((left, right) => left.rank - right.rank || left.id.localeCompare(right.id))
|
|
801
857
|
.slice(0, boundedLimit);
|
|
802
858
|
}
|
|
803
|
-
searchObservationFtsWithRecovery(table, query, limit, excludedJson) {
|
|
804
|
-
try {
|
|
805
|
-
return this.searchObservationFts(table, query, limit, excludedJson);
|
|
806
|
-
}
|
|
807
|
-
catch (error) {
|
|
808
|
-
if (!isRecoverableObservationFtsError(error))
|
|
809
|
-
throw error;
|
|
810
|
-
this.ensureObservationFts(true);
|
|
811
|
-
return this.searchObservationFts(table, query, limit, excludedJson);
|
|
812
|
-
}
|
|
813
|
-
}
|
|
814
859
|
searchObservationFts(table, query, limit, excludedJson) {
|
|
815
860
|
const sql = table === 'observations_fts' ? `${OBSERVATION_EXCLUSION_CTE}
|
|
816
861
|
SELECT o.id, bm25(observations_fts) AS rank
|
|
@@ -835,14 +880,59 @@ export class MemoryStore {
|
|
|
835
880
|
return this.db.prepare(sql).all(excludedJson, query, limit);
|
|
836
881
|
}
|
|
837
882
|
searchVector(embedding, limit = 20) {
|
|
838
|
-
const buffer = Buffer.from(embedding.buffer);
|
|
883
|
+
const buffer = Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength);
|
|
839
884
|
return this.db.prepare(`
|
|
840
885
|
SELECT v.id, vec_distance_cosine(v.embedding, ?) as distance
|
|
841
886
|
FROM memories_vec v
|
|
842
887
|
JOIN memories m ON m.id = v.id
|
|
888
|
+
JOIN embedding_profiles p ON p.source_type = 'memory' AND p.source_id = v.id
|
|
843
889
|
WHERE m.status IN ('active','approved') AND m.superseded_by IS NULL
|
|
890
|
+
AND p.profile_hash = ? AND p.generation = ?
|
|
844
891
|
ORDER BY distance LIMIT ?
|
|
845
|
-
`).all(buffer, limit);
|
|
892
|
+
`).all(buffer, this.embeddingProfile.profileHash, this.embeddingProfile.generation, limit);
|
|
893
|
+
}
|
|
894
|
+
getEmbeddingProfileState(sourceType) {
|
|
895
|
+
const totalTable = sourceType === 'memory' ? 'memories_vec' : 'observations';
|
|
896
|
+
const totalWhere = sourceType === 'memory' ? '' : ' WHERE embedding IS NOT NULL';
|
|
897
|
+
const total = Number(this.db.prepare(`SELECT COUNT(*) AS count FROM ${totalTable}${totalWhere}`).get().count);
|
|
898
|
+
const indexed = Number(this.db.prepare(`SELECT COUNT(*) AS count FROM embedding_profiles
|
|
899
|
+
WHERE source_type = ? AND profile_hash = ? AND generation = ?`).get(sourceType, this.embeddingProfile.profileHash, this.embeddingProfile.generation).count);
|
|
900
|
+
return {
|
|
901
|
+
status: this.embeddingProfile.dimension !== 384
|
|
902
|
+
? 'dimension_mismatch'
|
|
903
|
+
: indexed < total ? 'backfill_pending' : 'ready',
|
|
904
|
+
profileHash: this.embeddingProfile.profileHash,
|
|
905
|
+
generation: this.embeddingProfile.generation,
|
|
906
|
+
indexed,
|
|
907
|
+
total,
|
|
908
|
+
};
|
|
909
|
+
}
|
|
910
|
+
putEmbeddingProfile(sourceType, sourceId) {
|
|
911
|
+
const sourceContentHash = sourceType === 'memory'
|
|
912
|
+
? this.memoryEmbeddingSourceHash(sourceId)
|
|
913
|
+
: this.observationEmbeddingSourceHash(sourceId);
|
|
914
|
+
this.db.prepare(`INSERT OR REPLACE INTO embedding_profiles
|
|
915
|
+
(source_type, source_id, profile_hash, generation, source_content_hash) VALUES (?, ?, ?, ?, ?)`)
|
|
916
|
+
.run(sourceType, sourceId, this.embeddingProfile.profileHash, this.embeddingProfile.generation, sourceContentHash);
|
|
917
|
+
}
|
|
918
|
+
memoryEmbeddingSourceHash(id) {
|
|
919
|
+
const row = this.db.prepare('SELECT title, content, context FROM memories WHERE id = ?')
|
|
920
|
+
.get(id);
|
|
921
|
+
if (!row)
|
|
922
|
+
throw new Error(`EMBEDDING_SOURCE_MISSING:memory:${id}`);
|
|
923
|
+
return createHash('sha256').update(JSON.stringify(row)).digest('hex');
|
|
924
|
+
}
|
|
925
|
+
observationEmbeddingSourceHash(id) {
|
|
926
|
+
const row = this.db.prepare(`SELECT title, narrative, facts, files, tools, concepts
|
|
927
|
+
FROM observations WHERE id = ?`).get(id);
|
|
928
|
+
if (!row)
|
|
929
|
+
throw new Error(`EMBEDDING_SOURCE_MISSING:observation:${id}`);
|
|
930
|
+
return createHash('sha256').update(JSON.stringify(row)).digest('hex');
|
|
931
|
+
}
|
|
932
|
+
assertEmbeddingProfile(embedding) {
|
|
933
|
+
if (embedding.length !== this.embeddingProfile.dimension || embedding.length !== 384) {
|
|
934
|
+
throw new Error(`EMBEDDING_DIMENSION_MISMATCH:${embedding.length}:${this.embeddingProfile.dimension}`);
|
|
935
|
+
}
|
|
846
936
|
}
|
|
847
937
|
// ── Edges ────────────────────────────────────────
|
|
848
938
|
addEdge(sourceId, targetId, type, weight = 0.5) {
|
|
@@ -1320,49 +1410,6 @@ export class MemoryStore {
|
|
|
1320
1410
|
`).run(sessionId, normalizedProjectRoot, task, Date.now());
|
|
1321
1411
|
return this.getSession(sessionId);
|
|
1322
1412
|
}
|
|
1323
|
-
repairOrphanSessions(projectRoot = this.defaultProjectRoot) {
|
|
1324
|
-
const repair = this.db.transaction(() => {
|
|
1325
|
-
const blankEvents = this.db.prepare(`
|
|
1326
|
-
SELECT id FROM session_events WHERE TRIM(session_id) = '' ORDER BY created_at ASC
|
|
1327
|
-
`).all();
|
|
1328
|
-
const assignRecoveredSession = this.db.prepare('UPDATE session_events SET session_id = ? WHERE id = ? AND TRIM(session_id) = \'\'');
|
|
1329
|
-
for (const event of blankEvents) {
|
|
1330
|
-
assignRecoveredSession.run(`recovered:orphan:${event.id}`, event.id);
|
|
1331
|
-
}
|
|
1332
|
-
const orphanSessions = this.db.prepare(`
|
|
1333
|
-
SELECT
|
|
1334
|
-
e.session_id AS id,
|
|
1335
|
-
COALESCE(NULLIF(MAX(e.project), ''), ?) AS project_root,
|
|
1336
|
-
MIN(e.created_at) AS created_at,
|
|
1337
|
-
MAX(e.created_at) AS closed_at,
|
|
1338
|
-
COUNT(*) AS tool_count,
|
|
1339
|
-
SUM(CASE WHEN e.exit_code IS NOT NULL AND e.exit_code <> 0 THEN 1 ELSE 0 END) AS error_count
|
|
1340
|
-
FROM session_events e
|
|
1341
|
-
LEFT JOIN sessions s ON s.id = e.session_id
|
|
1342
|
-
WHERE s.id IS NULL
|
|
1343
|
-
GROUP BY e.session_id
|
|
1344
|
-
`).all(projectRoot);
|
|
1345
|
-
const insertRecovered = this.db.prepare(`
|
|
1346
|
-
INSERT OR IGNORE INTO sessions
|
|
1347
|
-
(id, project_root, task, status, phase_summary, signal_summary,
|
|
1348
|
-
tool_count, error_count, created_at, closed_at)
|
|
1349
|
-
VALUES (?, ?, 'Recovered orphan session events', 'closed', ?, '[]', ?, ?, ?, ?)
|
|
1350
|
-
`);
|
|
1351
|
-
let repaired = 0;
|
|
1352
|
-
for (const session of orphanSessions) {
|
|
1353
|
-
repaired += insertRecovered.run(session.id, session.project_root || projectRoot, JSON.stringify(['recovered_orphan_events']), session.tool_count, session.error_count, session.created_at, session.closed_at).changes;
|
|
1354
|
-
}
|
|
1355
|
-
return repaired;
|
|
1356
|
-
});
|
|
1357
|
-
const repaired = repair();
|
|
1358
|
-
if (repaired > 0) {
|
|
1359
|
-
emitSemanticControlLog({
|
|
1360
|
-
event: 'memory.engine.transitioned', identity: { projectRoot: this.defaultProjectRoot }, level: 'warn', timestamp: Date.now(),
|
|
1361
|
-
data: { stage: 'orphan_session_repair', recovered: repaired },
|
|
1362
|
-
});
|
|
1363
|
-
}
|
|
1364
|
-
return repaired;
|
|
1365
|
-
}
|
|
1366
1413
|
createSession(id, projectRoot, task = '') {
|
|
1367
1414
|
this.ensureSession(id, projectRoot, task);
|
|
1368
1415
|
emitSemanticControlLog({
|
|
@@ -1413,8 +1460,12 @@ export class MemoryStore {
|
|
|
1413
1460
|
this.ensureSession(obs.sessionId, obs.project ?? this.defaultProjectRoot);
|
|
1414
1461
|
const id = obs.id ?? `obs:${Date.now()}:${randomUUID().slice(0, 8)}`;
|
|
1415
1462
|
const now = obs.timestamp ?? Date.now();
|
|
1416
|
-
|
|
1417
|
-
|
|
1463
|
+
// A semantic projection is descriptive metadata, never an instance
|
|
1464
|
+
// identity. Only the command service may provide an explicit dedupKey
|
|
1465
|
+
// (the receipt-bound fact instance id); legacy callers fall back to the
|
|
1466
|
+
// non-semantic observation key.
|
|
1467
|
+
const dedupKey = obs.dedupKey ?? observationDedupKey(obs.type, obs.title);
|
|
1468
|
+
const duplicate = obs.dedupKey || obs.semantic ? null : this.findObservationByDedupKey(dedupKey);
|
|
1418
1469
|
if (duplicate) {
|
|
1419
1470
|
this.linkObservationToEvents(duplicate.id, obs.sourceEventIds ?? []);
|
|
1420
1471
|
this.incrementObservationAccess(duplicate.id, now);
|
|
@@ -1423,15 +1474,16 @@ export class MemoryStore {
|
|
|
1423
1474
|
}
|
|
1424
1475
|
this.db.prepare(`
|
|
1425
1476
|
INSERT INTO observations
|
|
1426
|
-
(id,
|
|
1477
|
+
(id, project_id, memory_namespace, actor_id, session_id, project,
|
|
1478
|
+
type, title, narrative, facts, files, tools, concepts,
|
|
1427
1479
|
importance, confidence, source, dedup_key, source_event_ids, timestamp, updated_at,
|
|
1428
1480
|
quality_status, quarantine_reason, quality_details, quarantined_at,
|
|
1429
1481
|
failure_category, resolution_state, evidence_status, evidence_details,
|
|
1430
1482
|
semantic_subject, semantic_predicate, semantic_value, semantic_polarity,
|
|
1431
1483
|
semantic_key, facet_key, entities, temporal, reinforce_count, last_reinforced_at,
|
|
1432
1484
|
capsule, projection, extraction_version)
|
|
1433
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1434
|
-
`).run(id, obs.sessionId, obs.project ?? this.
|
|
1485
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1486
|
+
`).run(id, this.projectIdentity.projectId, this.projectIdentity.memoryNamespace, obs.actorId, obs.sessionId, obs.project ?? this.projectIdentity.memoryNamespace, obs.type, obs.title, obs.narrative ?? '', JSON.stringify(obs.facts ?? []), JSON.stringify(obs.files ?? []), JSON.stringify(obs.tools ?? []), JSON.stringify(obs.concepts ?? []), obs.importance ?? 5, obs.confidence ?? 0.5, obs.source ?? 'distill', dedupKey, JSON.stringify(obs.sourceEventIds ?? []), now, now, obs.qualityStatus ?? 'accepted', obs.quarantineReason ?? null, obs.qualityDetails ? JSON.stringify(obs.qualityDetails) : null, obs.qualityStatus === 'quarantined' ? now : null, obs.failureCategory ?? null, obs.resolutionState ?? 'current', obs.evidenceStatus
|
|
1435
1487
|
?? (obs.source === 'explicit_intent'
|
|
1436
1488
|
? 'unverified'
|
|
1437
1489
|
: obs.source === 'manual' || (obs.sourceEventIds?.length ?? 0) > 0
|
|
@@ -1445,14 +1497,14 @@ export class MemoryStore {
|
|
|
1445
1497
|
sourceEvidenceIds: obs.sourceEventIds ?? [],
|
|
1446
1498
|
entities: obs.semantic?.entities ?? [],
|
|
1447
1499
|
temporalStatus: obs.semantic?.temporal?.status ?? 'current',
|
|
1448
|
-
extractionVersion: '
|
|
1500
|
+
extractionVersion: 'memory-capsule.v1',
|
|
1449
1501
|
}), JSON.stringify(obs.projection ?? {
|
|
1450
1502
|
subject: obs.semantic?.subject ?? '',
|
|
1451
1503
|
predicate: obs.semantic?.predicate ?? '',
|
|
1452
1504
|
value: obs.semantic?.value ?? '',
|
|
1453
1505
|
polarity: obs.semantic?.polarity ?? 'neutral',
|
|
1454
1506
|
facet: obs.semantic?.facet ?? 'generic',
|
|
1455
|
-
}), obs.capsule?.extractionVersion ?? '
|
|
1507
|
+
}), obs.capsule?.extractionVersion ?? 'memory-capsule.v1');
|
|
1456
1508
|
// Insert concept junction rows
|
|
1457
1509
|
if (obs.concepts && obs.concepts.length > 0) {
|
|
1458
1510
|
const insertConcept = this.db.prepare('INSERT OR IGNORE INTO observation_concepts (observation_id, concept) VALUES (?, ?)');
|
|
@@ -1471,117 +1523,23 @@ export class MemoryStore {
|
|
|
1471
1523
|
acceptSemanticObservation(obs) {
|
|
1472
1524
|
return this.db.transaction(() => {
|
|
1473
1525
|
const project = obs.project ?? this.defaultProjectRoot;
|
|
1474
|
-
const now = obs.timestamp ?? Date.now();
|
|
1475
|
-
const exact = this.findCurrentObservationBySemanticKey(project, obs.semantic.semanticKey)
|
|
1476
|
-
?? this.findCurrentObservationByNarrative(project, obs.narrative ?? '')
|
|
1477
|
-
?? this.findCurrentObservationByDurableNarrative(project, obs.narrative ?? '');
|
|
1478
|
-
if (exact) {
|
|
1479
|
-
const sourceEventIds = uniqueStrings([
|
|
1480
|
-
...parseStringArray(exact.source_event_ids),
|
|
1481
|
-
...(obs.sourceEventIds ?? []),
|
|
1482
|
-
]);
|
|
1483
|
-
const facts = uniqueStrings([...parseStringArray(exact.facts), ...(obs.facts ?? [])]);
|
|
1484
|
-
const concepts = uniqueStrings([
|
|
1485
|
-
...parseStringArray(exact.concepts),
|
|
1486
|
-
...(obs.concepts ?? []),
|
|
1487
|
-
obs.semantic.subject,
|
|
1488
|
-
obs.semantic.predicate,
|
|
1489
|
-
obs.semantic.value,
|
|
1490
|
-
...obs.semantic.entities.map(normalizeEntityKey),
|
|
1491
|
-
]);
|
|
1492
|
-
const entities = mergeSemanticEntities(parseStringArray(exact.entities), obs.semantic.entities);
|
|
1493
|
-
const narrative = obs.preferIncomingNarrative
|
|
1494
|
-
? obs.narrative ?? exact.narrative
|
|
1495
|
-
: semanticRichness(obs.narrative ?? '') > semanticRichness(exact.narrative)
|
|
1496
|
-
? obs.narrative ?? ''
|
|
1497
|
-
: exact.narrative;
|
|
1498
|
-
const title = obs.preferIncomingNarrative
|
|
1499
|
-
? obs.title
|
|
1500
|
-
: semanticRichness(obs.title) > semanticRichness(exact.title) ? obs.title : exact.title;
|
|
1501
|
-
const contentChanged = exact.title !== title
|
|
1502
|
-
|| exact.narrative !== narrative;
|
|
1503
|
-
this.db.prepare(`
|
|
1504
|
-
UPDATE observations
|
|
1505
|
-
SET type = ?, title = ?, narrative = ?, facts = ?, concepts = ?, importance = MAX(importance, ?),
|
|
1506
|
-
confidence = MAX(confidence, ?), source_event_ids = ?, updated_at = ?,
|
|
1507
|
-
access_count = access_count + 1, reinforce_count = reinforce_count + 1, last_reinforced_at = ?,
|
|
1508
|
-
source = CASE WHEN ? = 'explicit_intent' THEN 'explicit_intent' ELSE source END,
|
|
1509
|
-
quality_status = CASE WHEN quality_status = 'quarantined' AND ? = 'accepted' THEN 'accepted' ELSE quality_status END,
|
|
1510
|
-
quarantine_reason = CASE WHEN ? = 'accepted' THEN NULL ELSE quarantine_reason END,
|
|
1511
|
-
resolution_state = 'current', superseded_by_observation = NULL,
|
|
1512
|
-
semantic_subject = ?, semantic_predicate = ?, semantic_value = ?, semantic_polarity = ?,
|
|
1513
|
-
semantic_key = ?, facet_key = ?, dedup_key = ?, entities = ?,
|
|
1514
|
-
temporal = COALESCE(?, temporal), evidence_status = ?, evidence_details = ?,
|
|
1515
|
-
embedding = CASE WHEN ? = 1 THEN NULL ELSE embedding END
|
|
1516
|
-
WHERE id = ?
|
|
1517
|
-
`).run(obs.type, title, narrative, JSON.stringify(facts), JSON.stringify(concepts), obs.importance ?? 5, obs.confidence ?? 0.5, JSON.stringify(sourceEventIds), now, now, obs.source ?? 'distill', obs.qualityStatus ?? 'accepted', obs.qualityStatus ?? 'accepted', obs.semantic.subject, obs.semantic.predicate, obs.semantic.value, obs.semantic.polarity, obs.semantic.semanticKey, obs.semantic.facetKey, obs.semantic.semanticKey, JSON.stringify(entities), obs.semantic.temporal ? JSON.stringify(obs.semantic.temporal) : null, strongerEvidenceStatus(exact.evidence_status, obs.evidenceStatus), JSON.stringify(mergeEvidenceDetails(exact.evidence_details, obs.evidenceDetails ?? [])), contentChanged ? 1 : 0, exact.id);
|
|
1518
|
-
if (contentChanged) {
|
|
1519
|
-
this.db.prepare("DELETE FROM embeddings WHERE source_type = 'observation' AND source_id = ?").run(exact.id);
|
|
1520
|
-
}
|
|
1521
|
-
this.replaceObservationConcepts(exact.id, concepts);
|
|
1522
|
-
this.replaceObservationEntities(exact.id, project, entities);
|
|
1523
|
-
for (const eventId of obs.sourceEventIds ?? [])
|
|
1524
|
-
this.linkObservationToEvents(exact.id, [eventId]);
|
|
1525
|
-
const supersededIds = this.supersedeSemanticFacetConflicts(project, exact.id, obs.semantic, now);
|
|
1526
|
-
return { row: this.getObservation(exact.id), action: 'REINFORCE', supersededIds, linkedIds: [] };
|
|
1527
|
-
}
|
|
1528
1526
|
const row = this.addObservation(obs);
|
|
1529
1527
|
if ((obs.qualityStatus ?? 'accepted') === 'quarantined') {
|
|
1530
1528
|
return { row, action: 'HOLD', supersededIds: [], linkedIds: [] };
|
|
1531
1529
|
}
|
|
1532
|
-
const
|
|
1533
|
-
const
|
|
1530
|
+
const relatedIds = this.findCurrentObservationIdsByEntities(project, obs.semantic.entities, new Set([row.id])).slice(0, 12);
|
|
1531
|
+
const now = obs.timestamp ?? Date.now();
|
|
1534
1532
|
for (const relatedId of relatedIds) {
|
|
1535
1533
|
this.addObservationRelation(row.id, relatedId, 'related_to', now);
|
|
1536
1534
|
}
|
|
1537
1535
|
return {
|
|
1538
1536
|
row: this.getObservation(row.id),
|
|
1539
|
-
action:
|
|
1540
|
-
supersededIds,
|
|
1537
|
+
action: relatedIds.length > 0 ? 'LINK' : 'ADD',
|
|
1538
|
+
supersededIds: [],
|
|
1541
1539
|
linkedIds: relatedIds,
|
|
1542
1540
|
};
|
|
1543
1541
|
})();
|
|
1544
1542
|
}
|
|
1545
|
-
findCurrentObservationBySemanticKey(project, semanticKey) {
|
|
1546
|
-
return this.db.prepare(`
|
|
1547
|
-
SELECT * FROM observations
|
|
1548
|
-
WHERE project = ? AND semantic_key = ?
|
|
1549
|
-
ORDER BY CASE WHEN resolution_state = 'current' THEN 0 ELSE 1 END,
|
|
1550
|
-
COALESCE(explicit_accepted_at, timestamp) DESC, id DESC
|
|
1551
|
-
LIMIT 1
|
|
1552
|
-
`).get(project, semanticKey) ?? null;
|
|
1553
|
-
}
|
|
1554
|
-
findCurrentObservationByNarrative(project, narrative) {
|
|
1555
|
-
if (!narrative.trim())
|
|
1556
|
-
return null;
|
|
1557
|
-
return this.db.prepare(`
|
|
1558
|
-
SELECT * FROM observations
|
|
1559
|
-
WHERE project = ? AND narrative = ? AND resolution_state = 'current'
|
|
1560
|
-
AND quality_status = 'accepted'
|
|
1561
|
-
ORDER BY COALESCE(explicit_accepted_at, timestamp) DESC, id DESC
|
|
1562
|
-
LIMIT 1
|
|
1563
|
-
`).get(project, narrative) ?? null;
|
|
1564
|
-
}
|
|
1565
|
-
findCurrentObservationByDurableNarrative(project, narrative) {
|
|
1566
|
-
const durableNarrative = sanitizeDurableMemoryNarrative(narrative);
|
|
1567
|
-
if (!durableNarrative)
|
|
1568
|
-
return null;
|
|
1569
|
-
const candidates = this.db.prepare(`
|
|
1570
|
-
SELECT * FROM observations
|
|
1571
|
-
WHERE project = ? AND resolution_state = 'current'
|
|
1572
|
-
AND quality_status = 'accepted'
|
|
1573
|
-
ORDER BY COALESCE(explicit_accepted_at, timestamp) DESC, id DESC
|
|
1574
|
-
`).all(project);
|
|
1575
|
-
return candidates.find(candidate => sanitizeDurableMemoryNarrative(candidate.narrative) === durableNarrative) ?? null;
|
|
1576
|
-
}
|
|
1577
|
-
findCurrentObservationsByFacetKey(project, facetKey) {
|
|
1578
|
-
return this.db.prepare(`
|
|
1579
|
-
SELECT * FROM observations
|
|
1580
|
-
WHERE project = ? AND facet_key = ? AND resolution_state = 'current'
|
|
1581
|
-
AND quality_status = 'accepted'
|
|
1582
|
-
ORDER BY COALESCE(explicit_accepted_at, timestamp) DESC, id DESC
|
|
1583
|
-
`).all(project, facetKey);
|
|
1584
|
-
}
|
|
1585
1543
|
searchObservationEntities(project, entities, limit = 30) {
|
|
1586
1544
|
const normalized = uniqueStrings(entities.map(normalizeEntityKey)).filter(Boolean);
|
|
1587
1545
|
if (normalized.length === 0)
|
|
@@ -1654,22 +1612,6 @@ export class MemoryStore {
|
|
|
1654
1612
|
addSemanticObservationRelation(sourceObservationId, targetObservationId, type, createdAt = Date.now()) {
|
|
1655
1613
|
this.addObservationRelation(sourceObservationId, targetObservationId, type, createdAt);
|
|
1656
1614
|
}
|
|
1657
|
-
supersedeSemanticFacetConflicts(project, winnerId, semantic, now) {
|
|
1658
|
-
const conflicts = this.findCurrentObservationsByFacetKey(project, semantic.facetKey)
|
|
1659
|
-
.filter(candidate => candidate.id !== winnerId)
|
|
1660
|
-
.filter(candidate => semanticObservationsConflict(candidate, semantic));
|
|
1661
|
-
const supersededIds = [];
|
|
1662
|
-
for (const conflict of conflicts) {
|
|
1663
|
-
this.db.prepare(`
|
|
1664
|
-
UPDATE observations
|
|
1665
|
-
SET resolution_state = 'superseded', superseded_by_observation = ?, dedup_key = NULL, updated_at = ?
|
|
1666
|
-
WHERE id = ? AND resolution_state = 'current'
|
|
1667
|
-
`).run(winnerId, now, conflict.id);
|
|
1668
|
-
this.addObservationRelation(winnerId, conflict.id, 'supersedes', now);
|
|
1669
|
-
supersededIds.push(conflict.id);
|
|
1670
|
-
}
|
|
1671
|
-
return supersededIds;
|
|
1672
|
-
}
|
|
1673
1615
|
addObservationWithEmbedding(obs, embedding) {
|
|
1674
1616
|
return this.db.transaction(() => {
|
|
1675
1617
|
const row = this.addObservation(obs);
|
|
@@ -1677,230 +1619,13 @@ export class MemoryStore {
|
|
|
1677
1619
|
return this.getObservation(row.id);
|
|
1678
1620
|
})();
|
|
1679
1621
|
}
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
? normalizeSemanticMemory({
|
|
1688
|
-
project: input.project,
|
|
1689
|
-
type: input.enrichment.type,
|
|
1690
|
-
title: input.enrichment.title,
|
|
1691
|
-
narrative: input.enrichment.narrative,
|
|
1692
|
-
subject: baseSemantic.subject,
|
|
1693
|
-
predicate: baseSemantic.predicate,
|
|
1694
|
-
value: input.content,
|
|
1695
|
-
polarity: 'neutral',
|
|
1696
|
-
entities: baseSemantic.entities,
|
|
1697
|
-
temporal: baseSemantic.temporal ?? undefined,
|
|
1698
|
-
})
|
|
1699
|
-
: baseSemantic;
|
|
1700
|
-
const result = this.acceptSemanticObservation({
|
|
1701
|
-
sessionId: input.sessionId,
|
|
1702
|
-
project: input.project,
|
|
1703
|
-
type: input.enrichment.type,
|
|
1704
|
-
title: input.enrichment.title,
|
|
1705
|
-
narrative: input.enrichment.narrative,
|
|
1706
|
-
facts: input.enrichment.facts,
|
|
1707
|
-
files: input.enrichment.files,
|
|
1708
|
-
tools: input.enrichment.tools,
|
|
1709
|
-
concepts: input.enrichment.concepts,
|
|
1710
|
-
importance: 10,
|
|
1711
|
-
confidence: 0.95,
|
|
1712
|
-
source: 'explicit_intent',
|
|
1713
|
-
sourceEventIds: input.sourceEventId ? [input.sourceEventId] : [],
|
|
1714
|
-
qualityStatus: hold ? 'quarantined' : 'accepted',
|
|
1715
|
-
quarantineReason: hold ? 'ambiguous_explicit_preference' : undefined,
|
|
1716
|
-
evidenceStatus: 'verified',
|
|
1717
|
-
evidenceDetails: [
|
|
1718
|
-
...input.enrichment.evidence.map(item => ({ ...item })),
|
|
1719
|
-
...(input.enrichment.evidence.length === 0 ? [{
|
|
1720
|
-
kind: 'source_event',
|
|
1721
|
-
source: input.sourceEventId ?? `explicit:${input.sessionId}`,
|
|
1722
|
-
summary: 'Observation was accepted from a direct explicit-memory request.',
|
|
1723
|
-
}] : []),
|
|
1724
|
-
],
|
|
1725
|
-
semantic,
|
|
1726
|
-
});
|
|
1727
|
-
const latest = this.db.prepare(`
|
|
1728
|
-
SELECT MAX(COALESCE(explicit_accepted_at, timestamp)) AS latest
|
|
1729
|
-
FROM observations
|
|
1730
|
-
WHERE project = ? AND source = 'explicit_intent'
|
|
1731
|
-
`).get(input.project);
|
|
1732
|
-
const acceptedAt = Math.max(Date.now(), (latest.latest ?? 0) + 1);
|
|
1733
|
-
this.db.prepare(`
|
|
1734
|
-
UPDATE observations
|
|
1735
|
-
SET explicit_accepted_at = MAX(COALESCE(explicit_accepted_at, 0), ?), updated_at = MAX(updated_at, ?)
|
|
1736
|
-
WHERE id = ?
|
|
1737
|
-
`).run(acceptedAt, acceptedAt, result.row.id);
|
|
1738
|
-
if (input.sourceEventId)
|
|
1739
|
-
this.markEventsProcessed([input.sourceEventId]);
|
|
1740
|
-
return this.getObservation(result.row.id);
|
|
1741
|
-
}
|
|
1742
|
-
return this.db.transaction(() => {
|
|
1743
|
-
this.ensureSession(input.sessionId, input.project);
|
|
1744
|
-
const latest = this.db.prepare(`
|
|
1745
|
-
SELECT MAX(COALESCE(explicit_accepted_at, timestamp)) AS latest
|
|
1746
|
-
FROM observations
|
|
1747
|
-
WHERE project = ? AND type IN ('user_preference', 'project_convention', 'decision_made')
|
|
1748
|
-
AND source = 'explicit_intent'
|
|
1749
|
-
`).get(input.project);
|
|
1750
|
-
const acceptedAt = Math.max(Date.now(), (latest.latest ?? 0) + 1);
|
|
1751
|
-
const dedupKey = observationDedupKey('user_preference', input.title);
|
|
1752
|
-
const exact = this.db.prepare('SELECT * FROM observations WHERE dedup_key = ? AND project = ?').get(dedupKey, input.project) ?? null;
|
|
1753
|
-
let row;
|
|
1754
|
-
if (exact) {
|
|
1755
|
-
const contentChanged = exact.title !== input.title || exact.narrative !== input.content;
|
|
1756
|
-
const sourceEventIds = new Set(parseStringArray(exact.source_event_ids));
|
|
1757
|
-
if (input.sourceEventId)
|
|
1758
|
-
sourceEventIds.add(input.sourceEventId);
|
|
1759
|
-
this.db.prepare(`
|
|
1760
|
-
UPDATE observations
|
|
1761
|
-
SET title = ?, narrative = ?, source = 'explicit_intent',
|
|
1762
|
-
confidence = MAX(confidence, 0.95),
|
|
1763
|
-
importance = MAX(importance, 10), quality_status = 'accepted',
|
|
1764
|
-
quarantine_reason = NULL, quality_details = NULL, quarantined_at = NULL,
|
|
1765
|
-
resolution_state = 'current', superseded_by_observation = NULL,
|
|
1766
|
-
source_event_ids = ?, updated_at = ?, explicit_accepted_at = ?,
|
|
1767
|
-
embedding = CASE WHEN ? = 1 THEN NULL ELSE embedding END
|
|
1768
|
-
WHERE id = ?
|
|
1769
|
-
`).run(input.title, input.content, JSON.stringify([...sourceEventIds]), acceptedAt, acceptedAt, contentChanged ? 1 : 0, exact.id);
|
|
1770
|
-
if (contentChanged) {
|
|
1771
|
-
this.db.prepare("DELETE FROM embeddings WHERE source_type = 'observation' AND source_id = ?").run(exact.id);
|
|
1772
|
-
}
|
|
1773
|
-
this.incrementObservationAccess(exact.id, acceptedAt);
|
|
1774
|
-
row = this.getObservation(exact.id);
|
|
1775
|
-
}
|
|
1776
|
-
else {
|
|
1777
|
-
row = this.addObservation({
|
|
1778
|
-
sessionId: input.sessionId,
|
|
1779
|
-
project: input.project,
|
|
1780
|
-
type: 'user_preference',
|
|
1781
|
-
title: input.title,
|
|
1782
|
-
narrative: input.content,
|
|
1783
|
-
importance: 10,
|
|
1784
|
-
confidence: 0.95,
|
|
1785
|
-
source: 'explicit_intent',
|
|
1786
|
-
sourceEventIds: input.sourceEventId ? [input.sourceEventId] : [],
|
|
1787
|
-
timestamp: acceptedAt,
|
|
1788
|
-
qualityStatus: 'accepted',
|
|
1789
|
-
});
|
|
1790
|
-
this.db.prepare('UPDATE observations SET explicit_accepted_at = ? WHERE id = ?').run(acceptedAt, row.id);
|
|
1791
|
-
row = this.getObservation(row.id);
|
|
1792
|
-
}
|
|
1793
|
-
this.supersedeConflictingExplicitPreferences(row, analyzePreference(input.content));
|
|
1794
|
-
if (input.sourceEventId)
|
|
1795
|
-
this.markEventsProcessed([input.sourceEventId]);
|
|
1796
|
-
return this.getObservation(row.id);
|
|
1797
|
-
})();
|
|
1798
|
-
}
|
|
1799
|
-
applyExplicitMemoryEnrichment(observationId, enrichment) {
|
|
1800
|
-
return this.db.transaction(() => {
|
|
1801
|
-
const current = this.getObservation(observationId);
|
|
1802
|
-
if (!current)
|
|
1803
|
-
throw new Error(`Observation ${observationId} not found`);
|
|
1804
|
-
if (current.source !== 'explicit_intent') {
|
|
1805
|
-
throw new Error(`Observation ${observationId} is not an explicit memory`);
|
|
1806
|
-
}
|
|
1807
|
-
const dedupKey = enrichment.semanticKey;
|
|
1808
|
-
const collisions = this.db.prepare(`
|
|
1809
|
-
SELECT id, timestamp, explicit_accepted_at FROM observations
|
|
1810
|
-
WHERE project = ? AND dedup_key = ? AND id <> ?
|
|
1811
|
-
`).all(current.project, dedupKey, observationId);
|
|
1812
|
-
let dedupKeyForCurrent = dedupKey;
|
|
1813
|
-
if (collisions.length > 0) {
|
|
1814
|
-
const candidates = [
|
|
1815
|
-
{
|
|
1816
|
-
id: current.id,
|
|
1817
|
-
timestamp: current.timestamp,
|
|
1818
|
-
explicit_accepted_at: current.explicit_accepted_at,
|
|
1819
|
-
},
|
|
1820
|
-
...collisions,
|
|
1821
|
-
].sort((left, right) => (right.explicit_accepted_at ?? right.timestamp) - (left.explicit_accepted_at ?? left.timestamp)
|
|
1822
|
-
|| right.id.localeCompare(left.id));
|
|
1823
|
-
const winner = candidates[0];
|
|
1824
|
-
const now = Date.now();
|
|
1825
|
-
const supersede = this.db.prepare(`
|
|
1826
|
-
UPDATE observations
|
|
1827
|
-
SET resolution_state = 'superseded', superseded_by_observation = ?,
|
|
1828
|
-
dedup_key = NULL, updated_at = ?
|
|
1829
|
-
WHERE id = ?
|
|
1830
|
-
`);
|
|
1831
|
-
for (const candidate of candidates.slice(1)) {
|
|
1832
|
-
supersede.run(winner.id, now, candidate.id);
|
|
1833
|
-
}
|
|
1834
|
-
this.db.prepare(`
|
|
1835
|
-
UPDATE observations
|
|
1836
|
-
SET resolution_state = 'current', superseded_by_observation = NULL,
|
|
1837
|
-
dedup_key = ?, updated_at = ?
|
|
1838
|
-
WHERE id = ?
|
|
1839
|
-
`).run(dedupKey, now, winner.id);
|
|
1840
|
-
dedupKeyForCurrent = winner.id === current.id ? dedupKey : null;
|
|
1841
|
-
}
|
|
1842
|
-
const nextFacts = JSON.stringify(enrichment.facts);
|
|
1843
|
-
const nextFiles = JSON.stringify(enrichment.files);
|
|
1844
|
-
const nextTools = JSON.stringify(enrichment.tools);
|
|
1845
|
-
const nextConcepts = JSON.stringify(enrichment.concepts);
|
|
1846
|
-
const semanticContentChanged = current.type !== enrichment.type
|
|
1847
|
-
|| current.title !== enrichment.title
|
|
1848
|
-
|| current.narrative !== enrichment.narrative
|
|
1849
|
-
|| current.facts !== nextFacts
|
|
1850
|
-
|| current.concepts !== nextConcepts;
|
|
1851
|
-
const now = Date.now();
|
|
1852
|
-
this.db.prepare(`
|
|
1853
|
-
UPDATE observations
|
|
1854
|
-
SET type = ?, title = ?, narrative = ?, facts = ?, files = ?, tools = ?, concepts = ?,
|
|
1855
|
-
dedup_key = ?, evidence_status = ?, evidence_details = ?, updated_at = ?,
|
|
1856
|
-
embedding = CASE WHEN ? = 1 THEN NULL ELSE embedding END,
|
|
1857
|
-
semantic_subject = ?, semantic_predicate = ?, semantic_value = ?, semantic_polarity = ?,
|
|
1858
|
-
semantic_key = ?, facet_key = ?, entities = ?, temporal = ?
|
|
1859
|
-
WHERE id = ?
|
|
1860
|
-
`).run(enrichment.type, enrichment.title, enrichment.narrative, nextFacts, nextFiles, nextTools, nextConcepts, dedupKeyForCurrent, enrichment.evidenceStatus, JSON.stringify(enrichment.evidence), now, semanticContentChanged ? 1 : 0, enrichment.subject, enrichment.predicate, enrichment.value, enrichment.polarity, enrichment.semanticKey, enrichment.facetKey, JSON.stringify(enrichment.entities), enrichment.temporal ? JSON.stringify(enrichment.temporal) : null, observationId);
|
|
1861
|
-
if (semanticContentChanged) {
|
|
1862
|
-
this.db.prepare("DELETE FROM embeddings WHERE source_type = 'observation' AND source_id = ?").run(observationId);
|
|
1863
|
-
}
|
|
1864
|
-
this.db.prepare('DELETE FROM observation_concepts WHERE observation_id = ?').run(observationId);
|
|
1865
|
-
const insertConcept = this.db.prepare('INSERT OR IGNORE INTO observation_concepts (observation_id, concept) VALUES (?, ?)');
|
|
1866
|
-
for (const concept of enrichment.concepts)
|
|
1867
|
-
insertConcept.run(observationId, concept);
|
|
1868
|
-
this.replaceObservationEntities(observationId, current.project, enrichment.entities);
|
|
1869
|
-
this.reconcileExplicitPreferenceConflicts(current.project);
|
|
1870
|
-
this.bumpRetrievalRevision();
|
|
1871
|
-
return this.getObservation(observationId);
|
|
1872
|
-
})();
|
|
1873
|
-
}
|
|
1874
|
-
reconcileExplicitPreferenceConflicts(project = this.defaultProjectRoot) {
|
|
1875
|
-
return this.db.transaction(() => {
|
|
1876
|
-
const rows = this.listCurrentExplicitPreferences(project).sort((left, right) => this.explicitPreferenceAcceptanceTime(left)
|
|
1877
|
-
- this.explicitPreferenceAcceptanceTime(right)
|
|
1878
|
-
|| left.timestamp - right.timestamp
|
|
1879
|
-
|| left.id.localeCompare(right.id));
|
|
1880
|
-
let superseded = 0;
|
|
1881
|
-
const active = [];
|
|
1882
|
-
for (const row of rows) {
|
|
1883
|
-
const analysis = analyzePreferenceObservation(row);
|
|
1884
|
-
if (analysis.status !== 'high_confidence') {
|
|
1885
|
-
active.push({ row, analysis });
|
|
1886
|
-
continue;
|
|
1887
|
-
}
|
|
1888
|
-
for (let index = active.length - 1; index >= 0; index--) {
|
|
1889
|
-
const previous = active[index];
|
|
1890
|
-
if (!analysesConflict(previous.analysis, analysis))
|
|
1891
|
-
continue;
|
|
1892
|
-
this.db.prepare(`
|
|
1893
|
-
UPDATE observations
|
|
1894
|
-
SET resolution_state = 'superseded', superseded_by_observation = ?
|
|
1895
|
-
WHERE id = ? AND resolution_state = 'current'
|
|
1896
|
-
`).run(row.id, previous.row.id);
|
|
1897
|
-
active.splice(index, 1);
|
|
1898
|
-
superseded += 1;
|
|
1899
|
-
}
|
|
1900
|
-
active.push({ row, analysis });
|
|
1901
|
-
}
|
|
1902
|
-
return superseded;
|
|
1903
|
-
})();
|
|
1622
|
+
markObservationExplicitlyAccepted(observationId, acceptedAt = Date.now()) {
|
|
1623
|
+
this.db.prepare(`
|
|
1624
|
+
UPDATE observations
|
|
1625
|
+
SET explicit_accepted_at = MAX(COALESCE(explicit_accepted_at, 0), ?),
|
|
1626
|
+
updated_at = MAX(updated_at, ?)
|
|
1627
|
+
WHERE id = ? AND source = 'explicit_intent'
|
|
1628
|
+
`).run(acceptedAt, acceptedAt, observationId);
|
|
1904
1629
|
}
|
|
1905
1630
|
listCurrentExplicitPreferences(project = this.defaultProjectRoot) {
|
|
1906
1631
|
return this.db.prepare(`
|
|
@@ -1911,28 +1636,6 @@ export class MemoryStore {
|
|
|
1911
1636
|
ORDER BY COALESCE(explicit_accepted_at, timestamp) DESC, timestamp DESC, id DESC
|
|
1912
1637
|
`).all(project);
|
|
1913
1638
|
}
|
|
1914
|
-
explicitPreferenceAcceptanceTime(row) {
|
|
1915
|
-
return row.explicit_accepted_at ?? row.timestamp;
|
|
1916
|
-
}
|
|
1917
|
-
supersedeConflictingExplicitPreferences(winner, winnerAnalysis) {
|
|
1918
|
-
if (winnerAnalysis.status !== 'high_confidence')
|
|
1919
|
-
return 0;
|
|
1920
|
-
const update = this.db.prepare(`
|
|
1921
|
-
UPDATE observations
|
|
1922
|
-
SET resolution_state = 'superseded', superseded_by_observation = ?
|
|
1923
|
-
WHERE id = ? AND resolution_state = 'current'
|
|
1924
|
-
`);
|
|
1925
|
-
let superseded = 0;
|
|
1926
|
-
for (const candidate of this.listCurrentExplicitPreferences(winner.project)) {
|
|
1927
|
-
if (candidate.id === winner.id)
|
|
1928
|
-
continue;
|
|
1929
|
-
const analysis = analyzePreferenceObservation(candidate);
|
|
1930
|
-
if (!analysesConflict(analysis, winnerAnalysis))
|
|
1931
|
-
continue;
|
|
1932
|
-
superseded += update.run(winner.id, candidate.id).changes;
|
|
1933
|
-
}
|
|
1934
|
-
return superseded;
|
|
1935
|
-
}
|
|
1936
1639
|
getObservation(id) {
|
|
1937
1640
|
return this.db.prepare('SELECT * FROM observations WHERE id = ?').get(id) ?? null;
|
|
1938
1641
|
}
|
|
@@ -1970,13 +1673,14 @@ export class MemoryStore {
|
|
|
1970
1673
|
`).get(type, cutoff, buffer, threshold, buffer) ?? null;
|
|
1971
1674
|
}
|
|
1972
1675
|
incrementObservationAccess(id, now = Date.now()) {
|
|
1973
|
-
this.
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1676
|
+
const updated = this.db.prepare(`
|
|
1677
|
+
UPDATE observations
|
|
1678
|
+
SET access_count = access_count + 1, updated_at = ?
|
|
1679
|
+
WHERE id = ?
|
|
1680
|
+
`).run(now, id);
|
|
1681
|
+
if (updated.changes !== 1)
|
|
1682
|
+
throw new Error(`Observation ${id} not found`);
|
|
1683
|
+
this.bumpRetrievalRevision();
|
|
1980
1684
|
}
|
|
1981
1685
|
promoteObservationToExplicitIntent(id, now = Date.now()) {
|
|
1982
1686
|
this.db.prepare(`
|
|
@@ -2039,21 +1743,31 @@ export class MemoryStore {
|
|
|
2039
1743
|
}
|
|
2040
1744
|
listObservationsMissingEmbedding(limit = 32, afterId) {
|
|
2041
1745
|
return this.db.prepare(`
|
|
2042
|
-
SELECT
|
|
2043
|
-
|
|
2044
|
-
AND
|
|
2045
|
-
|
|
2046
|
-
|
|
1746
|
+
SELECT o.* FROM observations o
|
|
1747
|
+
LEFT JOIN embedding_profiles p ON p.source_type = 'observation' AND p.source_id = o.id
|
|
1748
|
+
AND p.profile_hash = ? AND p.generation = ?
|
|
1749
|
+
WHERE (o.embedding IS NULL OR p.source_id IS NULL) AND o.quality_status = 'accepted'
|
|
1750
|
+
AND o.resolution_state <> 'superseded' AND (? IS NULL OR o.id > ?)
|
|
1751
|
+
ORDER BY o.id LIMIT ?
|
|
1752
|
+
`).all(this.embeddingProfile.profileHash, this.embeddingProfile.generation, afterId ?? null, afterId ?? null, limit);
|
|
2047
1753
|
}
|
|
2048
1754
|
countObservationVectors() {
|
|
2049
1755
|
const row = this.db.prepare(`
|
|
2050
1756
|
SELECT COUNT(*) AS observations,
|
|
2051
1757
|
SUM(CASE WHEN quality_status = 'accepted' AND resolution_state <> 'superseded' THEN 1 ELSE 0 END) AS eligible,
|
|
2052
|
-
SUM(CASE WHEN quality_status = 'accepted' AND resolution_state <> 'superseded'
|
|
2053
|
-
|
|
1758
|
+
SUM(CASE WHEN quality_status = 'accepted' AND resolution_state <> 'superseded'
|
|
1759
|
+
AND embedding IS NOT NULL AND EXISTS (
|
|
1760
|
+
SELECT 1 FROM embedding_profiles p WHERE p.source_type = 'observation'
|
|
1761
|
+
AND p.source_id = observations.id AND p.profile_hash = ? AND p.generation = ?
|
|
1762
|
+
) THEN 1 ELSE 0 END) AS vectors,
|
|
1763
|
+
SUM(CASE WHEN quality_status = 'accepted' AND resolution_state <> 'superseded'
|
|
1764
|
+
AND (embedding IS NULL OR NOT EXISTS (
|
|
1765
|
+
SELECT 1 FROM embedding_profiles p WHERE p.source_type = 'observation'
|
|
1766
|
+
AND p.source_id = observations.id AND p.profile_hash = ? AND p.generation = ?
|
|
1767
|
+
)) THEN 1 ELSE 0 END) AS missing,
|
|
2054
1768
|
SUM(CASE WHEN quality_status = 'quarantined' THEN 1 ELSE 0 END) AS quarantined
|
|
2055
1769
|
FROM observations
|
|
2056
|
-
`).get();
|
|
1770
|
+
`).get(this.embeddingProfile.profileHash, this.embeddingProfile.generation, this.embeddingProfile.profileHash, this.embeddingProfile.generation);
|
|
2057
1771
|
return {
|
|
2058
1772
|
observations: row.observations,
|
|
2059
1773
|
eligible: row.eligible ?? 0,
|
|
@@ -2080,6 +1794,31 @@ export class MemoryStore {
|
|
|
2080
1794
|
WHERE id = ?
|
|
2081
1795
|
`).run(state, supersededBy ?? null, Date.now(), id);
|
|
2082
1796
|
}
|
|
1797
|
+
reinforceObservation(id, input = {}, now = Date.now()) {
|
|
1798
|
+
const current = this.getObservation(id);
|
|
1799
|
+
if (!current || current.resolution_state !== 'current') {
|
|
1800
|
+
throw new Error(`MEMORY_FACT_NOT_CURRENT:${id}`);
|
|
1801
|
+
}
|
|
1802
|
+
const mergedSourceEventIds = uniqueStrings([
|
|
1803
|
+
...parseStringArray(current.source_event_ids),
|
|
1804
|
+
...(input.sourceEventIds ?? []),
|
|
1805
|
+
]);
|
|
1806
|
+
const evidenceStatus = strongerEvidenceStatus(current.evidence_status, input.evidenceStatus);
|
|
1807
|
+
const evidenceDetails = mergeEvidenceDetails(current.evidence_details, input.evidenceDetails ?? []);
|
|
1808
|
+
this.db.prepare(`
|
|
1809
|
+
UPDATE observations
|
|
1810
|
+
SET source_event_ids = ?, reinforce_count = reinforce_count + 1,
|
|
1811
|
+
source = CASE WHEN ? = 'explicit_intent' THEN 'explicit_intent' ELSE source END,
|
|
1812
|
+
importance = MAX(importance, ?), confidence = MAX(confidence, ?),
|
|
1813
|
+
evidence_status = ?, evidence_details = ?,
|
|
1814
|
+
last_reinforced_at = ?, access_count = access_count + 1, updated_at = ?
|
|
1815
|
+
WHERE id = ? AND resolution_state = 'current'
|
|
1816
|
+
`).run(JSON.stringify(mergedSourceEventIds), input.source ?? current.source, input.importance ?? current.importance, input.confidence ?? current.confidence, evidenceStatus, JSON.stringify(evidenceDetails), now, now, id);
|
|
1817
|
+
for (const eventId of input.sourceEventIds ?? [])
|
|
1818
|
+
this.linkObservationToEvents(id, [eventId]);
|
|
1819
|
+
this.bumpRetrievalRevision();
|
|
1820
|
+
return this.getObservation(id);
|
|
1821
|
+
}
|
|
2083
1822
|
getObservationsByConcept(concept, limit = 50) {
|
|
2084
1823
|
return this.listObservations({ concept, limit });
|
|
2085
1824
|
}
|
|
@@ -2168,7 +1907,7 @@ export class MemoryStore {
|
|
|
2168
1907
|
const timestamp = input.timestamp ?? Date.now();
|
|
2169
1908
|
const id = `racc:${timestamp}:${randomUUID().slice(0, 8)}`;
|
|
2170
1909
|
this.db.transaction(() => {
|
|
2171
|
-
const accessType = input.accessType
|
|
1910
|
+
const accessType = input.accessType;
|
|
2172
1911
|
const inserted = this.db.prepare(`
|
|
2173
1912
|
INSERT OR IGNORE INTO retrieval_access_log
|
|
2174
1913
|
(id, source_type, source_id, access_type, project, session_id, request_id,
|
|
@@ -2176,7 +1915,7 @@ export class MemoryStore {
|
|
|
2176
1915
|
context_receipt, tool_evidence, outcome_reason, verification_receipt, idempotency_key)
|
|
2177
1916
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
2178
1917
|
`).run(id, input.sourceType, input.sourceId, accessType, input.project ?? this.defaultProjectRoot, input.sessionId ?? null, input.requestId ?? null, input.executionId ?? null, input.query ?? null, input.purpose ?? null, input.rank ?? null, input.score ?? null, timestamp, input.workflowRunId ?? null, input.contextReceipt ?? null, input.toolEvidence ? JSON.stringify([...new Set(input.toolEvidence)]) : null, input.outcomeReason ?? null, input.verificationReceipt ?? null, input.idempotencyKey ?? null).changes > 0;
|
|
2179
|
-
const reinforces = accessType === '
|
|
1918
|
+
const reinforces = accessType === 'consumed' && inserted;
|
|
2180
1919
|
if (input.sourceType === 'memory' && reinforces) {
|
|
2181
1920
|
const updated = this.db.prepare(`
|
|
2182
1921
|
UPDATE memories SET access_count = access_count + 1,
|
|
@@ -2212,21 +1951,17 @@ export class MemoryStore {
|
|
|
2212
1951
|
`).get(sourceType, sourceId);
|
|
2213
1952
|
return { positive: Number(row.positive ?? 0), negative: Number(row.negative ?? 0) };
|
|
2214
1953
|
}
|
|
2215
|
-
recordAccess(memoryId, accessType, sessionId) {
|
|
2216
|
-
this.recordRetrievalAccess({
|
|
2217
|
-
sourceType: 'memory',
|
|
2218
|
-
sourceId: memoryId,
|
|
2219
|
-
accessType,
|
|
2220
|
-
sessionId,
|
|
2221
|
-
});
|
|
2222
|
-
}
|
|
2223
1954
|
incrementAccess(id) {
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
1955
|
+
const now = Date.now();
|
|
1956
|
+
const updated = this.db.prepare(`
|
|
1957
|
+
UPDATE memories SET access_count = access_count + 1,
|
|
1958
|
+
last_reinforced_at = COALESCE(last_reinforced_at, ?),
|
|
1959
|
+
last_accessed_at = ?, updated_at = ?
|
|
1960
|
+
WHERE id = ?
|
|
1961
|
+
`).run(now, now, now, id);
|
|
1962
|
+
if (updated.changes !== 1)
|
|
1963
|
+
throw new Error(`Memory ${id} not found`);
|
|
1964
|
+
this.bumpRetrievalRevision();
|
|
2230
1965
|
}
|
|
2231
1966
|
listRetrievalAccessLog(limit = 100) {
|
|
2232
1967
|
return this.db.prepare(`
|
|
@@ -2478,26 +2213,6 @@ function uniqueStrings(values) {
|
|
|
2478
2213
|
}
|
|
2479
2214
|
return result;
|
|
2480
2215
|
}
|
|
2481
|
-
function semanticRichness(value) {
|
|
2482
|
-
const exactIdentifiers = value.match(/@[a-z0-9._-]+\/[a-z0-9._-]+|--[a-z0-9-]+|[a-z0-9]+(?:[-./][a-z0-9]+)+/giu)?.length ?? 0;
|
|
2483
|
-
return [...value.normalize('NFKC').trim()].length + exactIdentifiers * 40;
|
|
2484
|
-
}
|
|
2485
|
-
function semanticObservationsConflict(existing, incoming) {
|
|
2486
|
-
if (existing.semantic_key === incoming.semanticKey)
|
|
2487
|
-
return false;
|
|
2488
|
-
if (existing.semantic_polarity === 'neutral' || incoming.polarity === 'neutral')
|
|
2489
|
-
return false;
|
|
2490
|
-
const existingValue = normalizeEntityKey(existing.semantic_value);
|
|
2491
|
-
const incomingValue = normalizeEntityKey(incoming.value);
|
|
2492
|
-
if (existingValue === incomingValue)
|
|
2493
|
-
return existing.semantic_polarity !== incoming.polarity;
|
|
2494
|
-
const subject = normalizeEntityKey(incoming.subject);
|
|
2495
|
-
if (new Set(['package manager', 'test runner', 'build tool']).has(subject))
|
|
2496
|
-
return true;
|
|
2497
|
-
if (incoming.predicate === 'version')
|
|
2498
|
-
return true;
|
|
2499
|
-
return ['use', 'prefer', 'require'].includes(incoming.predicate);
|
|
2500
|
-
}
|
|
2501
2216
|
function strongerEvidenceStatus(existing, incoming) {
|
|
2502
2217
|
return existing === 'verified' || incoming === 'verified' ? 'verified' : 'unverified';
|
|
2503
2218
|
}
|
|
@@ -2521,19 +2236,6 @@ function mergeEvidenceDetails(existing, incoming) {
|
|
|
2521
2236
|
}
|
|
2522
2237
|
return result.slice(0, 64);
|
|
2523
2238
|
}
|
|
2524
|
-
function semanticFromExplicitEnrichment(enrichment) {
|
|
2525
|
-
return {
|
|
2526
|
-
subject: enrichment.subject,
|
|
2527
|
-
predicate: enrichment.predicate,
|
|
2528
|
-
value: enrichment.value,
|
|
2529
|
-
polarity: enrichment.polarity,
|
|
2530
|
-
entities: enrichment.entities,
|
|
2531
|
-
facet: enrichment.facet,
|
|
2532
|
-
temporal: enrichment.temporal,
|
|
2533
|
-
semanticKey: enrichment.semanticKey,
|
|
2534
|
-
facetKey: enrichment.facetKey,
|
|
2535
|
-
};
|
|
2536
|
-
}
|
|
2537
2239
|
function camelToSnake(str) {
|
|
2538
2240
|
return str.replace(/[A-Z]/g, letter => `_${letter.toLowerCase()}`);
|
|
2539
2241
|
}
|