@devflow-tools/memory-engine 0.17.7 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +10 -2
- package/dist/consolidator.d.ts.map +1 -1
- package/dist/consolidator.js +2 -0
- package/dist/consolidator.js.map +1 -1
- package/dist/durable-memory-text.js +1 -1
- package/dist/durable-memory-text.js.map +1 -1
- package/dist/embedding-model-store.d.ts +1 -4
- package/dist/embedding-model-store.d.ts.map +1 -1
- package/dist/embedding-model-store.js +3 -74
- package/dist/embedding-model-store.js.map +1 -1
- package/dist/embedding-provider.d.ts +3 -2
- package/dist/embedding-provider.d.ts.map +1 -1
- package/dist/embedding-provider.js +151 -43
- package/dist/embedding-provider.js.map +1 -1
- package/dist/hybrid-search.d.ts +5 -1
- package/dist/hybrid-search.d.ts.map +1 -1
- package/dist/hybrid-search.js +92 -56
- package/dist/hybrid-search.js.map +1 -1
- package/dist/index.d.ts +16 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -2
- package/dist/index.js.map +1 -1
- package/dist/lease-store.d.ts.map +1 -1
- package/dist/lease-store.js +30 -0
- package/dist/lease-store.js.map +1 -1
- package/dist/memory-command-service.d.ts +73 -0
- package/dist/memory-command-service.d.ts.map +1 -0
- package/dist/memory-command-service.js +283 -0
- package/dist/memory-command-service.js.map +1 -0
- package/dist/memory-engine.d.ts +19 -15
- package/dist/memory-engine.d.ts.map +1 -1
- package/dist/memory-engine.js +207 -158
- package/dist/memory-engine.js.map +1 -1
- package/dist/memory-episode-repository.d.ts +77 -0
- package/dist/memory-episode-repository.d.ts.map +1 -0
- package/dist/memory-episode-repository.js +262 -0
- package/dist/memory-episode-repository.js.map +1 -0
- package/dist/memory-fact-provider-port.d.ts +83 -0
- package/dist/memory-fact-provider-port.d.ts.map +1 -0
- package/dist/memory-fact-provider-port.js +22 -0
- package/dist/memory-fact-provider-port.js.map +1 -0
- package/dist/memory-fact-validator.d.ts +16 -0
- package/dist/memory-fact-validator.d.ts.map +1 -0
- package/dist/memory-fact-validator.js +52 -0
- package/dist/memory-fact-validator.js.map +1 -0
- package/dist/memory-gate.d.ts +14 -17
- package/dist/memory-gate.d.ts.map +1 -1
- package/dist/memory-gate.js +377 -320
- package/dist/memory-gate.js.map +1 -1
- package/dist/memory-identity-compiler.d.ts +18 -0
- package/dist/memory-identity-compiler.d.ts.map +1 -0
- package/dist/memory-identity-compiler.js +26 -0
- package/dist/memory-identity-compiler.js.map +1 -0
- package/dist/memory-index-projector.d.ts +22 -0
- package/dist/memory-index-projector.d.ts.map +1 -0
- package/dist/memory-index-projector.js +11 -0
- package/dist/memory-index-projector.js.map +1 -0
- package/dist/memory-lifecycle-reducer.d.ts +21 -0
- package/dist/memory-lifecycle-reducer.d.ts.map +1 -0
- package/dist/memory-lifecycle-reducer.js +34 -0
- package/dist/memory-lifecycle-reducer.js.map +1 -0
- package/dist/memory-maintenance.d.ts +1 -4
- package/dist/memory-maintenance.d.ts.map +1 -1
- package/dist/memory-maintenance.js +3 -20
- package/dist/memory-maintenance.js.map +1 -1
- package/dist/memory-query-service.d.ts +24 -0
- package/dist/memory-query-service.d.ts.map +1 -0
- package/dist/memory-query-service.js +180 -0
- package/dist/memory-query-service.js.map +1 -0
- package/dist/memory-relevance.d.ts +16 -1
- package/dist/memory-relevance.d.ts.map +1 -1
- package/dist/memory-relevance.js +83 -139
- package/dist/memory-relevance.js.map +1 -1
- package/dist/memory-store.d.ts +52 -35
- package/dist/memory-store.d.ts.map +1 -1
- package/dist/memory-store.js +406 -690
- package/dist/memory-store.js.map +1 -1
- package/dist/semantic-memory-compiler.d.ts +0 -2
- package/dist/semantic-memory-compiler.d.ts.map +1 -1
- package/dist/semantic-memory-compiler.js +4 -4
- package/dist/semantic-memory-compiler.js.map +1 -1
- package/dist/session-summarizer.d.ts +3 -0
- package/dist/session-summarizer.d.ts.map +1 -1
- package/dist/session-summarizer.js.map +1 -1
- package/dist/signal-analyzer.d.ts +2 -2
- package/dist/signal-analyzer.d.ts.map +1 -1
- package/dist/signal-analyzer.js +10 -16
- package/dist/signal-analyzer.js.map +1 -1
- package/package.json +14 -5
- package/dist/lease-store.test.d.ts +0 -2
- package/dist/lease-store.test.d.ts.map +0 -1
- package/dist/lease-store.test.js +0 -182
- package/dist/lease-store.test.js.map +0 -1
- package/dist/migrations/v10-to-v11.d.ts +0 -3
- package/dist/migrations/v10-to-v11.d.ts.map +0 -1
- package/dist/migrations/v10-to-v11.js +0 -21
- package/dist/migrations/v10-to-v11.js.map +0 -1
- package/dist/migrations/v11-to-v12.d.ts +0 -66
- package/dist/migrations/v11-to-v12.d.ts.map +0 -1
- package/dist/migrations/v11-to-v12.js +0 -577
- package/dist/migrations/v11-to-v12.js.map +0 -1
- package/dist/migrations/v12-to-v13.d.ts +0 -3
- package/dist/migrations/v12-to-v13.d.ts.map +0 -1
- package/dist/migrations/v12-to-v13.js +0 -50
- package/dist/migrations/v12-to-v13.js.map +0 -1
- package/dist/migrations/v2-to-v3.d.ts +0 -3
- package/dist/migrations/v2-to-v3.d.ts.map +0 -1
- package/dist/migrations/v2-to-v3.js +0 -122
- package/dist/migrations/v2-to-v3.js.map +0 -1
- package/dist/migrations/v2-to-v3.test.d.ts +0 -2
- package/dist/migrations/v2-to-v3.test.d.ts.map +0 -1
- package/dist/migrations/v2-to-v3.test.js +0 -88
- package/dist/migrations/v2-to-v3.test.js.map +0 -1
- package/dist/migrations/v3-to-v4.d.ts +0 -3
- package/dist/migrations/v3-to-v4.d.ts.map +0 -1
- package/dist/migrations/v3-to-v4.js +0 -97
- package/dist/migrations/v3-to-v4.js.map +0 -1
- package/dist/migrations/v4-to-v5.d.ts +0 -3
- package/dist/migrations/v4-to-v5.d.ts.map +0 -1
- package/dist/migrations/v4-to-v5.js +0 -34
- package/dist/migrations/v4-to-v5.js.map +0 -1
- package/dist/migrations/v4-to-v5.test.d.ts +0 -2
- package/dist/migrations/v4-to-v5.test.d.ts.map +0 -1
- package/dist/migrations/v4-to-v5.test.js +0 -32
- package/dist/migrations/v4-to-v5.test.js.map +0 -1
- package/dist/migrations/v6-to-v7.d.ts +0 -3
- package/dist/migrations/v6-to-v7.d.ts.map +0 -1
- package/dist/migrations/v6-to-v7.js +0 -18
- package/dist/migrations/v6-to-v7.js.map +0 -1
- package/dist/migrations/v6-to-v7.test.d.ts +0 -2
- package/dist/migrations/v6-to-v7.test.d.ts.map +0 -1
- package/dist/migrations/v6-to-v7.test.js +0 -33
- package/dist/migrations/v6-to-v7.test.js.map +0 -1
- package/dist/migrations/v7-to-v8.d.ts +0 -3
- package/dist/migrations/v7-to-v8.d.ts.map +0 -1
- package/dist/migrations/v7-to-v8.js +0 -20
- package/dist/migrations/v7-to-v8.js.map +0 -1
- package/dist/migrations/v8-to-v9.d.ts +0 -3
- package/dist/migrations/v8-to-v9.d.ts.map +0 -1
- package/dist/migrations/v8-to-v9.js +0 -18
- package/dist/migrations/v8-to-v9.js.map +0 -1
- package/dist/migrations/v9-to-v10.d.ts +0 -3
- package/dist/migrations/v9-to-v10.d.ts.map +0 -1
- package/dist/migrations/v9-to-v10.js +0 -45
- package/dist/migrations/v9-to-v10.js.map +0 -1
- package/dist/migrations/v9-to-v10.test.d.ts +0 -2
- package/dist/migrations/v9-to-v10.test.d.ts.map +0 -1
- package/dist/migrations/v9-to-v10.test.js +0 -40
- package/dist/migrations/v9-to-v10.test.js.map +0 -1
- package/dist/preference-conflicts.test.d.ts +0 -2
- package/dist/preference-conflicts.test.d.ts.map +0 -1
- package/dist/preference-conflicts.test.js +0 -80
- package/dist/preference-conflicts.test.js.map +0 -1
- package/dist/semantic-memory.test.d.ts +0 -2
- package/dist/semantic-memory.test.d.ts.map +0 -1
- package/dist/semantic-memory.test.js +0 -61
- package/dist/semantic-memory.test.js.map +0 -1
package/dist/memory-store.js
CHANGED
|
@@ -3,27 +3,20 @@ import * as sqliteVec from 'sqlite-vec';
|
|
|
3
3
|
import { dirname, join } from 'node:path';
|
|
4
4
|
import { existsSync, mkdirSync } from 'node:fs';
|
|
5
5
|
import { createHash, randomUUID } from 'node:crypto';
|
|
6
|
-
import {
|
|
7
|
-
import { applyV3ToV4Migration } from './migrations/v3-to-v4.js';
|
|
8
|
-
import { applyV4ToV5Migration } from './migrations/v4-to-v5.js';
|
|
9
|
-
import { applyV6ToV7Migration } from './migrations/v6-to-v7.js';
|
|
10
|
-
import { applyV7ToV8Migration } from './migrations/v7-to-v8.js';
|
|
11
|
-
import { applyV8ToV9Migration } from './migrations/v8-to-v9.js';
|
|
12
|
-
import { applyV9ToV10Migration } from './migrations/v9-to-v10.js';
|
|
13
|
-
import { applyV10ToV11Migration } from './migrations/v10-to-v11.js';
|
|
14
|
-
import { applyV11ToV12Migration } from './migrations/v11-to-v12.js';
|
|
15
|
-
import { applyV12ToV13Migration } from './migrations/v12-to-v13.js';
|
|
6
|
+
import { createMiniLmEmbeddingProfile, } from '@devflow-tools/sdk';
|
|
16
7
|
import { observationDedupKey, } from './observation-types.js';
|
|
17
|
-
import {
|
|
18
|
-
import { mergeSemanticEntities, normalizeEntityKey, normalizeSemanticMemory, } from './semantic-memory.js';
|
|
8
|
+
import { mergeSemanticEntities, normalizeEntityKey, } from './semantic-memory.js';
|
|
19
9
|
import { emitSemanticControlLog } from '@devflow-tools/telemetry';
|
|
20
10
|
// ── Schema ─────────────────────────────────────────────
|
|
21
|
-
const
|
|
11
|
+
const MEMORY_SCHEMA = `
|
|
22
12
|
-- ============================================================
|
|
23
13
|
-- Core: memories (expanded from v1)
|
|
24
14
|
-- ============================================================
|
|
25
15
|
CREATE TABLE IF NOT EXISTS memories (
|
|
26
16
|
id TEXT PRIMARY KEY,
|
|
17
|
+
project_id TEXT NOT NULL,
|
|
18
|
+
memory_namespace TEXT NOT NULL,
|
|
19
|
+
actor_id TEXT NOT NULL,
|
|
27
20
|
type TEXT NOT NULL DEFAULT 'fact'
|
|
28
21
|
CHECK(type IN ('constraint','pattern','preference','lesson','fact',
|
|
29
22
|
'semantic','episodic','procedural','failure_lesson',
|
|
@@ -42,7 +35,7 @@ CREATE TABLE IF NOT EXISTS memories (
|
|
|
42
35
|
confidence REAL NOT NULL DEFAULT 0.5,
|
|
43
36
|
tags TEXT NOT NULL DEFAULT '[]',
|
|
44
37
|
scope TEXT NOT NULL DEFAULT '',
|
|
45
|
-
session_id TEXT,
|
|
38
|
+
session_id TEXT NOT NULL,
|
|
46
39
|
created_at INTEGER NOT NULL,
|
|
47
40
|
updated_at INTEGER NOT NULL,
|
|
48
41
|
last_reinforced_at INTEGER,
|
|
@@ -150,17 +143,26 @@ CREATE TABLE IF NOT EXISTS session_events (
|
|
|
150
143
|
dialog_score REAL NOT NULL DEFAULT 0,
|
|
151
144
|
user_message TEXT,
|
|
152
145
|
kind TEXT,
|
|
153
|
-
payload TEXT
|
|
146
|
+
payload TEXT,
|
|
147
|
+
maintenance_status TEXT NOT NULL DEFAULT 'active',
|
|
148
|
+
quarantine_reason TEXT
|
|
154
149
|
);
|
|
155
150
|
CREATE INDEX IF NOT EXISTS idx_session_events_session ON session_events(session_id);
|
|
156
151
|
CREATE INDEX IF NOT EXISTS idx_session_events_project ON session_events(project);
|
|
157
152
|
CREATE INDEX IF NOT EXISTS idx_session_events_processed ON session_events(processed);
|
|
153
|
+
CREATE INDEX IF NOT EXISTS idx_session_events_maintenance
|
|
154
|
+
ON session_events(maintenance_status, processed, created_at);
|
|
155
|
+
CREATE INDEX IF NOT EXISTS idx_session_events_reconciliation
|
|
156
|
+
ON session_events(session_id, processed, maintenance_status, created_at, id);
|
|
158
157
|
|
|
159
158
|
-- ============================================================
|
|
160
159
|
-- Observations (Summarizer output — intermediate layer)
|
|
161
160
|
-- ============================================================
|
|
162
161
|
CREATE TABLE IF NOT EXISTS observations (
|
|
163
162
|
id TEXT PRIMARY KEY,
|
|
163
|
+
project_id TEXT NOT NULL,
|
|
164
|
+
memory_namespace TEXT NOT NULL,
|
|
165
|
+
actor_id TEXT NOT NULL,
|
|
164
166
|
session_id TEXT NOT NULL,
|
|
165
167
|
project TEXT NOT NULL DEFAULT '',
|
|
166
168
|
type TEXT NOT NULL,
|
|
@@ -192,12 +194,35 @@ CREATE TABLE IF NOT EXISTS observations (
|
|
|
192
194
|
entities TEXT NOT NULL DEFAULT '[]',
|
|
193
195
|
temporal TEXT,
|
|
194
196
|
reinforce_count INTEGER NOT NULL DEFAULT 0,
|
|
195
|
-
last_reinforced_at INTEGER
|
|
197
|
+
last_reinforced_at INTEGER,
|
|
198
|
+
quality_status TEXT NOT NULL DEFAULT 'accepted',
|
|
199
|
+
quarantine_reason TEXT,
|
|
200
|
+
quality_details TEXT,
|
|
201
|
+
quarantined_at INTEGER,
|
|
202
|
+
resolution_state TEXT NOT NULL DEFAULT 'current',
|
|
203
|
+
superseded_by_observation TEXT,
|
|
204
|
+
failure_category TEXT,
|
|
205
|
+
capsule TEXT NOT NULL DEFAULT '{}',
|
|
206
|
+
projection TEXT NOT NULL DEFAULT '{}',
|
|
207
|
+
extraction_version TEXT NOT NULL DEFAULT 'canonical.v1'
|
|
196
208
|
);
|
|
197
209
|
CREATE INDEX IF NOT EXISTS idx_observations_session ON observations(session_id);
|
|
198
210
|
CREATE INDEX IF NOT EXISTS idx_observations_project ON observations(project);
|
|
199
211
|
CREATE INDEX IF NOT EXISTS idx_observations_memory ON observations(memory_id);
|
|
200
212
|
CREATE INDEX IF NOT EXISTS idx_observations_type ON observations(type);
|
|
213
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_observations_dedup ON observations(dedup_key);
|
|
214
|
+
CREATE INDEX IF NOT EXISTS idx_observations_quality
|
|
215
|
+
ON observations(quality_status, resolution_state, timestamp);
|
|
216
|
+
CREATE INDEX IF NOT EXISTS idx_observations_failure_category
|
|
217
|
+
ON observations(failure_category, timestamp);
|
|
218
|
+
CREATE INDEX IF NOT EXISTS idx_observations_explicit_acceptance
|
|
219
|
+
ON observations(project, type, source, explicit_accepted_at);
|
|
220
|
+
CREATE INDEX IF NOT EXISTS idx_observations_evidence
|
|
221
|
+
ON observations(project, source, evidence_status, resolution_state);
|
|
222
|
+
CREATE INDEX IF NOT EXISTS idx_observations_semantic_key
|
|
223
|
+
ON observations(project, semantic_key, resolution_state, quality_status);
|
|
224
|
+
CREATE INDEX IF NOT EXISTS idx_observations_facet_key
|
|
225
|
+
ON observations(project, facet_key, resolution_state, quality_status);
|
|
201
226
|
|
|
202
227
|
CREATE TABLE IF NOT EXISTS turn_observation_lineage (
|
|
203
228
|
turn_event_id TEXT NOT NULL,
|
|
@@ -239,6 +264,17 @@ CREATE TABLE IF NOT EXISTS embeddings (
|
|
|
239
264
|
);
|
|
240
265
|
CREATE INDEX IF NOT EXISTS idx_embeddings_source ON embeddings(source_type, source_id);
|
|
241
266
|
|
|
267
|
+
CREATE TABLE IF NOT EXISTS embedding_profiles (
|
|
268
|
+
source_type TEXT NOT NULL CHECK(source_type IN ('memory','observation')),
|
|
269
|
+
source_id TEXT NOT NULL,
|
|
270
|
+
profile_hash TEXT NOT NULL,
|
|
271
|
+
generation INTEGER NOT NULL CHECK(generation > 0),
|
|
272
|
+
source_content_hash TEXT NOT NULL,
|
|
273
|
+
PRIMARY KEY (source_type, source_id)
|
|
274
|
+
);
|
|
275
|
+
CREATE INDEX IF NOT EXISTS idx_embedding_profiles_generation
|
|
276
|
+
ON embedding_profiles(source_type, profile_hash, generation);
|
|
277
|
+
|
|
242
278
|
-- ============================================================
|
|
243
279
|
-- Access log (drives reinforcement boost)
|
|
244
280
|
-- ============================================================
|
|
@@ -247,6 +283,10 @@ CREATE TABLE IF NOT EXISTS access_log (
|
|
|
247
283
|
memory_id TEXT NOT NULL REFERENCES memories(id) ON DELETE CASCADE,
|
|
248
284
|
access_type TEXT NOT NULL CHECK(access_type IN ('search_hit','user_view','agent_use','manual_ref')),
|
|
249
285
|
session_id TEXT,
|
|
286
|
+
query TEXT,
|
|
287
|
+
purpose TEXT,
|
|
288
|
+
rank INTEGER,
|
|
289
|
+
score REAL,
|
|
250
290
|
timestamp INTEGER NOT NULL
|
|
251
291
|
);
|
|
252
292
|
CREATE INDEX IF NOT EXISTS idx_access_log_memory ON access_log(memory_id);
|
|
@@ -281,6 +321,126 @@ CREATE TABLE IF NOT EXISTS signals (
|
|
|
281
321
|
);
|
|
282
322
|
CREATE INDEX IF NOT EXISTS idx_signals_session ON signals(session_id);
|
|
283
323
|
CREATE INDEX IF NOT EXISTS idx_signals_type ON signals(type);
|
|
324
|
+
|
|
325
|
+
CREATE TABLE IF NOT EXISTS distill_leases (
|
|
326
|
+
batch_id TEXT PRIMARY KEY,
|
|
327
|
+
event_ids TEXT NOT NULL,
|
|
328
|
+
session_ids TEXT NOT NULL,
|
|
329
|
+
event_count INTEGER NOT NULL,
|
|
330
|
+
created_at INTEGER NOT NULL,
|
|
331
|
+
expires_at INTEGER NOT NULL,
|
|
332
|
+
status TEXT NOT NULL CHECK(status IN ('pending','committed','expired'))
|
|
333
|
+
);
|
|
334
|
+
CREATE INDEX IF NOT EXISTS idx_leases_expires ON distill_leases(status, expires_at);
|
|
335
|
+
|
|
336
|
+
CREATE TABLE IF NOT EXISTS distill_lease_pages (
|
|
337
|
+
batch_id TEXT NOT NULL,
|
|
338
|
+
page INTEGER NOT NULL,
|
|
339
|
+
total_pages INTEGER NOT NULL,
|
|
340
|
+
read_at INTEGER NOT NULL,
|
|
341
|
+
PRIMARY KEY (batch_id, page),
|
|
342
|
+
FOREIGN KEY (batch_id) REFERENCES distill_leases(batch_id) ON DELETE CASCADE
|
|
343
|
+
);
|
|
344
|
+
CREATE INDEX IF NOT EXISTS idx_distill_pages_batch ON distill_lease_pages(batch_id, page);
|
|
345
|
+
|
|
346
|
+
CREATE TABLE IF NOT EXISTS distill_receipts (
|
|
347
|
+
receipt_id TEXT PRIMARY KEY,
|
|
348
|
+
batch_id TEXT NOT NULL UNIQUE,
|
|
349
|
+
status TEXT NOT NULL CHECK(status IN ('committed', 'committed_empty')),
|
|
350
|
+
observation_count INTEGER NOT NULL,
|
|
351
|
+
event_count INTEGER NOT NULL,
|
|
352
|
+
created_at INTEGER NOT NULL,
|
|
353
|
+
FOREIGN KEY (batch_id) REFERENCES distill_leases(batch_id) ON DELETE CASCADE
|
|
354
|
+
);
|
|
355
|
+
|
|
356
|
+
CREATE TABLE IF NOT EXISTS retrieval_access_log (
|
|
357
|
+
id TEXT PRIMARY KEY,
|
|
358
|
+
source_type TEXT NOT NULL CHECK(source_type IN ('memory','observation')),
|
|
359
|
+
source_id TEXT NOT NULL,
|
|
360
|
+
access_type TEXT NOT NULL CHECK(access_type IN ('selected','consumed','verified','rejected','positive_outcome','negative_outcome','unknown_outcome')),
|
|
361
|
+
session_id TEXT,
|
|
362
|
+
query TEXT,
|
|
363
|
+
purpose TEXT,
|
|
364
|
+
rank INTEGER,
|
|
365
|
+
score REAL,
|
|
366
|
+
timestamp INTEGER NOT NULL,
|
|
367
|
+
project TEXT NOT NULL DEFAULT '',
|
|
368
|
+
request_id TEXT,
|
|
369
|
+
execution_id TEXT,
|
|
370
|
+
workflow_run_id TEXT,
|
|
371
|
+
context_receipt TEXT,
|
|
372
|
+
tool_evidence TEXT,
|
|
373
|
+
outcome_reason TEXT,
|
|
374
|
+
verification_receipt TEXT,
|
|
375
|
+
idempotency_key TEXT
|
|
376
|
+
);
|
|
377
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_access_source
|
|
378
|
+
ON retrieval_access_log(source_type, source_id, timestamp);
|
|
379
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_access_timestamp ON retrieval_access_log(timestamp);
|
|
380
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_access_request
|
|
381
|
+
ON retrieval_access_log(project, request_id, execution_id, timestamp);
|
|
382
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_retrieval_access_idempotency
|
|
383
|
+
ON retrieval_access_log(idempotency_key) WHERE idempotency_key IS NOT NULL;
|
|
384
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_access_outcome
|
|
385
|
+
ON retrieval_access_log(source_type, source_id, access_type, timestamp DESC);
|
|
386
|
+
|
|
387
|
+
CREATE TABLE IF NOT EXISTS maintenance_checkpoints (
|
|
388
|
+
action TEXT PRIMARY KEY,
|
|
389
|
+
cursor TEXT,
|
|
390
|
+
details TEXT NOT NULL DEFAULT '{}',
|
|
391
|
+
updated_at INTEGER NOT NULL
|
|
392
|
+
);
|
|
393
|
+
|
|
394
|
+
CREATE TABLE IF NOT EXISTS memory_maintenance_runs (
|
|
395
|
+
id TEXT PRIMARY KEY,
|
|
396
|
+
mode TEXT NOT NULL CHECK(mode IN ('dry-run','apply')),
|
|
397
|
+
actions TEXT NOT NULL,
|
|
398
|
+
manifest TEXT NOT NULL,
|
|
399
|
+
created_at INTEGER NOT NULL
|
|
400
|
+
);
|
|
401
|
+
CREATE INDEX IF NOT EXISTS idx_memory_maintenance_runs_created
|
|
402
|
+
ON memory_maintenance_runs(created_at);
|
|
403
|
+
|
|
404
|
+
CREATE TABLE IF NOT EXISTS event_processing_audit (
|
|
405
|
+
id TEXT PRIMARY KEY,
|
|
406
|
+
event_id TEXT NOT NULL,
|
|
407
|
+
disposition TEXT NOT NULL,
|
|
408
|
+
reason TEXT NOT NULL,
|
|
409
|
+
run_id TEXT,
|
|
410
|
+
timestamp INTEGER NOT NULL
|
|
411
|
+
);
|
|
412
|
+
CREATE INDEX IF NOT EXISTS idx_event_processing_event
|
|
413
|
+
ON event_processing_audit(event_id, timestamp);
|
|
414
|
+
CREATE INDEX IF NOT EXISTS idx_event_processing_run_disposition
|
|
415
|
+
ON event_processing_audit(run_id, disposition);
|
|
416
|
+
|
|
417
|
+
CREATE TABLE IF NOT EXISTS event_duplicate_quarantine (
|
|
418
|
+
group_key TEXT PRIMARY KEY,
|
|
419
|
+
event_ids TEXT NOT NULL,
|
|
420
|
+
reason TEXT NOT NULL,
|
|
421
|
+
created_at INTEGER NOT NULL,
|
|
422
|
+
resolved_at INTEGER
|
|
423
|
+
);
|
|
424
|
+
|
|
425
|
+
CREATE TABLE IF NOT EXISTS observation_entities (
|
|
426
|
+
observation_id TEXT NOT NULL REFERENCES observations(id) ON DELETE CASCADE,
|
|
427
|
+
project TEXT NOT NULL,
|
|
428
|
+
normalized_entity TEXT NOT NULL,
|
|
429
|
+
display_entity TEXT NOT NULL,
|
|
430
|
+
PRIMARY KEY (observation_id, normalized_entity)
|
|
431
|
+
);
|
|
432
|
+
CREATE INDEX IF NOT EXISTS idx_observation_entities_lookup
|
|
433
|
+
ON observation_entities(project, normalized_entity, observation_id);
|
|
434
|
+
|
|
435
|
+
CREATE TABLE IF NOT EXISTS observation_relations (
|
|
436
|
+
source_observation_id TEXT NOT NULL REFERENCES observations(id) ON DELETE CASCADE,
|
|
437
|
+
target_observation_id TEXT NOT NULL REFERENCES observations(id) ON DELETE CASCADE,
|
|
438
|
+
type TEXT NOT NULL CHECK(type IN ('related_to', 'supersedes', 'references', 'contradicts', 'continues')),
|
|
439
|
+
created_at INTEGER NOT NULL,
|
|
440
|
+
PRIMARY KEY (source_observation_id, target_observation_id, type)
|
|
441
|
+
);
|
|
442
|
+
CREATE INDEX IF NOT EXISTS idx_observation_relations_target
|
|
443
|
+
ON observation_relations(target_observation_id, type);
|
|
284
444
|
`;
|
|
285
445
|
const FTS_TRIGGERS = `
|
|
286
446
|
CREATE TRIGGER IF NOT EXISTS memories_ai AFTER INSERT ON memories BEGIN
|
|
@@ -351,7 +511,6 @@ END;
|
|
|
351
511
|
const MAX_OBSERVATION_FTS_QUERY_LENGTH = 512;
|
|
352
512
|
const MAX_OBSERVATION_FTS_TERMS = 16;
|
|
353
513
|
const MAX_OBSERVATION_FTS_TERM_LENGTH = 64;
|
|
354
|
-
const OBSERVATION_FTS_SCHEMA_VERSION = 6;
|
|
355
514
|
const OBSERVATION_EXCLUSION_CTE = `
|
|
356
515
|
WITH excluded_observation_ids(id) AS (
|
|
357
516
|
SELECT CAST(value AS TEXT) FROM json_each(?)
|
|
@@ -402,30 +561,18 @@ function normalizeObservationLexicalQuery(query) {
|
|
|
402
561
|
: null,
|
|
403
562
|
};
|
|
404
563
|
}
|
|
405
|
-
export function isRecoverableObservationFtsError(error) {
|
|
406
|
-
const candidate = error;
|
|
407
|
-
const code = typeof candidate?.code === 'string' ? candidate.code.toUpperCase() : '';
|
|
408
|
-
const message = typeof candidate?.message === 'string' ? candidate.message : '';
|
|
409
|
-
if (/^SQLITE_(?:BUSY|LOCKED|READONLY|IOERR|FULL|CANTOPEN|PERM)/.test(code))
|
|
410
|
-
return false;
|
|
411
|
-
if (code === 'SQLITE_CORRUPT_VTAB')
|
|
412
|
-
return true;
|
|
413
|
-
const referencesObservationFts = /observations_(?:terms_)?fts(?:_(?:data|idx|content|docsize|config))?/i
|
|
414
|
-
.test(message);
|
|
415
|
-
if (!referencesObservationFts)
|
|
416
|
-
return false;
|
|
417
|
-
if (/^SQLITE_CORRUPT(?:_|$)/.test(code))
|
|
418
|
-
return true;
|
|
419
|
-
return code === 'SQLITE_ERROR' && /no such table|malformed schema|vtable constructor failed/i.test(message);
|
|
420
|
-
}
|
|
421
564
|
// ── MemoryStore ─────────────────────────────────────────
|
|
422
565
|
export class MemoryStore {
|
|
566
|
+
embeddingProfile;
|
|
567
|
+
projectIdentity;
|
|
423
568
|
db;
|
|
424
569
|
dbPath;
|
|
425
570
|
defaultProjectRoot;
|
|
426
571
|
leaseStore = null;
|
|
427
572
|
retrievalRevision = 0;
|
|
428
|
-
constructor(rootPath) {
|
|
573
|
+
constructor(rootPath, embeddingProfile = createMiniLmEmbeddingProfile('memory_retrieval'), projectIdentity) {
|
|
574
|
+
this.embeddingProfile = embeddingProfile;
|
|
575
|
+
this.projectIdentity = projectIdentity;
|
|
429
576
|
this.defaultProjectRoot = rootPath;
|
|
430
577
|
const memDir = join(rootPath, '.devflow', 'memory');
|
|
431
578
|
if (!existsSync(memDir))
|
|
@@ -436,22 +583,10 @@ export class MemoryStore {
|
|
|
436
583
|
this.db.pragma('busy_timeout = 5000');
|
|
437
584
|
this.db.pragma('foreign_keys = ON');
|
|
438
585
|
sqliteVec.load(this.db);
|
|
439
|
-
this.
|
|
586
|
+
this.assertCanonicalSchemaOrEmpty();
|
|
587
|
+
this.db.exec(MEMORY_SCHEMA);
|
|
588
|
+
this.db.exec(OBSERVATION_FTS_SCHEMA);
|
|
440
589
|
this.db.exec(FTS_TRIGGERS);
|
|
441
|
-
this.dropOrphanedObservationFtsTriggers();
|
|
442
|
-
this.migrateV1ToV2();
|
|
443
|
-
applyV2ToV3Migration(this.db);
|
|
444
|
-
applyV3ToV4Migration(this.db);
|
|
445
|
-
applyV4ToV5Migration(this.db);
|
|
446
|
-
this.ensureObservationFts();
|
|
447
|
-
applyV6ToV7Migration(this.db);
|
|
448
|
-
applyV7ToV8Migration(this.db);
|
|
449
|
-
applyV8ToV9Migration(this.db);
|
|
450
|
-
applyV9ToV10Migration(this.db);
|
|
451
|
-
applyV10ToV11Migration(this.db);
|
|
452
|
-
applyV11ToV12Migration(this.db, rootPath);
|
|
453
|
-
applyV12ToV13Migration(this.db);
|
|
454
|
-
this.repairOrphanSessions(rootPath);
|
|
455
590
|
emitSemanticControlLog({
|
|
456
591
|
event: 'memory.engine.transitioned', identity: { projectRoot: this.defaultProjectRoot }, level: 'debug', timestamp: Date.now(),
|
|
457
592
|
data: { stage: 'store_initialized', database: 'memories.db', tableCount: this.countTables() },
|
|
@@ -464,152 +599,42 @@ export class MemoryStore {
|
|
|
464
599
|
const row = this.db.prepare("SELECT COUNT(*) as c FROM sqlite_master WHERE type='table'").get();
|
|
465
600
|
return row.c;
|
|
466
601
|
}
|
|
467
|
-
|
|
468
|
-
const
|
|
469
|
-
if (
|
|
602
|
+
assertCanonicalSchemaOrEmpty() {
|
|
603
|
+
const memories = this.getTableColumns('memories');
|
|
604
|
+
if (memories.length === 0)
|
|
470
605
|
return;
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
if (needsRebuild) {
|
|
493
|
-
this.db.exec(`
|
|
494
|
-
DROP TRIGGER IF EXISTS observations_fts_ai;
|
|
495
|
-
DROP TRIGGER IF EXISTS observations_fts_ad;
|
|
496
|
-
DROP TRIGGER IF EXISTS observations_fts_au;
|
|
497
|
-
`);
|
|
498
|
-
}
|
|
499
|
-
if (forceRebuild) {
|
|
500
|
-
this.db.exec(`
|
|
501
|
-
DROP TABLE IF EXISTS observations_fts;
|
|
502
|
-
DROP TABLE IF EXISTS observations_terms_fts;
|
|
503
|
-
`);
|
|
504
|
-
}
|
|
505
|
-
this.db.exec(OBSERVATION_FTS_SCHEMA);
|
|
506
|
-
if (!needsRebuild)
|
|
507
|
-
return;
|
|
508
|
-
this.db.exec(`
|
|
509
|
-
DELETE FROM observations_fts;
|
|
510
|
-
DELETE FROM observations_terms_fts;
|
|
511
|
-
|
|
512
|
-
INSERT INTO observations_fts(rowid, id, title, narrative, facts, concepts)
|
|
513
|
-
SELECT o.rowid, o.id, o.title, o.narrative, o.facts, o.concepts
|
|
514
|
-
FROM observations o
|
|
515
|
-
WHERE o.quality_status = 'accepted'
|
|
516
|
-
AND o.resolution_state = 'current';
|
|
517
|
-
|
|
518
|
-
INSERT INTO observations_terms_fts(rowid, id, title, narrative, facts, concepts)
|
|
519
|
-
SELECT o.rowid, o.id, o.title, o.narrative, o.facts, o.concepts
|
|
520
|
-
FROM observations o
|
|
521
|
-
WHERE o.quality_status = 'accepted'
|
|
522
|
-
AND o.resolution_state = 'current';
|
|
523
|
-
`);
|
|
524
|
-
this.db.prepare('INSERT OR REPLACE INTO schema_versions(version, applied_at) VALUES (?, ?)').run(OBSERVATION_FTS_SCHEMA_VERSION, Date.now());
|
|
525
|
-
})();
|
|
526
|
-
}
|
|
527
|
-
// ── V1 → V2 Migration ────────────────────────────
|
|
528
|
-
migrateV1ToV2() {
|
|
529
|
-
const tables = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table'").all().map(r => r.name);
|
|
530
|
-
// If old 'events' table exists, migrate data to session_events
|
|
531
|
-
if (tables.includes('events') && tables.includes('session_events')) {
|
|
532
|
-
const count = this.db.prepare('SELECT COUNT(*) as c FROM events').get();
|
|
533
|
-
if (count.c > 0) {
|
|
534
|
-
try {
|
|
535
|
-
this.db.exec(`
|
|
536
|
-
INSERT OR IGNORE INTO session_events
|
|
537
|
-
(id, session_id, project, tool, command, exit_code, stderr,
|
|
538
|
-
duration_ms, created_at, processed, l1_score, dialog_score, user_message)
|
|
539
|
-
SELECT id, session_id, '', tool, command, exit_code, stderr,
|
|
540
|
-
duration_ms, created_at, processed,
|
|
541
|
-
COALESCE(l1_score, 0), COALESCE(dialog_score, 0), user_message
|
|
542
|
-
FROM events
|
|
543
|
-
`);
|
|
544
|
-
emitSemanticControlLog({
|
|
545
|
-
event: 'memory.write.completed', identity: { projectRoot: this.defaultProjectRoot }, level: 'info', timestamp: Date.now(),
|
|
546
|
-
data: { stage: 'event_migration', migrated: count.c },
|
|
547
|
-
});
|
|
548
|
-
}
|
|
549
|
-
catch (err) {
|
|
550
|
-
emitSemanticControlLog({
|
|
551
|
-
event: 'memory.write.degraded', identity: { projectRoot: this.defaultProjectRoot }, level: 'warn', timestamp: Date.now(),
|
|
552
|
-
data: { stage: 'event_migration', reason: memoryStoreErrorReason(err), fallback: 'existing_schema' },
|
|
553
|
-
});
|
|
554
|
-
}
|
|
555
|
-
}
|
|
556
|
-
}
|
|
557
|
-
// Add new columns to memories table if missing
|
|
558
|
-
const memCols = this.getTableColumns('memories');
|
|
559
|
-
const newCols = [
|
|
560
|
-
{ name: 'title', sql: "ALTER TABLE memories ADD COLUMN title TEXT NOT NULL DEFAULT ''" },
|
|
561
|
-
{ name: 'type', sql: "ALTER TABLE memories ADD COLUMN type TEXT" },
|
|
562
|
-
{ name: 'version', sql: 'ALTER TABLE memories ADD COLUMN version INTEGER NOT NULL DEFAULT 1' },
|
|
563
|
-
{ name: 'supersedes', sql: 'ALTER TABLE memories ADD COLUMN supersedes TEXT' },
|
|
564
|
-
{ name: 'superseded_by', sql: 'ALTER TABLE memories ADD COLUMN superseded_by TEXT' },
|
|
565
|
-
{ name: 'tags', sql: "ALTER TABLE memories ADD COLUMN tags TEXT NOT NULL DEFAULT '[]'" },
|
|
566
|
-
{ name: 'context', sql: "ALTER TABLE memories ADD COLUMN context TEXT NOT NULL DEFAULT ''" },
|
|
567
|
-
{ name: 'strength', sql: 'ALTER TABLE memories ADD COLUMN strength REAL NOT NULL DEFAULT 0.5' },
|
|
568
|
-
{ name: 'last_reinforced_at', sql: 'ALTER TABLE memories ADD COLUMN last_reinforced_at INTEGER' },
|
|
569
|
-
{ name: 'reinforce_count', sql: 'ALTER TABLE memories ADD COLUMN reinforce_count INTEGER NOT NULL DEFAULT 0' },
|
|
570
|
-
{ name: 'updated_at', sql: 'ALTER TABLE memories ADD COLUMN updated_at INTEGER NOT NULL DEFAULT 0' },
|
|
606
|
+
const requiredColumns = new Map([
|
|
607
|
+
['memories', [
|
|
608
|
+
'project_id', 'memory_namespace', 'actor_id', 'session_id',
|
|
609
|
+
'type', 'title', 'status', 'updated_at', 'last_accessed_at',
|
|
610
|
+
]],
|
|
611
|
+
['session_events', ['project', 'kind', 'payload', 'maintenance_status', 'quarantine_reason']],
|
|
612
|
+
['observations', [
|
|
613
|
+
'project_id', 'memory_namespace', 'actor_id', 'session_id',
|
|
614
|
+
'source', 'dedup_key', 'quality_status', 'resolution_state', 'evidence_status',
|
|
615
|
+
'semantic_key', 'facet_key', 'entities', 'capsule', 'projection', 'extraction_version',
|
|
616
|
+
]],
|
|
617
|
+
['embedding_profiles', ['profile_hash', 'generation', 'source_content_hash']],
|
|
618
|
+
['retrieval_access_log', [
|
|
619
|
+
'project', 'request_id', 'execution_id', 'context_receipt', 'tool_evidence',
|
|
620
|
+
'verification_receipt', 'idempotency_key',
|
|
621
|
+
]],
|
|
622
|
+
]);
|
|
623
|
+
const requiredTables = [
|
|
624
|
+
'distill_leases', 'distill_lease_pages', 'distill_receipts', 'maintenance_checkpoints',
|
|
625
|
+
'memory_maintenance_runs', 'event_processing_audit', 'event_duplicate_quarantine',
|
|
626
|
+
'observation_entities', 'observation_relations', 'observations_fts', 'observations_terms_fts',
|
|
571
627
|
];
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
}
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
try {
|
|
583
|
-
this.db.exec("UPDATE memories SET type = category WHERE type IS NULL");
|
|
584
|
-
}
|
|
585
|
-
catch { /* ignore */ }
|
|
586
|
-
}
|
|
587
|
-
if (!memCols.includes('title')) {
|
|
588
|
-
try {
|
|
589
|
-
this.db.exec("UPDATE memories SET title = COALESCE(title, '') WHERE title IS NULL OR title = ''");
|
|
590
|
-
}
|
|
591
|
-
catch { /* ignore */ }
|
|
592
|
-
}
|
|
593
|
-
if (!memCols.includes('updated_at')) {
|
|
594
|
-
try {
|
|
595
|
-
this.db.exec('UPDATE memories SET updated_at = created_at WHERE updated_at = 0');
|
|
596
|
-
}
|
|
597
|
-
catch { /* ignore */ }
|
|
598
|
-
}
|
|
599
|
-
// Add kind and payload columns to session_events if missing
|
|
600
|
-
const evtCols = this.getTableColumns('session_events');
|
|
601
|
-
if (!evtCols.includes('kind')) {
|
|
602
|
-
try {
|
|
603
|
-
this.db.exec('ALTER TABLE session_events ADD COLUMN kind TEXT');
|
|
604
|
-
}
|
|
605
|
-
catch { }
|
|
606
|
-
}
|
|
607
|
-
if (!evtCols.includes('payload')) {
|
|
608
|
-
try {
|
|
609
|
-
this.db.exec('ALTER TABLE session_events ADD COLUMN payload TEXT');
|
|
610
|
-
}
|
|
611
|
-
catch { }
|
|
612
|
-
}
|
|
628
|
+
const tables = new Set(this.db.prepare("SELECT name FROM sqlite_master WHERE type IN ('table', 'view')").all().map(row => row.name));
|
|
629
|
+
const invalid = [
|
|
630
|
+
...requiredTables.filter(table => !tables.has(table)).map(table => `table:${table}`),
|
|
631
|
+
...[...requiredColumns].flatMap(([table, columns]) => {
|
|
632
|
+
const actual = new Set(this.getTableColumns(table));
|
|
633
|
+
return columns.filter(column => !actual.has(column)).map(column => `column:${table}.${column}`);
|
|
634
|
+
}),
|
|
635
|
+
];
|
|
636
|
+
if (invalid.length > 0)
|
|
637
|
+
throw new Error(`MEMORY_SCHEMA_UNSUPPORTED:${invalid.join(',')}`);
|
|
613
638
|
}
|
|
614
639
|
getTableColumns(table) {
|
|
615
640
|
try {
|
|
@@ -622,18 +647,17 @@ export class MemoryStore {
|
|
|
622
647
|
}
|
|
623
648
|
// ── Memory CRUD ──────────────────────────────────
|
|
624
649
|
add(entry) {
|
|
625
|
-
|
|
626
|
-
this.ensureSession(entry.sessionId, this.defaultProjectRoot);
|
|
627
|
-
}
|
|
650
|
+
this.ensureSession(entry.sessionId, this.projectIdentity.memoryNamespace);
|
|
628
651
|
const id = entry.id ?? `mem:${Date.now()}:${randomUUID().slice(0, 8)}`;
|
|
629
652
|
const now = Date.now();
|
|
630
|
-
const memType = entry.type ??
|
|
653
|
+
const memType = entry.type ?? 'fact';
|
|
631
654
|
const title = entry.title ?? entry.content.slice(0, 80);
|
|
632
655
|
this.db.prepare(`
|
|
633
|
-
INSERT INTO memories (id,
|
|
656
|
+
INSERT INTO memories (id, project_id, memory_namespace, actor_id,
|
|
657
|
+
type, title, content, original_text, source, status,
|
|
634
658
|
confidence, strength, scope, session_id, created_at, updated_at)
|
|
635
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
636
|
-
`).run(id, memType, title, entry.content, entry.originalText ?? null, entry.source ?? 'auto', entry.status ?? 'active', entry.confidence ?? 0.5, entry.strength ?? 0.5, entry.scope ?? '', entry.sessionId
|
|
659
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
660
|
+
`).run(id, this.projectIdentity.projectId, this.projectIdentity.memoryNamespace, entry.actorId, memType, title, entry.content, entry.originalText ?? null, entry.source ?? 'auto', entry.status ?? 'active', entry.confidence ?? 0.5, entry.strength ?? 0.5, entry.scope ?? '', entry.sessionId, now, now);
|
|
637
661
|
emitSemanticControlLog({
|
|
638
662
|
event: 'memory.write.completed', identity: { projectRoot: this.defaultProjectRoot, sessionId: entry.sessionId }, level: 'info', timestamp: now,
|
|
639
663
|
data: { stage: 'memory_add', memoryId: id, type: memType, status: entry.status ?? 'active' },
|
|
@@ -643,7 +667,10 @@ export class MemoryStore {
|
|
|
643
667
|
get(id) {
|
|
644
668
|
return this.db.prepare('SELECT * FROM memories WHERE id = ?').get(id) ?? null;
|
|
645
669
|
}
|
|
646
|
-
|
|
670
|
+
updateScoped(scope, updates) {
|
|
671
|
+
const invalidatesEmbedding = updates.title !== undefined
|
|
672
|
+
|| updates.content !== undefined
|
|
673
|
+
|| updates.context !== undefined;
|
|
647
674
|
const sets = [];
|
|
648
675
|
const vals = [];
|
|
649
676
|
for (const [key, value] of Object.entries(updates)) {
|
|
@@ -657,20 +684,42 @@ export class MemoryStore {
|
|
|
657
684
|
vals.push(Date.now());
|
|
658
685
|
if (sets.length === 0)
|
|
659
686
|
return;
|
|
660
|
-
vals.push(id);
|
|
661
|
-
this.db.
|
|
687
|
+
vals.push(scope.id, scope.projectId, scope.memoryNamespace, scope.actorId, scope.sessionId);
|
|
688
|
+
this.db.transaction(() => {
|
|
689
|
+
const updated = this.db.prepare(`UPDATE memories SET ${sets.join(', ')}
|
|
690
|
+
WHERE id = ? AND project_id = ? AND memory_namespace = ? AND actor_id = ? AND session_id = ?`).run(...vals);
|
|
691
|
+
if (updated.changes !== 1)
|
|
692
|
+
throw new Error('MEMORY_SCOPE_MISMATCH');
|
|
693
|
+
if (invalidatesEmbedding) {
|
|
694
|
+
this.db.prepare('DELETE FROM memories_vec WHERE id = ?').run(scope.id);
|
|
695
|
+
this.db.prepare("DELETE FROM embeddings WHERE source_type = 'memory' AND source_id = ?").run(scope.id);
|
|
696
|
+
this.db.prepare("DELETE FROM embedding_profiles WHERE source_type = 'memory' AND source_id = ?").run(scope.id);
|
|
697
|
+
}
|
|
698
|
+
})();
|
|
662
699
|
emitSemanticControlLog({
|
|
663
700
|
event: 'memory.write.completed', identity: { projectRoot: this.defaultProjectRoot }, level: 'debug', timestamp: Date.now(),
|
|
664
|
-
data: { stage: 'memory_update', memoryId: id, fields: Object.keys(updates) },
|
|
701
|
+
data: { stage: 'memory_update', memoryId: scope.id, fields: Object.keys(updates) },
|
|
665
702
|
});
|
|
666
703
|
}
|
|
667
|
-
|
|
668
|
-
this.db.
|
|
704
|
+
deleteScoped(scope) {
|
|
705
|
+
this.db.transaction(() => {
|
|
706
|
+
const exists = this.db.prepare(`SELECT id FROM memories
|
|
707
|
+
WHERE id = ? AND project_id = ? AND memory_namespace = ? AND actor_id = ? AND session_id = ?`)
|
|
708
|
+
.get(scope.id, scope.projectId, scope.memoryNamespace, scope.actorId, scope.sessionId);
|
|
709
|
+
if (!exists)
|
|
710
|
+
throw new Error('MEMORY_SCOPE_MISMATCH');
|
|
711
|
+
this.db.prepare('DELETE FROM memories_vec WHERE id = ?').run(scope.id);
|
|
712
|
+
this.db.prepare("DELETE FROM embeddings WHERE source_type = 'memory' AND source_id = ?").run(scope.id);
|
|
713
|
+
this.db.prepare("DELETE FROM embedding_profiles WHERE source_type = 'memory' AND source_id = ?").run(scope.id);
|
|
714
|
+
this.db.prepare(`DELETE FROM memories
|
|
715
|
+
WHERE id = ? AND project_id = ? AND memory_namespace = ? AND actor_id = ? AND session_id = ?`)
|
|
716
|
+
.run(scope.id, scope.projectId, scope.memoryNamespace, scope.actorId, scope.sessionId);
|
|
717
|
+
})();
|
|
669
718
|
}
|
|
670
719
|
list(filter) {
|
|
671
720
|
let sql = 'SELECT * FROM memories WHERE superseded_by IS NULL';
|
|
672
721
|
const vals = [];
|
|
673
|
-
const typeFilter = filter?.type
|
|
722
|
+
const typeFilter = filter?.type;
|
|
674
723
|
if (typeFilter) {
|
|
675
724
|
sql += ' AND type = ?';
|
|
676
725
|
vals.push(typeFilter);
|
|
@@ -702,23 +751,27 @@ export class MemoryStore {
|
|
|
702
751
|
const placeholders = types.map(() => '?').join(',');
|
|
703
752
|
return this.db.prepare(`SELECT * FROM memories WHERE type IN (${placeholders}) AND superseded_by IS NULL AND status IN ('active','approved') ORDER BY updated_at DESC LIMIT ?`).all(...types, limit);
|
|
704
753
|
}
|
|
705
|
-
listByCategory(category, limit = 100) {
|
|
706
|
-
return this.listByType(category, limit);
|
|
707
|
-
}
|
|
708
754
|
countByStatus(status) {
|
|
709
755
|
const row = this.db.prepare('SELECT COUNT(*) as c FROM memories WHERE status = ?').get(status);
|
|
710
756
|
return row?.c ?? 0;
|
|
711
757
|
}
|
|
712
758
|
// ── Embedding ────────────────────────────────────
|
|
713
759
|
insertEmbedding(id, embedding) {
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
this.db.
|
|
760
|
+
this.assertEmbeddingProfile(embedding);
|
|
761
|
+
const buffer = Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength);
|
|
762
|
+
this.db.transaction(() => {
|
|
763
|
+
this.db.prepare('INSERT OR REPLACE INTO memories_vec(id, embedding) VALUES (?, ?)').run(id, buffer);
|
|
764
|
+
this.db.prepare('INSERT OR REPLACE INTO embeddings(source_type, source_id, vector) VALUES (?, ?, ?)')
|
|
765
|
+
.run('memory', id, buffer);
|
|
766
|
+
this.putEmbeddingProfile('memory', id);
|
|
767
|
+
})();
|
|
717
768
|
}
|
|
718
769
|
insertObservationEmbedding(obsId, embedding) {
|
|
770
|
+
this.assertEmbeddingProfile(embedding);
|
|
719
771
|
const buffer = Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength);
|
|
720
772
|
this.db.prepare('INSERT OR REPLACE INTO embeddings(source_type, source_id, vector) VALUES (?, ?, ?)').run('observation', obsId, buffer);
|
|
721
773
|
this.db.prepare('UPDATE observations SET embedding = ? WHERE id = ?').run(buffer, obsId);
|
|
774
|
+
this.putEmbeddingProfile('observation', obsId);
|
|
722
775
|
this.bumpRetrievalRevision();
|
|
723
776
|
}
|
|
724
777
|
insertObservationEmbeddings(entries) {
|
|
@@ -728,9 +781,11 @@ export class MemoryStore {
|
|
|
728
781
|
const update = this.db.prepare('UPDATE observations SET embedding = ? WHERE id = ?');
|
|
729
782
|
this.db.transaction(() => {
|
|
730
783
|
for (const { id, embedding } of entries) {
|
|
784
|
+
this.assertEmbeddingProfile(embedding);
|
|
731
785
|
const buffer = Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength);
|
|
732
786
|
insert.run('observation', id, buffer);
|
|
733
787
|
update.run(buffer, id);
|
|
788
|
+
this.putEmbeddingProfile('observation', id);
|
|
734
789
|
}
|
|
735
790
|
})();
|
|
736
791
|
this.bumpRetrievalRevision();
|
|
@@ -755,12 +810,14 @@ export class MemoryStore {
|
|
|
755
810
|
const rows = this.db.prepare(`${OBSERVATION_EXCLUSION_CTE}
|
|
756
811
|
SELECT o.*, vec_distance_cosine(o.embedding, ?) AS distance
|
|
757
812
|
FROM observations o
|
|
813
|
+
JOIN embedding_profiles p ON p.source_type = 'observation' AND p.source_id = o.id
|
|
758
814
|
WHERE o.embedding IS NOT NULL
|
|
815
|
+
AND p.profile_hash = ? AND p.generation = ?
|
|
759
816
|
AND o.quality_status = 'accepted'
|
|
760
817
|
AND o.resolution_state <> 'superseded'
|
|
761
818
|
AND o.id NOT IN (SELECT id FROM excluded_observation_ids)
|
|
762
819
|
ORDER BY distance LIMIT ?
|
|
763
|
-
`).all(excludedJson, buffer, limit);
|
|
820
|
+
`).all(excludedJson, buffer, this.embeddingProfile.profileHash, this.embeddingProfile.generation, limit);
|
|
764
821
|
return rows.map(row => ({ observation: row, distance: row.distance }));
|
|
765
822
|
}
|
|
766
823
|
// ── Search ───────────────────────────────────────
|
|
@@ -789,27 +846,16 @@ export class MemoryStore {
|
|
|
789
846
|
}
|
|
790
847
|
};
|
|
791
848
|
if (normalizedQuery.trigramQuery) {
|
|
792
|
-
addRanked(this.
|
|
849
|
+
addRanked(this.searchObservationFts('observations_fts', normalizedQuery.trigramQuery, boundedLimit, excludedJson));
|
|
793
850
|
}
|
|
794
851
|
if (normalizedQuery.termQuery) {
|
|
795
|
-
addRanked(this.
|
|
852
|
+
addRanked(this.searchObservationFts('observations_terms_fts', normalizedQuery.termQuery, boundedLimit, excludedJson));
|
|
796
853
|
}
|
|
797
854
|
return [...scores]
|
|
798
855
|
.map(([id, score]) => ({ id, rank: -score }))
|
|
799
856
|
.sort((left, right) => left.rank - right.rank || left.id.localeCompare(right.id))
|
|
800
857
|
.slice(0, boundedLimit);
|
|
801
858
|
}
|
|
802
|
-
searchObservationFtsWithRecovery(table, query, limit, excludedJson) {
|
|
803
|
-
try {
|
|
804
|
-
return this.searchObservationFts(table, query, limit, excludedJson);
|
|
805
|
-
}
|
|
806
|
-
catch (error) {
|
|
807
|
-
if (!isRecoverableObservationFtsError(error))
|
|
808
|
-
throw error;
|
|
809
|
-
this.ensureObservationFts(true);
|
|
810
|
-
return this.searchObservationFts(table, query, limit, excludedJson);
|
|
811
|
-
}
|
|
812
|
-
}
|
|
813
859
|
searchObservationFts(table, query, limit, excludedJson) {
|
|
814
860
|
const sql = table === 'observations_fts' ? `${OBSERVATION_EXCLUSION_CTE}
|
|
815
861
|
SELECT o.id, bm25(observations_fts) AS rank
|
|
@@ -834,14 +880,59 @@ export class MemoryStore {
|
|
|
834
880
|
return this.db.prepare(sql).all(excludedJson, query, limit);
|
|
835
881
|
}
|
|
836
882
|
searchVector(embedding, limit = 20) {
|
|
837
|
-
const buffer = Buffer.from(embedding.buffer);
|
|
883
|
+
const buffer = Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength);
|
|
838
884
|
return this.db.prepare(`
|
|
839
885
|
SELECT v.id, vec_distance_cosine(v.embedding, ?) as distance
|
|
840
886
|
FROM memories_vec v
|
|
841
887
|
JOIN memories m ON m.id = v.id
|
|
888
|
+
JOIN embedding_profiles p ON p.source_type = 'memory' AND p.source_id = v.id
|
|
842
889
|
WHERE m.status IN ('active','approved') AND m.superseded_by IS NULL
|
|
890
|
+
AND p.profile_hash = ? AND p.generation = ?
|
|
843
891
|
ORDER BY distance LIMIT ?
|
|
844
|
-
`).all(buffer, limit);
|
|
892
|
+
`).all(buffer, this.embeddingProfile.profileHash, this.embeddingProfile.generation, limit);
|
|
893
|
+
}
|
|
894
|
+
getEmbeddingProfileState(sourceType) {
|
|
895
|
+
const totalTable = sourceType === 'memory' ? 'memories_vec' : 'observations';
|
|
896
|
+
const totalWhere = sourceType === 'memory' ? '' : ' WHERE embedding IS NOT NULL';
|
|
897
|
+
const total = Number(this.db.prepare(`SELECT COUNT(*) AS count FROM ${totalTable}${totalWhere}`).get().count);
|
|
898
|
+
const indexed = Number(this.db.prepare(`SELECT COUNT(*) AS count FROM embedding_profiles
|
|
899
|
+
WHERE source_type = ? AND profile_hash = ? AND generation = ?`).get(sourceType, this.embeddingProfile.profileHash, this.embeddingProfile.generation).count);
|
|
900
|
+
return {
|
|
901
|
+
status: this.embeddingProfile.dimension !== 384
|
|
902
|
+
? 'dimension_mismatch'
|
|
903
|
+
: indexed < total ? 'backfill_pending' : 'ready',
|
|
904
|
+
profileHash: this.embeddingProfile.profileHash,
|
|
905
|
+
generation: this.embeddingProfile.generation,
|
|
906
|
+
indexed,
|
|
907
|
+
total,
|
|
908
|
+
};
|
|
909
|
+
}
|
|
910
|
+
putEmbeddingProfile(sourceType, sourceId) {
|
|
911
|
+
const sourceContentHash = sourceType === 'memory'
|
|
912
|
+
? this.memoryEmbeddingSourceHash(sourceId)
|
|
913
|
+
: this.observationEmbeddingSourceHash(sourceId);
|
|
914
|
+
this.db.prepare(`INSERT OR REPLACE INTO embedding_profiles
|
|
915
|
+
(source_type, source_id, profile_hash, generation, source_content_hash) VALUES (?, ?, ?, ?, ?)`)
|
|
916
|
+
.run(sourceType, sourceId, this.embeddingProfile.profileHash, this.embeddingProfile.generation, sourceContentHash);
|
|
917
|
+
}
|
|
918
|
+
memoryEmbeddingSourceHash(id) {
|
|
919
|
+
const row = this.db.prepare('SELECT title, content, context FROM memories WHERE id = ?')
|
|
920
|
+
.get(id);
|
|
921
|
+
if (!row)
|
|
922
|
+
throw new Error(`EMBEDDING_SOURCE_MISSING:memory:${id}`);
|
|
923
|
+
return createHash('sha256').update(JSON.stringify(row)).digest('hex');
|
|
924
|
+
}
|
|
925
|
+
observationEmbeddingSourceHash(id) {
|
|
926
|
+
const row = this.db.prepare(`SELECT title, narrative, facts, files, tools, concepts
|
|
927
|
+
FROM observations WHERE id = ?`).get(id);
|
|
928
|
+
if (!row)
|
|
929
|
+
throw new Error(`EMBEDDING_SOURCE_MISSING:observation:${id}`);
|
|
930
|
+
return createHash('sha256').update(JSON.stringify(row)).digest('hex');
|
|
931
|
+
}
|
|
932
|
+
assertEmbeddingProfile(embedding) {
|
|
933
|
+
if (embedding.length !== this.embeddingProfile.dimension || embedding.length !== 384) {
|
|
934
|
+
throw new Error(`EMBEDDING_DIMENSION_MISMATCH:${embedding.length}:${this.embeddingProfile.dimension}`);
|
|
935
|
+
}
|
|
845
936
|
}
|
|
846
937
|
// ── Edges ────────────────────────────────────────
|
|
847
938
|
addEdge(sourceId, targetId, type, weight = 0.5) {
|
|
@@ -1319,49 +1410,6 @@ export class MemoryStore {
|
|
|
1319
1410
|
`).run(sessionId, normalizedProjectRoot, task, Date.now());
|
|
1320
1411
|
return this.getSession(sessionId);
|
|
1321
1412
|
}
|
|
1322
|
-
repairOrphanSessions(projectRoot = this.defaultProjectRoot) {
|
|
1323
|
-
const repair = this.db.transaction(() => {
|
|
1324
|
-
const blankEvents = this.db.prepare(`
|
|
1325
|
-
SELECT id FROM session_events WHERE TRIM(session_id) = '' ORDER BY created_at ASC
|
|
1326
|
-
`).all();
|
|
1327
|
-
const assignRecoveredSession = this.db.prepare('UPDATE session_events SET session_id = ? WHERE id = ? AND TRIM(session_id) = \'\'');
|
|
1328
|
-
for (const event of blankEvents) {
|
|
1329
|
-
assignRecoveredSession.run(`recovered:orphan:${event.id}`, event.id);
|
|
1330
|
-
}
|
|
1331
|
-
const orphanSessions = this.db.prepare(`
|
|
1332
|
-
SELECT
|
|
1333
|
-
e.session_id AS id,
|
|
1334
|
-
COALESCE(NULLIF(MAX(e.project), ''), ?) AS project_root,
|
|
1335
|
-
MIN(e.created_at) AS created_at,
|
|
1336
|
-
MAX(e.created_at) AS closed_at,
|
|
1337
|
-
COUNT(*) AS tool_count,
|
|
1338
|
-
SUM(CASE WHEN e.exit_code IS NOT NULL AND e.exit_code <> 0 THEN 1 ELSE 0 END) AS error_count
|
|
1339
|
-
FROM session_events e
|
|
1340
|
-
LEFT JOIN sessions s ON s.id = e.session_id
|
|
1341
|
-
WHERE s.id IS NULL
|
|
1342
|
-
GROUP BY e.session_id
|
|
1343
|
-
`).all(projectRoot);
|
|
1344
|
-
const insertRecovered = this.db.prepare(`
|
|
1345
|
-
INSERT OR IGNORE INTO sessions
|
|
1346
|
-
(id, project_root, task, status, phase_summary, signal_summary,
|
|
1347
|
-
tool_count, error_count, created_at, closed_at)
|
|
1348
|
-
VALUES (?, ?, 'Recovered orphan session events', 'closed', ?, '[]', ?, ?, ?, ?)
|
|
1349
|
-
`);
|
|
1350
|
-
let repaired = 0;
|
|
1351
|
-
for (const session of orphanSessions) {
|
|
1352
|
-
repaired += insertRecovered.run(session.id, session.project_root || projectRoot, JSON.stringify(['recovered_orphan_events']), session.tool_count, session.error_count, session.created_at, session.closed_at).changes;
|
|
1353
|
-
}
|
|
1354
|
-
return repaired;
|
|
1355
|
-
});
|
|
1356
|
-
const repaired = repair();
|
|
1357
|
-
if (repaired > 0) {
|
|
1358
|
-
emitSemanticControlLog({
|
|
1359
|
-
event: 'memory.engine.transitioned', identity: { projectRoot: this.defaultProjectRoot }, level: 'warn', timestamp: Date.now(),
|
|
1360
|
-
data: { stage: 'orphan_session_repair', recovered: repaired },
|
|
1361
|
-
});
|
|
1362
|
-
}
|
|
1363
|
-
return repaired;
|
|
1364
|
-
}
|
|
1365
1413
|
createSession(id, projectRoot, task = '') {
|
|
1366
1414
|
this.ensureSession(id, projectRoot, task);
|
|
1367
1415
|
emitSemanticControlLog({
|
|
@@ -1412,8 +1460,12 @@ export class MemoryStore {
|
|
|
1412
1460
|
this.ensureSession(obs.sessionId, obs.project ?? this.defaultProjectRoot);
|
|
1413
1461
|
const id = obs.id ?? `obs:${Date.now()}:${randomUUID().slice(0, 8)}`;
|
|
1414
1462
|
const now = obs.timestamp ?? Date.now();
|
|
1415
|
-
|
|
1416
|
-
|
|
1463
|
+
// A semantic projection is descriptive metadata, never an instance
|
|
1464
|
+
// identity. Only the command service may provide an explicit dedupKey
|
|
1465
|
+
// (the receipt-bound fact instance id); legacy callers fall back to the
|
|
1466
|
+
// non-semantic observation key.
|
|
1467
|
+
const dedupKey = obs.dedupKey ?? observationDedupKey(obs.type, obs.title);
|
|
1468
|
+
const duplicate = obs.dedupKey || obs.semantic ? null : this.findObservationByDedupKey(dedupKey);
|
|
1417
1469
|
if (duplicate) {
|
|
1418
1470
|
this.linkObservationToEvents(duplicate.id, obs.sourceEventIds ?? []);
|
|
1419
1471
|
this.incrementObservationAccess(duplicate.id, now);
|
|
@@ -1422,15 +1474,16 @@ export class MemoryStore {
|
|
|
1422
1474
|
}
|
|
1423
1475
|
this.db.prepare(`
|
|
1424
1476
|
INSERT INTO observations
|
|
1425
|
-
(id,
|
|
1477
|
+
(id, project_id, memory_namespace, actor_id, session_id, project,
|
|
1478
|
+
type, title, narrative, facts, files, tools, concepts,
|
|
1426
1479
|
importance, confidence, source, dedup_key, source_event_ids, timestamp, updated_at,
|
|
1427
1480
|
quality_status, quarantine_reason, quality_details, quarantined_at,
|
|
1428
1481
|
failure_category, resolution_state, evidence_status, evidence_details,
|
|
1429
1482
|
semantic_subject, semantic_predicate, semantic_value, semantic_polarity,
|
|
1430
1483
|
semantic_key, facet_key, entities, temporal, reinforce_count, last_reinforced_at,
|
|
1431
1484
|
capsule, projection, extraction_version)
|
|
1432
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1433
|
-
`).run(id, obs.sessionId, obs.project ?? this.
|
|
1485
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1486
|
+
`).run(id, this.projectIdentity.projectId, this.projectIdentity.memoryNamespace, obs.actorId, obs.sessionId, obs.project ?? this.projectIdentity.memoryNamespace, obs.type, obs.title, obs.narrative ?? '', JSON.stringify(obs.facts ?? []), JSON.stringify(obs.files ?? []), JSON.stringify(obs.tools ?? []), JSON.stringify(obs.concepts ?? []), obs.importance ?? 5, obs.confidence ?? 0.5, obs.source ?? 'distill', dedupKey, JSON.stringify(obs.sourceEventIds ?? []), now, now, obs.qualityStatus ?? 'accepted', obs.quarantineReason ?? null, obs.qualityDetails ? JSON.stringify(obs.qualityDetails) : null, obs.qualityStatus === 'quarantined' ? now : null, obs.failureCategory ?? null, obs.resolutionState ?? 'current', obs.evidenceStatus
|
|
1434
1487
|
?? (obs.source === 'explicit_intent'
|
|
1435
1488
|
? 'unverified'
|
|
1436
1489
|
: obs.source === 'manual' || (obs.sourceEventIds?.length ?? 0) > 0
|
|
@@ -1444,14 +1497,14 @@ export class MemoryStore {
|
|
|
1444
1497
|
sourceEvidenceIds: obs.sourceEventIds ?? [],
|
|
1445
1498
|
entities: obs.semantic?.entities ?? [],
|
|
1446
1499
|
temporalStatus: obs.semantic?.temporal?.status ?? 'current',
|
|
1447
|
-
extractionVersion: '
|
|
1500
|
+
extractionVersion: 'memory-capsule.v1',
|
|
1448
1501
|
}), JSON.stringify(obs.projection ?? {
|
|
1449
1502
|
subject: obs.semantic?.subject ?? '',
|
|
1450
1503
|
predicate: obs.semantic?.predicate ?? '',
|
|
1451
1504
|
value: obs.semantic?.value ?? '',
|
|
1452
1505
|
polarity: obs.semantic?.polarity ?? 'neutral',
|
|
1453
1506
|
facet: obs.semantic?.facet ?? 'generic',
|
|
1454
|
-
}), obs.capsule?.extractionVersion ?? '
|
|
1507
|
+
}), obs.capsule?.extractionVersion ?? 'memory-capsule.v1');
|
|
1455
1508
|
// Insert concept junction rows
|
|
1456
1509
|
if (obs.concepts && obs.concepts.length > 0) {
|
|
1457
1510
|
const insertConcept = this.db.prepare('INSERT OR IGNORE INTO observation_concepts (observation_id, concept) VALUES (?, ?)');
|
|
@@ -1470,104 +1523,23 @@ export class MemoryStore {
|
|
|
1470
1523
|
acceptSemanticObservation(obs) {
|
|
1471
1524
|
return this.db.transaction(() => {
|
|
1472
1525
|
const project = obs.project ?? this.defaultProjectRoot;
|
|
1473
|
-
const now = obs.timestamp ?? Date.now();
|
|
1474
|
-
const exact = this.findCurrentObservationBySemanticKey(project, obs.semantic.semanticKey)
|
|
1475
|
-
?? this.findCurrentObservationByNarrative(project, obs.narrative ?? '');
|
|
1476
|
-
if (exact) {
|
|
1477
|
-
const sourceEventIds = uniqueStrings([
|
|
1478
|
-
...parseStringArray(exact.source_event_ids),
|
|
1479
|
-
...(obs.sourceEventIds ?? []),
|
|
1480
|
-
]);
|
|
1481
|
-
const facts = uniqueStrings([...parseStringArray(exact.facts), ...(obs.facts ?? [])]);
|
|
1482
|
-
const concepts = uniqueStrings([
|
|
1483
|
-
...parseStringArray(exact.concepts),
|
|
1484
|
-
...(obs.concepts ?? []),
|
|
1485
|
-
obs.semantic.subject,
|
|
1486
|
-
obs.semantic.predicate,
|
|
1487
|
-
obs.semantic.value,
|
|
1488
|
-
...obs.semantic.entities.map(normalizeEntityKey),
|
|
1489
|
-
]);
|
|
1490
|
-
const entities = mergeSemanticEntities(parseStringArray(exact.entities), obs.semantic.entities);
|
|
1491
|
-
const narrative = obs.preferIncomingNarrative
|
|
1492
|
-
? obs.narrative ?? exact.narrative
|
|
1493
|
-
: semanticRichness(obs.narrative ?? '') > semanticRichness(exact.narrative)
|
|
1494
|
-
? obs.narrative ?? ''
|
|
1495
|
-
: exact.narrative;
|
|
1496
|
-
const title = obs.preferIncomingNarrative
|
|
1497
|
-
? obs.title
|
|
1498
|
-
: semanticRichness(obs.title) > semanticRichness(exact.title) ? obs.title : exact.title;
|
|
1499
|
-
const contentChanged = exact.title !== title
|
|
1500
|
-
|| exact.narrative !== narrative;
|
|
1501
|
-
this.db.prepare(`
|
|
1502
|
-
UPDATE observations
|
|
1503
|
-
SET type = ?, title = ?, narrative = ?, facts = ?, concepts = ?, importance = MAX(importance, ?),
|
|
1504
|
-
confidence = MAX(confidence, ?), source_event_ids = ?, updated_at = ?,
|
|
1505
|
-
access_count = access_count + 1, reinforce_count = reinforce_count + 1, last_reinforced_at = ?,
|
|
1506
|
-
source = CASE WHEN ? = 'explicit_intent' THEN 'explicit_intent' ELSE source END,
|
|
1507
|
-
quality_status = CASE WHEN quality_status = 'quarantined' AND ? = 'accepted' THEN 'accepted' ELSE quality_status END,
|
|
1508
|
-
quarantine_reason = CASE WHEN ? = 'accepted' THEN NULL ELSE quarantine_reason END,
|
|
1509
|
-
resolution_state = 'current', superseded_by_observation = NULL,
|
|
1510
|
-
semantic_subject = ?, semantic_predicate = ?, semantic_value = ?, semantic_polarity = ?,
|
|
1511
|
-
semantic_key = ?, facet_key = ?, dedup_key = ?, entities = ?,
|
|
1512
|
-
temporal = COALESCE(?, temporal), evidence_status = ?, evidence_details = ?,
|
|
1513
|
-
embedding = CASE WHEN ? = 1 THEN NULL ELSE embedding END
|
|
1514
|
-
WHERE id = ?
|
|
1515
|
-
`).run(obs.type, title, narrative, JSON.stringify(facts), JSON.stringify(concepts), obs.importance ?? 5, obs.confidence ?? 0.5, JSON.stringify(sourceEventIds), now, now, obs.source ?? 'distill', obs.qualityStatus ?? 'accepted', obs.qualityStatus ?? 'accepted', obs.semantic.subject, obs.semantic.predicate, obs.semantic.value, obs.semantic.polarity, obs.semantic.semanticKey, obs.semantic.facetKey, obs.semantic.semanticKey, JSON.stringify(entities), obs.semantic.temporal ? JSON.stringify(obs.semantic.temporal) : null, strongerEvidenceStatus(exact.evidence_status, obs.evidenceStatus), JSON.stringify(mergeEvidenceDetails(exact.evidence_details, obs.evidenceDetails ?? [])), contentChanged ? 1 : 0, exact.id);
|
|
1516
|
-
if (contentChanged) {
|
|
1517
|
-
this.db.prepare("DELETE FROM embeddings WHERE source_type = 'observation' AND source_id = ?").run(exact.id);
|
|
1518
|
-
}
|
|
1519
|
-
this.replaceObservationConcepts(exact.id, concepts);
|
|
1520
|
-
this.replaceObservationEntities(exact.id, project, entities);
|
|
1521
|
-
for (const eventId of obs.sourceEventIds ?? [])
|
|
1522
|
-
this.linkObservationToEvents(exact.id, [eventId]);
|
|
1523
|
-
const supersededIds = this.supersedeSemanticFacetConflicts(project, exact.id, obs.semantic, now);
|
|
1524
|
-
return { row: this.getObservation(exact.id), action: 'REINFORCE', supersededIds, linkedIds: [] };
|
|
1525
|
-
}
|
|
1526
1526
|
const row = this.addObservation(obs);
|
|
1527
1527
|
if ((obs.qualityStatus ?? 'accepted') === 'quarantined') {
|
|
1528
1528
|
return { row, action: 'HOLD', supersededIds: [], linkedIds: [] };
|
|
1529
1529
|
}
|
|
1530
|
-
const
|
|
1531
|
-
const
|
|
1530
|
+
const relatedIds = this.findCurrentObservationIdsByEntities(project, obs.semantic.entities, new Set([row.id])).slice(0, 12);
|
|
1531
|
+
const now = obs.timestamp ?? Date.now();
|
|
1532
1532
|
for (const relatedId of relatedIds) {
|
|
1533
1533
|
this.addObservationRelation(row.id, relatedId, 'related_to', now);
|
|
1534
1534
|
}
|
|
1535
1535
|
return {
|
|
1536
1536
|
row: this.getObservation(row.id),
|
|
1537
|
-
action:
|
|
1538
|
-
supersededIds,
|
|
1537
|
+
action: relatedIds.length > 0 ? 'LINK' : 'ADD',
|
|
1538
|
+
supersededIds: [],
|
|
1539
1539
|
linkedIds: relatedIds,
|
|
1540
1540
|
};
|
|
1541
1541
|
})();
|
|
1542
1542
|
}
|
|
1543
|
-
findCurrentObservationBySemanticKey(project, semanticKey) {
|
|
1544
|
-
return this.db.prepare(`
|
|
1545
|
-
SELECT * FROM observations
|
|
1546
|
-
WHERE project = ? AND semantic_key = ?
|
|
1547
|
-
ORDER BY CASE WHEN resolution_state = 'current' THEN 0 ELSE 1 END,
|
|
1548
|
-
COALESCE(explicit_accepted_at, timestamp) DESC, id DESC
|
|
1549
|
-
LIMIT 1
|
|
1550
|
-
`).get(project, semanticKey) ?? null;
|
|
1551
|
-
}
|
|
1552
|
-
findCurrentObservationByNarrative(project, narrative) {
|
|
1553
|
-
if (!narrative.trim())
|
|
1554
|
-
return null;
|
|
1555
|
-
return this.db.prepare(`
|
|
1556
|
-
SELECT * FROM observations
|
|
1557
|
-
WHERE project = ? AND narrative = ? AND resolution_state = 'current'
|
|
1558
|
-
AND quality_status = 'accepted'
|
|
1559
|
-
ORDER BY COALESCE(explicit_accepted_at, timestamp) DESC, id DESC
|
|
1560
|
-
LIMIT 1
|
|
1561
|
-
`).get(project, narrative) ?? null;
|
|
1562
|
-
}
|
|
1563
|
-
findCurrentObservationsByFacetKey(project, facetKey) {
|
|
1564
|
-
return this.db.prepare(`
|
|
1565
|
-
SELECT * FROM observations
|
|
1566
|
-
WHERE project = ? AND facet_key = ? AND resolution_state = 'current'
|
|
1567
|
-
AND quality_status = 'accepted'
|
|
1568
|
-
ORDER BY COALESCE(explicit_accepted_at, timestamp) DESC, id DESC
|
|
1569
|
-
`).all(project, facetKey);
|
|
1570
|
-
}
|
|
1571
1543
|
searchObservationEntities(project, entities, limit = 30) {
|
|
1572
1544
|
const normalized = uniqueStrings(entities.map(normalizeEntityKey)).filter(Boolean);
|
|
1573
1545
|
if (normalized.length === 0)
|
|
@@ -1640,22 +1612,6 @@ export class MemoryStore {
|
|
|
1640
1612
|
addSemanticObservationRelation(sourceObservationId, targetObservationId, type, createdAt = Date.now()) {
|
|
1641
1613
|
this.addObservationRelation(sourceObservationId, targetObservationId, type, createdAt);
|
|
1642
1614
|
}
|
|
1643
|
-
supersedeSemanticFacetConflicts(project, winnerId, semantic, now) {
|
|
1644
|
-
const conflicts = this.findCurrentObservationsByFacetKey(project, semantic.facetKey)
|
|
1645
|
-
.filter(candidate => candidate.id !== winnerId)
|
|
1646
|
-
.filter(candidate => semanticObservationsConflict(candidate, semantic));
|
|
1647
|
-
const supersededIds = [];
|
|
1648
|
-
for (const conflict of conflicts) {
|
|
1649
|
-
this.db.prepare(`
|
|
1650
|
-
UPDATE observations
|
|
1651
|
-
SET resolution_state = 'superseded', superseded_by_observation = ?, dedup_key = NULL, updated_at = ?
|
|
1652
|
-
WHERE id = ? AND resolution_state = 'current'
|
|
1653
|
-
`).run(winnerId, now, conflict.id);
|
|
1654
|
-
this.addObservationRelation(winnerId, conflict.id, 'supersedes', now);
|
|
1655
|
-
supersededIds.push(conflict.id);
|
|
1656
|
-
}
|
|
1657
|
-
return supersededIds;
|
|
1658
|
-
}
|
|
1659
1615
|
addObservationWithEmbedding(obs, embedding) {
|
|
1660
1616
|
return this.db.transaction(() => {
|
|
1661
1617
|
const row = this.addObservation(obs);
|
|
@@ -1663,230 +1619,13 @@ export class MemoryStore {
|
|
|
1663
1619
|
return this.getObservation(row.id);
|
|
1664
1620
|
})();
|
|
1665
1621
|
}
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
? normalizeSemanticMemory({
|
|
1674
|
-
project: input.project,
|
|
1675
|
-
type: input.enrichment.type,
|
|
1676
|
-
title: input.enrichment.title,
|
|
1677
|
-
narrative: input.enrichment.narrative,
|
|
1678
|
-
subject: baseSemantic.subject,
|
|
1679
|
-
predicate: baseSemantic.predicate,
|
|
1680
|
-
value: input.content,
|
|
1681
|
-
polarity: 'neutral',
|
|
1682
|
-
entities: baseSemantic.entities,
|
|
1683
|
-
temporal: baseSemantic.temporal ?? undefined,
|
|
1684
|
-
})
|
|
1685
|
-
: baseSemantic;
|
|
1686
|
-
const result = this.acceptSemanticObservation({
|
|
1687
|
-
sessionId: input.sessionId,
|
|
1688
|
-
project: input.project,
|
|
1689
|
-
type: input.enrichment.type,
|
|
1690
|
-
title: input.enrichment.title,
|
|
1691
|
-
narrative: input.enrichment.narrative,
|
|
1692
|
-
facts: input.enrichment.facts,
|
|
1693
|
-
files: input.enrichment.files,
|
|
1694
|
-
tools: input.enrichment.tools,
|
|
1695
|
-
concepts: input.enrichment.concepts,
|
|
1696
|
-
importance: 10,
|
|
1697
|
-
confidence: 0.95,
|
|
1698
|
-
source: 'explicit_intent',
|
|
1699
|
-
sourceEventIds: input.sourceEventId ? [input.sourceEventId] : [],
|
|
1700
|
-
qualityStatus: hold ? 'quarantined' : 'accepted',
|
|
1701
|
-
quarantineReason: hold ? 'ambiguous_explicit_preference' : undefined,
|
|
1702
|
-
evidenceStatus: 'verified',
|
|
1703
|
-
evidenceDetails: [
|
|
1704
|
-
...input.enrichment.evidence.map(item => ({ ...item })),
|
|
1705
|
-
...(input.enrichment.evidence.length === 0 ? [{
|
|
1706
|
-
kind: 'source_event',
|
|
1707
|
-
source: input.sourceEventId ?? `explicit:${input.sessionId}`,
|
|
1708
|
-
summary: 'Observation was accepted from a direct explicit-memory request.',
|
|
1709
|
-
}] : []),
|
|
1710
|
-
],
|
|
1711
|
-
semantic,
|
|
1712
|
-
});
|
|
1713
|
-
const latest = this.db.prepare(`
|
|
1714
|
-
SELECT MAX(COALESCE(explicit_accepted_at, timestamp)) AS latest
|
|
1715
|
-
FROM observations
|
|
1716
|
-
WHERE project = ? AND source = 'explicit_intent'
|
|
1717
|
-
`).get(input.project);
|
|
1718
|
-
const acceptedAt = Math.max(Date.now(), (latest.latest ?? 0) + 1);
|
|
1719
|
-
this.db.prepare(`
|
|
1720
|
-
UPDATE observations
|
|
1721
|
-
SET explicit_accepted_at = MAX(COALESCE(explicit_accepted_at, 0), ?), updated_at = MAX(updated_at, ?)
|
|
1722
|
-
WHERE id = ?
|
|
1723
|
-
`).run(acceptedAt, acceptedAt, result.row.id);
|
|
1724
|
-
if (input.sourceEventId)
|
|
1725
|
-
this.markEventsProcessed([input.sourceEventId]);
|
|
1726
|
-
return this.getObservation(result.row.id);
|
|
1727
|
-
}
|
|
1728
|
-
return this.db.transaction(() => {
|
|
1729
|
-
this.ensureSession(input.sessionId, input.project);
|
|
1730
|
-
const latest = this.db.prepare(`
|
|
1731
|
-
SELECT MAX(COALESCE(explicit_accepted_at, timestamp)) AS latest
|
|
1732
|
-
FROM observations
|
|
1733
|
-
WHERE project = ? AND type IN ('user_preference', 'project_convention', 'decision_made')
|
|
1734
|
-
AND source = 'explicit_intent'
|
|
1735
|
-
`).get(input.project);
|
|
1736
|
-
const acceptedAt = Math.max(Date.now(), (latest.latest ?? 0) + 1);
|
|
1737
|
-
const dedupKey = observationDedupKey('user_preference', input.title);
|
|
1738
|
-
const exact = this.db.prepare('SELECT * FROM observations WHERE dedup_key = ? AND project = ?').get(dedupKey, input.project) ?? null;
|
|
1739
|
-
let row;
|
|
1740
|
-
if (exact) {
|
|
1741
|
-
const contentChanged = exact.title !== input.title || exact.narrative !== input.content;
|
|
1742
|
-
const sourceEventIds = new Set(parseStringArray(exact.source_event_ids));
|
|
1743
|
-
if (input.sourceEventId)
|
|
1744
|
-
sourceEventIds.add(input.sourceEventId);
|
|
1745
|
-
this.db.prepare(`
|
|
1746
|
-
UPDATE observations
|
|
1747
|
-
SET title = ?, narrative = ?, source = 'explicit_intent',
|
|
1748
|
-
confidence = MAX(confidence, 0.95),
|
|
1749
|
-
importance = MAX(importance, 10), quality_status = 'accepted',
|
|
1750
|
-
quarantine_reason = NULL, quality_details = NULL, quarantined_at = NULL,
|
|
1751
|
-
resolution_state = 'current', superseded_by_observation = NULL,
|
|
1752
|
-
source_event_ids = ?, updated_at = ?, explicit_accepted_at = ?,
|
|
1753
|
-
embedding = CASE WHEN ? = 1 THEN NULL ELSE embedding END
|
|
1754
|
-
WHERE id = ?
|
|
1755
|
-
`).run(input.title, input.content, JSON.stringify([...sourceEventIds]), acceptedAt, acceptedAt, contentChanged ? 1 : 0, exact.id);
|
|
1756
|
-
if (contentChanged) {
|
|
1757
|
-
this.db.prepare("DELETE FROM embeddings WHERE source_type = 'observation' AND source_id = ?").run(exact.id);
|
|
1758
|
-
}
|
|
1759
|
-
this.incrementObservationAccess(exact.id, acceptedAt);
|
|
1760
|
-
row = this.getObservation(exact.id);
|
|
1761
|
-
}
|
|
1762
|
-
else {
|
|
1763
|
-
row = this.addObservation({
|
|
1764
|
-
sessionId: input.sessionId,
|
|
1765
|
-
project: input.project,
|
|
1766
|
-
type: 'user_preference',
|
|
1767
|
-
title: input.title,
|
|
1768
|
-
narrative: input.content,
|
|
1769
|
-
importance: 10,
|
|
1770
|
-
confidence: 0.95,
|
|
1771
|
-
source: 'explicit_intent',
|
|
1772
|
-
sourceEventIds: input.sourceEventId ? [input.sourceEventId] : [],
|
|
1773
|
-
timestamp: acceptedAt,
|
|
1774
|
-
qualityStatus: 'accepted',
|
|
1775
|
-
});
|
|
1776
|
-
this.db.prepare('UPDATE observations SET explicit_accepted_at = ? WHERE id = ?').run(acceptedAt, row.id);
|
|
1777
|
-
row = this.getObservation(row.id);
|
|
1778
|
-
}
|
|
1779
|
-
this.supersedeConflictingExplicitPreferences(row, analyzePreference(input.content));
|
|
1780
|
-
if (input.sourceEventId)
|
|
1781
|
-
this.markEventsProcessed([input.sourceEventId]);
|
|
1782
|
-
return this.getObservation(row.id);
|
|
1783
|
-
})();
|
|
1784
|
-
}
|
|
1785
|
-
applyExplicitMemoryEnrichment(observationId, enrichment) {
|
|
1786
|
-
return this.db.transaction(() => {
|
|
1787
|
-
const current = this.getObservation(observationId);
|
|
1788
|
-
if (!current)
|
|
1789
|
-
throw new Error(`Observation ${observationId} not found`);
|
|
1790
|
-
if (current.source !== 'explicit_intent') {
|
|
1791
|
-
throw new Error(`Observation ${observationId} is not an explicit memory`);
|
|
1792
|
-
}
|
|
1793
|
-
const dedupKey = enrichment.semanticKey;
|
|
1794
|
-
const collisions = this.db.prepare(`
|
|
1795
|
-
SELECT id, timestamp, explicit_accepted_at FROM observations
|
|
1796
|
-
WHERE project = ? AND dedup_key = ? AND id <> ?
|
|
1797
|
-
`).all(current.project, dedupKey, observationId);
|
|
1798
|
-
let dedupKeyForCurrent = dedupKey;
|
|
1799
|
-
if (collisions.length > 0) {
|
|
1800
|
-
const candidates = [
|
|
1801
|
-
{
|
|
1802
|
-
id: current.id,
|
|
1803
|
-
timestamp: current.timestamp,
|
|
1804
|
-
explicit_accepted_at: current.explicit_accepted_at,
|
|
1805
|
-
},
|
|
1806
|
-
...collisions,
|
|
1807
|
-
].sort((left, right) => (right.explicit_accepted_at ?? right.timestamp) - (left.explicit_accepted_at ?? left.timestamp)
|
|
1808
|
-
|| right.id.localeCompare(left.id));
|
|
1809
|
-
const winner = candidates[0];
|
|
1810
|
-
const now = Date.now();
|
|
1811
|
-
const supersede = this.db.prepare(`
|
|
1812
|
-
UPDATE observations
|
|
1813
|
-
SET resolution_state = 'superseded', superseded_by_observation = ?,
|
|
1814
|
-
dedup_key = NULL, updated_at = ?
|
|
1815
|
-
WHERE id = ?
|
|
1816
|
-
`);
|
|
1817
|
-
for (const candidate of candidates.slice(1)) {
|
|
1818
|
-
supersede.run(winner.id, now, candidate.id);
|
|
1819
|
-
}
|
|
1820
|
-
this.db.prepare(`
|
|
1821
|
-
UPDATE observations
|
|
1822
|
-
SET resolution_state = 'current', superseded_by_observation = NULL,
|
|
1823
|
-
dedup_key = ?, updated_at = ?
|
|
1824
|
-
WHERE id = ?
|
|
1825
|
-
`).run(dedupKey, now, winner.id);
|
|
1826
|
-
dedupKeyForCurrent = winner.id === current.id ? dedupKey : null;
|
|
1827
|
-
}
|
|
1828
|
-
const nextFacts = JSON.stringify(enrichment.facts);
|
|
1829
|
-
const nextFiles = JSON.stringify(enrichment.files);
|
|
1830
|
-
const nextTools = JSON.stringify(enrichment.tools);
|
|
1831
|
-
const nextConcepts = JSON.stringify(enrichment.concepts);
|
|
1832
|
-
const semanticContentChanged = current.type !== enrichment.type
|
|
1833
|
-
|| current.title !== enrichment.title
|
|
1834
|
-
|| current.narrative !== enrichment.narrative
|
|
1835
|
-
|| current.facts !== nextFacts
|
|
1836
|
-
|| current.concepts !== nextConcepts;
|
|
1837
|
-
const now = Date.now();
|
|
1838
|
-
this.db.prepare(`
|
|
1839
|
-
UPDATE observations
|
|
1840
|
-
SET type = ?, title = ?, narrative = ?, facts = ?, files = ?, tools = ?, concepts = ?,
|
|
1841
|
-
dedup_key = ?, evidence_status = ?, evidence_details = ?, updated_at = ?,
|
|
1842
|
-
embedding = CASE WHEN ? = 1 THEN NULL ELSE embedding END,
|
|
1843
|
-
semantic_subject = ?, semantic_predicate = ?, semantic_value = ?, semantic_polarity = ?,
|
|
1844
|
-
semantic_key = ?, facet_key = ?, entities = ?, temporal = ?
|
|
1845
|
-
WHERE id = ?
|
|
1846
|
-
`).run(enrichment.type, enrichment.title, enrichment.narrative, nextFacts, nextFiles, nextTools, nextConcepts, dedupKeyForCurrent, enrichment.evidenceStatus, JSON.stringify(enrichment.evidence), now, semanticContentChanged ? 1 : 0, enrichment.subject, enrichment.predicate, enrichment.value, enrichment.polarity, enrichment.semanticKey, enrichment.facetKey, JSON.stringify(enrichment.entities), enrichment.temporal ? JSON.stringify(enrichment.temporal) : null, observationId);
|
|
1847
|
-
if (semanticContentChanged) {
|
|
1848
|
-
this.db.prepare("DELETE FROM embeddings WHERE source_type = 'observation' AND source_id = ?").run(observationId);
|
|
1849
|
-
}
|
|
1850
|
-
this.db.prepare('DELETE FROM observation_concepts WHERE observation_id = ?').run(observationId);
|
|
1851
|
-
const insertConcept = this.db.prepare('INSERT OR IGNORE INTO observation_concepts (observation_id, concept) VALUES (?, ?)');
|
|
1852
|
-
for (const concept of enrichment.concepts)
|
|
1853
|
-
insertConcept.run(observationId, concept);
|
|
1854
|
-
this.replaceObservationEntities(observationId, current.project, enrichment.entities);
|
|
1855
|
-
this.reconcileExplicitPreferenceConflicts(current.project);
|
|
1856
|
-
this.bumpRetrievalRevision();
|
|
1857
|
-
return this.getObservation(observationId);
|
|
1858
|
-
})();
|
|
1859
|
-
}
|
|
1860
|
-
reconcileExplicitPreferenceConflicts(project = this.defaultProjectRoot) {
|
|
1861
|
-
return this.db.transaction(() => {
|
|
1862
|
-
const rows = this.listCurrentExplicitPreferences(project).sort((left, right) => this.explicitPreferenceAcceptanceTime(left)
|
|
1863
|
-
- this.explicitPreferenceAcceptanceTime(right)
|
|
1864
|
-
|| left.timestamp - right.timestamp
|
|
1865
|
-
|| left.id.localeCompare(right.id));
|
|
1866
|
-
let superseded = 0;
|
|
1867
|
-
const active = [];
|
|
1868
|
-
for (const row of rows) {
|
|
1869
|
-
const analysis = analyzePreferenceObservation(row);
|
|
1870
|
-
if (analysis.status !== 'high_confidence') {
|
|
1871
|
-
active.push({ row, analysis });
|
|
1872
|
-
continue;
|
|
1873
|
-
}
|
|
1874
|
-
for (let index = active.length - 1; index >= 0; index--) {
|
|
1875
|
-
const previous = active[index];
|
|
1876
|
-
if (!analysesConflict(previous.analysis, analysis))
|
|
1877
|
-
continue;
|
|
1878
|
-
this.db.prepare(`
|
|
1879
|
-
UPDATE observations
|
|
1880
|
-
SET resolution_state = 'superseded', superseded_by_observation = ?
|
|
1881
|
-
WHERE id = ? AND resolution_state = 'current'
|
|
1882
|
-
`).run(row.id, previous.row.id);
|
|
1883
|
-
active.splice(index, 1);
|
|
1884
|
-
superseded += 1;
|
|
1885
|
-
}
|
|
1886
|
-
active.push({ row, analysis });
|
|
1887
|
-
}
|
|
1888
|
-
return superseded;
|
|
1889
|
-
})();
|
|
1622
|
+
markObservationExplicitlyAccepted(observationId, acceptedAt = Date.now()) {
|
|
1623
|
+
this.db.prepare(`
|
|
1624
|
+
UPDATE observations
|
|
1625
|
+
SET explicit_accepted_at = MAX(COALESCE(explicit_accepted_at, 0), ?),
|
|
1626
|
+
updated_at = MAX(updated_at, ?)
|
|
1627
|
+
WHERE id = ? AND source = 'explicit_intent'
|
|
1628
|
+
`).run(acceptedAt, acceptedAt, observationId);
|
|
1890
1629
|
}
|
|
1891
1630
|
listCurrentExplicitPreferences(project = this.defaultProjectRoot) {
|
|
1892
1631
|
return this.db.prepare(`
|
|
@@ -1897,28 +1636,6 @@ export class MemoryStore {
|
|
|
1897
1636
|
ORDER BY COALESCE(explicit_accepted_at, timestamp) DESC, timestamp DESC, id DESC
|
|
1898
1637
|
`).all(project);
|
|
1899
1638
|
}
|
|
1900
|
-
explicitPreferenceAcceptanceTime(row) {
|
|
1901
|
-
return row.explicit_accepted_at ?? row.timestamp;
|
|
1902
|
-
}
|
|
1903
|
-
supersedeConflictingExplicitPreferences(winner, winnerAnalysis) {
|
|
1904
|
-
if (winnerAnalysis.status !== 'high_confidence')
|
|
1905
|
-
return 0;
|
|
1906
|
-
const update = this.db.prepare(`
|
|
1907
|
-
UPDATE observations
|
|
1908
|
-
SET resolution_state = 'superseded', superseded_by_observation = ?
|
|
1909
|
-
WHERE id = ? AND resolution_state = 'current'
|
|
1910
|
-
`);
|
|
1911
|
-
let superseded = 0;
|
|
1912
|
-
for (const candidate of this.listCurrentExplicitPreferences(winner.project)) {
|
|
1913
|
-
if (candidate.id === winner.id)
|
|
1914
|
-
continue;
|
|
1915
|
-
const analysis = analyzePreferenceObservation(candidate);
|
|
1916
|
-
if (!analysesConflict(analysis, winnerAnalysis))
|
|
1917
|
-
continue;
|
|
1918
|
-
superseded += update.run(winner.id, candidate.id).changes;
|
|
1919
|
-
}
|
|
1920
|
-
return superseded;
|
|
1921
|
-
}
|
|
1922
1639
|
getObservation(id) {
|
|
1923
1640
|
return this.db.prepare('SELECT * FROM observations WHERE id = ?').get(id) ?? null;
|
|
1924
1641
|
}
|
|
@@ -1956,13 +1673,14 @@ export class MemoryStore {
|
|
|
1956
1673
|
`).get(type, cutoff, buffer, threshold, buffer) ?? null;
|
|
1957
1674
|
}
|
|
1958
1675
|
incrementObservationAccess(id, now = Date.now()) {
|
|
1959
|
-
this.
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1676
|
+
const updated = this.db.prepare(`
|
|
1677
|
+
UPDATE observations
|
|
1678
|
+
SET access_count = access_count + 1, updated_at = ?
|
|
1679
|
+
WHERE id = ?
|
|
1680
|
+
`).run(now, id);
|
|
1681
|
+
if (updated.changes !== 1)
|
|
1682
|
+
throw new Error(`Observation ${id} not found`);
|
|
1683
|
+
this.bumpRetrievalRevision();
|
|
1966
1684
|
}
|
|
1967
1685
|
promoteObservationToExplicitIntent(id, now = Date.now()) {
|
|
1968
1686
|
this.db.prepare(`
|
|
@@ -2025,21 +1743,31 @@ export class MemoryStore {
|
|
|
2025
1743
|
}
|
|
2026
1744
|
listObservationsMissingEmbedding(limit = 32, afterId) {
|
|
2027
1745
|
return this.db.prepare(`
|
|
2028
|
-
SELECT
|
|
2029
|
-
|
|
2030
|
-
AND
|
|
2031
|
-
|
|
2032
|
-
|
|
1746
|
+
SELECT o.* FROM observations o
|
|
1747
|
+
LEFT JOIN embedding_profiles p ON p.source_type = 'observation' AND p.source_id = o.id
|
|
1748
|
+
AND p.profile_hash = ? AND p.generation = ?
|
|
1749
|
+
WHERE (o.embedding IS NULL OR p.source_id IS NULL) AND o.quality_status = 'accepted'
|
|
1750
|
+
AND o.resolution_state <> 'superseded' AND (? IS NULL OR o.id > ?)
|
|
1751
|
+
ORDER BY o.id LIMIT ?
|
|
1752
|
+
`).all(this.embeddingProfile.profileHash, this.embeddingProfile.generation, afterId ?? null, afterId ?? null, limit);
|
|
2033
1753
|
}
|
|
2034
1754
|
countObservationVectors() {
|
|
2035
1755
|
const row = this.db.prepare(`
|
|
2036
1756
|
SELECT COUNT(*) AS observations,
|
|
2037
1757
|
SUM(CASE WHEN quality_status = 'accepted' AND resolution_state <> 'superseded' THEN 1 ELSE 0 END) AS eligible,
|
|
2038
|
-
SUM(CASE WHEN quality_status = 'accepted' AND resolution_state <> 'superseded'
|
|
2039
|
-
|
|
1758
|
+
SUM(CASE WHEN quality_status = 'accepted' AND resolution_state <> 'superseded'
|
|
1759
|
+
AND embedding IS NOT NULL AND EXISTS (
|
|
1760
|
+
SELECT 1 FROM embedding_profiles p WHERE p.source_type = 'observation'
|
|
1761
|
+
AND p.source_id = observations.id AND p.profile_hash = ? AND p.generation = ?
|
|
1762
|
+
) THEN 1 ELSE 0 END) AS vectors,
|
|
1763
|
+
SUM(CASE WHEN quality_status = 'accepted' AND resolution_state <> 'superseded'
|
|
1764
|
+
AND (embedding IS NULL OR NOT EXISTS (
|
|
1765
|
+
SELECT 1 FROM embedding_profiles p WHERE p.source_type = 'observation'
|
|
1766
|
+
AND p.source_id = observations.id AND p.profile_hash = ? AND p.generation = ?
|
|
1767
|
+
)) THEN 1 ELSE 0 END) AS missing,
|
|
2040
1768
|
SUM(CASE WHEN quality_status = 'quarantined' THEN 1 ELSE 0 END) AS quarantined
|
|
2041
1769
|
FROM observations
|
|
2042
|
-
`).get();
|
|
1770
|
+
`).get(this.embeddingProfile.profileHash, this.embeddingProfile.generation, this.embeddingProfile.profileHash, this.embeddingProfile.generation);
|
|
2043
1771
|
return {
|
|
2044
1772
|
observations: row.observations,
|
|
2045
1773
|
eligible: row.eligible ?? 0,
|
|
@@ -2066,6 +1794,31 @@ export class MemoryStore {
|
|
|
2066
1794
|
WHERE id = ?
|
|
2067
1795
|
`).run(state, supersededBy ?? null, Date.now(), id);
|
|
2068
1796
|
}
|
|
1797
|
+
reinforceObservation(id, input = {}, now = Date.now()) {
|
|
1798
|
+
const current = this.getObservation(id);
|
|
1799
|
+
if (!current || current.resolution_state !== 'current') {
|
|
1800
|
+
throw new Error(`MEMORY_FACT_NOT_CURRENT:${id}`);
|
|
1801
|
+
}
|
|
1802
|
+
const mergedSourceEventIds = uniqueStrings([
|
|
1803
|
+
...parseStringArray(current.source_event_ids),
|
|
1804
|
+
...(input.sourceEventIds ?? []),
|
|
1805
|
+
]);
|
|
1806
|
+
const evidenceStatus = strongerEvidenceStatus(current.evidence_status, input.evidenceStatus);
|
|
1807
|
+
const evidenceDetails = mergeEvidenceDetails(current.evidence_details, input.evidenceDetails ?? []);
|
|
1808
|
+
this.db.prepare(`
|
|
1809
|
+
UPDATE observations
|
|
1810
|
+
SET source_event_ids = ?, reinforce_count = reinforce_count + 1,
|
|
1811
|
+
source = CASE WHEN ? = 'explicit_intent' THEN 'explicit_intent' ELSE source END,
|
|
1812
|
+
importance = MAX(importance, ?), confidence = MAX(confidence, ?),
|
|
1813
|
+
evidence_status = ?, evidence_details = ?,
|
|
1814
|
+
last_reinforced_at = ?, access_count = access_count + 1, updated_at = ?
|
|
1815
|
+
WHERE id = ? AND resolution_state = 'current'
|
|
1816
|
+
`).run(JSON.stringify(mergedSourceEventIds), input.source ?? current.source, input.importance ?? current.importance, input.confidence ?? current.confidence, evidenceStatus, JSON.stringify(evidenceDetails), now, now, id);
|
|
1817
|
+
for (const eventId of input.sourceEventIds ?? [])
|
|
1818
|
+
this.linkObservationToEvents(id, [eventId]);
|
|
1819
|
+
this.bumpRetrievalRevision();
|
|
1820
|
+
return this.getObservation(id);
|
|
1821
|
+
}
|
|
2069
1822
|
getObservationsByConcept(concept, limit = 50) {
|
|
2070
1823
|
return this.listObservations({ concept, limit });
|
|
2071
1824
|
}
|
|
@@ -2154,7 +1907,7 @@ export class MemoryStore {
|
|
|
2154
1907
|
const timestamp = input.timestamp ?? Date.now();
|
|
2155
1908
|
const id = `racc:${timestamp}:${randomUUID().slice(0, 8)}`;
|
|
2156
1909
|
this.db.transaction(() => {
|
|
2157
|
-
const accessType = input.accessType
|
|
1910
|
+
const accessType = input.accessType;
|
|
2158
1911
|
const inserted = this.db.prepare(`
|
|
2159
1912
|
INSERT OR IGNORE INTO retrieval_access_log
|
|
2160
1913
|
(id, source_type, source_id, access_type, project, session_id, request_id,
|
|
@@ -2162,7 +1915,7 @@ export class MemoryStore {
|
|
|
2162
1915
|
context_receipt, tool_evidence, outcome_reason, verification_receipt, idempotency_key)
|
|
2163
1916
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
2164
1917
|
`).run(id, input.sourceType, input.sourceId, accessType, input.project ?? this.defaultProjectRoot, input.sessionId ?? null, input.requestId ?? null, input.executionId ?? null, input.query ?? null, input.purpose ?? null, input.rank ?? null, input.score ?? null, timestamp, input.workflowRunId ?? null, input.contextReceipt ?? null, input.toolEvidence ? JSON.stringify([...new Set(input.toolEvidence)]) : null, input.outcomeReason ?? null, input.verificationReceipt ?? null, input.idempotencyKey ?? null).changes > 0;
|
|
2165
|
-
const reinforces = accessType === '
|
|
1918
|
+
const reinforces = accessType === 'consumed' && inserted;
|
|
2166
1919
|
if (input.sourceType === 'memory' && reinforces) {
|
|
2167
1920
|
const updated = this.db.prepare(`
|
|
2168
1921
|
UPDATE memories SET access_count = access_count + 1,
|
|
@@ -2198,21 +1951,17 @@ export class MemoryStore {
|
|
|
2198
1951
|
`).get(sourceType, sourceId);
|
|
2199
1952
|
return { positive: Number(row.positive ?? 0), negative: Number(row.negative ?? 0) };
|
|
2200
1953
|
}
|
|
2201
|
-
recordAccess(memoryId, accessType, sessionId) {
|
|
2202
|
-
this.recordRetrievalAccess({
|
|
2203
|
-
sourceType: 'memory',
|
|
2204
|
-
sourceId: memoryId,
|
|
2205
|
-
accessType,
|
|
2206
|
-
sessionId,
|
|
2207
|
-
});
|
|
2208
|
-
}
|
|
2209
1954
|
incrementAccess(id) {
|
|
2210
|
-
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
1955
|
+
const now = Date.now();
|
|
1956
|
+
const updated = this.db.prepare(`
|
|
1957
|
+
UPDATE memories SET access_count = access_count + 1,
|
|
1958
|
+
last_reinforced_at = COALESCE(last_reinforced_at, ?),
|
|
1959
|
+
last_accessed_at = ?, updated_at = ?
|
|
1960
|
+
WHERE id = ?
|
|
1961
|
+
`).run(now, now, now, id);
|
|
1962
|
+
if (updated.changes !== 1)
|
|
1963
|
+
throw new Error(`Memory ${id} not found`);
|
|
1964
|
+
this.bumpRetrievalRevision();
|
|
2216
1965
|
}
|
|
2217
1966
|
listRetrievalAccessLog(limit = 100) {
|
|
2218
1967
|
return this.db.prepare(`
|
|
@@ -2464,26 +2213,6 @@ function uniqueStrings(values) {
|
|
|
2464
2213
|
}
|
|
2465
2214
|
return result;
|
|
2466
2215
|
}
|
|
2467
|
-
function semanticRichness(value) {
|
|
2468
|
-
const exactIdentifiers = value.match(/@[a-z0-9._-]+\/[a-z0-9._-]+|--[a-z0-9-]+|[a-z0-9]+(?:[-./][a-z0-9]+)+/giu)?.length ?? 0;
|
|
2469
|
-
return [...value.normalize('NFKC').trim()].length + exactIdentifiers * 40;
|
|
2470
|
-
}
|
|
2471
|
-
function semanticObservationsConflict(existing, incoming) {
|
|
2472
|
-
if (existing.semantic_key === incoming.semanticKey)
|
|
2473
|
-
return false;
|
|
2474
|
-
if (existing.semantic_polarity === 'neutral' || incoming.polarity === 'neutral')
|
|
2475
|
-
return false;
|
|
2476
|
-
const existingValue = normalizeEntityKey(existing.semantic_value);
|
|
2477
|
-
const incomingValue = normalizeEntityKey(incoming.value);
|
|
2478
|
-
if (existingValue === incomingValue)
|
|
2479
|
-
return existing.semantic_polarity !== incoming.polarity;
|
|
2480
|
-
const subject = normalizeEntityKey(incoming.subject);
|
|
2481
|
-
if (new Set(['package manager', 'test runner', 'build tool']).has(subject))
|
|
2482
|
-
return true;
|
|
2483
|
-
if (incoming.predicate === 'version')
|
|
2484
|
-
return true;
|
|
2485
|
-
return ['use', 'prefer', 'require'].includes(incoming.predicate);
|
|
2486
|
-
}
|
|
2487
2216
|
function strongerEvidenceStatus(existing, incoming) {
|
|
2488
2217
|
return existing === 'verified' || incoming === 'verified' ? 'verified' : 'unverified';
|
|
2489
2218
|
}
|
|
@@ -2507,19 +2236,6 @@ function mergeEvidenceDetails(existing, incoming) {
|
|
|
2507
2236
|
}
|
|
2508
2237
|
return result.slice(0, 64);
|
|
2509
2238
|
}
|
|
2510
|
-
function semanticFromExplicitEnrichment(enrichment) {
|
|
2511
|
-
return {
|
|
2512
|
-
subject: enrichment.subject,
|
|
2513
|
-
predicate: enrichment.predicate,
|
|
2514
|
-
value: enrichment.value,
|
|
2515
|
-
polarity: enrichment.polarity,
|
|
2516
|
-
entities: enrichment.entities,
|
|
2517
|
-
facet: enrichment.facet,
|
|
2518
|
-
temporal: enrichment.temporal,
|
|
2519
|
-
semanticKey: enrichment.semanticKey,
|
|
2520
|
-
facetKey: enrichment.facetKey,
|
|
2521
|
-
};
|
|
2522
|
-
}
|
|
2523
2239
|
function camelToSnake(str) {
|
|
2524
2240
|
return str.replace(/[A-Z]/g, letter => `_${letter.toLowerCase()}`);
|
|
2525
2241
|
}
|