@devflow-tools/memory-engine 0.17.0 → 0.17.2

package/dist/memory-gate.js

@@ -12,7 +12,8 @@ import { join } from 'node:path';
 import { isObservationType, observationDedupKey, } from './observation-types.js';
 import { evaluateObservationQuality } from './observation-quality.js';
 import { mergeSemanticEntities, normalizeEntityKey, normalizeSemanticMemory, } from './semantic-memory.js';
-import { validateMemoryCapsules } from './memory-capsule.js';
+import { validateEvidenceSpans, validateMemoryCapsules } from './memory-capsule.js';
+import { emitSemanticControlLog } from '@devflow-tools/telemetry';
 const BATCH_CONCEPT_STOP_WORDS = new Set([
     'a', 'an', 'the', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
     'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'should',
@@ -180,10 +181,12 @@ export class MemoryGate {
         if (this.engine && signals.length > 0) {
             try {
                 this.engine.recordSignals(signals);
-                console.log('[MemoryGate] Signals detected', {
-                    count: signals.length,
-                    types: signals.map(s => s.type),
-                    sessionId: event.sessionId,
+                emitSemanticControlLog({
+                    event: 'memory.write.completed',
+                    identity: { projectRoot: this.projectRoot, sessionId: event.sessionId },
+                    level: 'debug',
+                    timestamp: Date.now(),
+                    data: { stage: 'signals', count: signals.length, types: signals.map(signal => signal.type) },
                 });
             }
             catch { /* non-critical */ }
@@ -263,9 +266,9 @@ export class MemoryGate {
                 .filter(event => sessionId === undefined || event.session_id === sessionId);
             if (events.length === 0)
                 return null;
-            const groups = groupEventsBySession(events);
+            const groups = groupEventsBySession(events, { projectRoot: this.projectRoot });
             const preserveSessionIds = this.getActiveSessionIds(groups);
-            const { selected, discarded } = triageGroupsWithDisposition(groups, 8, { preserveSessionIds });
+            const { selected, discarded } = triageGroupsWithDisposition(groups, 8, { preserveSessionIds, projectRoot: this.projectRoot });
             const discardedEventIds = discarded.flatMap(group => group.events).map(event => event.id);
             if (discardedEventIds.length > 0) {
                 const discardedSessionIds = [...new Set(discarded.flatMap(group => group.events).map(event => event.session_id).filter(Boolean))];
@@ -452,10 +455,12 @@ export class MemoryGate {
             leases.commitLease(batchId);
         });
         transaction();
-        console.log('[MemoryGate] Distilled observations saved by batch', {
-            batchId,
-            observationCount: validObs.length,
-            eventsMarked: lease.eventIds.length,
+        emitSemanticControlLog({
+            event: 'memory.write.completed',
+            identity: { projectRoot: this.projectRoot, sessionId: lease.sessionIds[0] },
+            level: 'info',
+            timestamp: Date.now(),
+            data: { stage: 'distill_batch', batchId, observationCount: validObs.length, eventsMarked: lease.eventIds.length },
         });
         return validObs.length;
     }
@@ -503,6 +508,7 @@ export class MemoryGate {
             observations: validObs,
             sourceEvents: [event],
             defaultSourceEventIds: [turnEventId],
+            evidenceText: readCanonicalPrompt(event.payload, event.user_message) || undefined,
         });
         if (capsules.rejected.length > 0 || capsules.accepted.length !== validObs.length) {
             throw new Error(`memory_commit_turn failed semantic capsule validation: ${JSON.stringify(capsules.rejected)}`);
@@ -579,6 +585,156 @@ export class MemoryGate {
         })();
         return saved;
     }
+    async refineExplicitTurnObservations(turnEventId, sessionId, observations) {
+        if (!this.engine)
+            await this.forceWarmUp();
+        const engine = this.requireEngine();
+        const event = engine.store.getEvent(turnEventId);
+        if (!event || event.session_id !== sessionId) {
+            throw new Error(`Memory turn event ${turnEventId} was not found for session ${sessionId}`);
+        }
+        const sourceText = readCanonicalPrompt(event.payload, event.user_message);
+        if (!sourceText)
+            throw new Error(`Memory turn event ${turnEventId} has no canonical user prompt`);
+        for (const observation of observations) {
+            validateEvidenceSpans(sourceText, observation.evidenceSpans ?? []);
+        }
+        const durableCandidates = observations.filter(observation => observation.disposition !== 'turn_constraint');
+        const turnConstraints = observations.filter(observation => observation.disposition === 'turn_constraint');
+        const validObs = durableCandidates.length > 0 ? validateObservationBatch(durableCandidates) : [];
+        if (!validObs)
+            throw new Error('Explicit memory refinement contains invalid durable candidates');
+        const capsules = validateMemoryCapsules({
+            projectRoot: this.projectRoot,
+            observations: validObs,
+            sourceEvents: [event],
+            defaultSourceEventIds: [turnEventId],
+            evidenceText: sourceText,
+            requireEvidenceSpans: true,
+        });
+        if (capsules.rejected.length > 0 || capsules.accepted.length !== validObs.length) {
+            throw new Error(`Explicit memory refinement failed evidence validation: ${JSON.stringify(capsules.rejected)}`);
+        }
+        const existingIds = new Set(engine.store.listObservations().map(observation => observation.id));
+        const turnId = eventTurnId(event, turnEventId);
+        const memories = [];
+        const retiredMemoryIds = [];
+        const operations = [];
+        this.getStoreDb().transaction(() => {
+            for (const acceptedCapsule of capsules.accepted) {
+                const observation = acceptedCapsule.observation;
+                const semantic = normalizeSemanticMemory({
+                    project: this.projectRoot,
+                    type: observation.type,
+                    title: observation.title,
+                    narrative: observation.narrative ?? '',
+                    concepts: observation.concepts,
+                    subject: observation.subject,
+                    predicate: observation.predicate,
+                    value: observation.value,
+                    polarity: observation.polarity,
+                    entities: observation.entities,
+                    temporal: observation.temporal,
+                });
+                const write = engine.store.acceptSemanticObservation({
+                    sessionId,
+                    project: this.projectRoot,
+                    type: observation.type,
+                    title: observation.title,
+                    narrative: observation.narrative,
+                    facts: [...new Set([...(observation.facts ?? []), ...(observation.evidenceSpans ?? [])])],
+                    concepts: mergeSemanticEntities(normalizeBatchConcepts(observation), [semantic.subject, semantic.predicate, semantic.value], semantic.entities.map(normalizeEntityKey)),
+                    importance: Math.max(8, observation.importance),
+                    confidence: Math.max(0.9, observation.confidence),
+                    source: 'explicit_intent',
+                    sourceEventIds: [turnEventId],
+                    qualityStatus: 'accepted',
+                    evidenceStatus: 'verified',
+                    evidenceDetails: (observation.evidenceSpans ?? []).map(span => ({
+                        kind: 'host_evidence_span',
+                        source: turnEventId,
+                        summary: span,
+                    })),
+                    semantic,
+                    capsule: acceptedCapsule.capsule,
+                    projection: acceptedCapsule.projection,
+                    preferIncomingNarrative: true,
+                });
+                const row = write.row;
+                operations.push(write.action.toLocaleLowerCase('en-US'));
+                memories.push({
+                    id: row.id,
+                    type: row.type,
+                    title: row.title,
+                    content: row.narrative,
+                    source: row.source,
+                    scope: row.project,
+                    confidence: row.confidence,
+                    importance: row.importance,
+                    score: 1,
+                    createdAt: row.timestamp,
+                    indexStatus: row.embedding ? 'indexed' : 'pending',
+                });
+                engine.store.recordPendingTurnObservation({
+                    turnEventId,
+                    turnId,
+                    sessionId,
+                    project: this.projectRoot,
+                    observationId: row.id,
+                    createdByTurn: !existingIds.has(row.id),
+                    createdAt: event.created_at,
+                });
+            }
+            const constraintSpans = turnConstraints.flatMap(candidate => candidate.evidenceSpans ?? []);
+            const baseline = engine.store.listObservations({ project: this.projectRoot })
+                .filter(observation => observation.source === 'explicit_intent')
+                .filter(observation => parseLearningStringArray(observation.source_event_ids).includes(turnEventId));
+            const refinedIds = new Set(memories.map(memory => memory.id));
+            for (const observation of baseline) {
+                if (refinedIds.has(observation.id) || observation.resolution_state === 'superseded')
+                    continue;
+                const replacementIndex = durableCandidates.findIndex(candidate => ((candidate.evidenceSpans ?? []).some(span => sameGroundedClause(observation.narrative, span))));
+                const replacement = replacementIndex >= 0
+                    ? memories[replacementIndex]
+                    : durableCandidates.length === 1 ? memories[0] : undefined;
+                if (!replacement || replacement.id === observation.id)
+                    continue;
+                engine.store.setObservationResolution(observation.id, 'superseded', replacement.id);
+                engine.store.addSemanticObservationRelation(replacement.id, observation.id, 'supersedes');
+                retiredMemoryIds.push(observation.id);
+                operations.push('supersede');
+            }
+            for (const observation of baseline) {
+                if (observation.resolution_state === 'superseded' || retiredMemoryIds.includes(observation.id))
+                    continue;
+                if (!constraintSpans.some(span => sameGroundedClause(observation.narrative, span)))
+                    continue;
+                if (engine.store.quarantineObservation(observation.id, 'host_classified_turn_constraint', {
+                    turnId,
+                    eventId: turnEventId,
+                    evidenceSpans: constraintSpans,
+                })) {
+                    retiredMemoryIds.push(observation.id);
+                    operations.push('retire');
+                }
+            }
+        })();
+        emitSemanticControlLog({
+            event: 'memory.semantic.refined',
+            identity: { projectRoot: this.projectRoot, sessionId, turnId },
+            level: 'info',
+            timestamp: Date.now(),
+            data: {
+                candidateCount: observations.length,
+                durableCount: durableCandidates.length,
+                turnConstraintCount: turnConstraints.length,
+                memoryIds: memories.map(memory => memory.id),
+                retiredMemoryIds,
+                operations,
+            },
+        });
+        return { memories, retiredMemoryIds, operations };
+    }
     async skipTurnEvent(turnEventId, sessionId, reason) {
         if (!this.engine)
             await this.forceWarmUp();
@@ -639,7 +795,13 @@ export class MemoryGate {
      * @deprecated Use saveDistilledObservationsByBatch(batchId, observations) instead.
      */
     async saveDistilledObservations(observations, sessionId) {
-        console.warn('[MemoryGate] saveDistilledObservations by sessionId is deprecated; use saveDistilledObservationsByBatch instead');
+        emitSemanticControlLog({
+            event: 'memory.write.degraded',
+            identity: { projectRoot: this.projectRoot, sessionId },
+            level: 'warn',
+            timestamp: Date.now(),
+            data: { stage: 'legacy_distill', reason: 'deprecated_session_write', fallback: 'legacy_compatibility_path' },
+        });
         if (!this.engine)
             return 0;
         const validObs = validateObservationBatch(observations);
@@ -684,10 +846,12 @@ export class MemoryGate {
                 engine.store.markEventsProcessed(unprocessedIds);
         });
         transaction();
-        console.log('[MemoryGate] Distilled observations saved', {
-            sessionId,
-            observationCount: validObs.length,
-            eventsMarked: unprocessedIds.length,
+        emitSemanticControlLog({
+            event: 'memory.write.completed',
+            identity: { projectRoot: this.projectRoot, sessionId },
+            level: 'info',
+            timestamp: Date.now(),
+            data: { stage: 'legacy_distill', observationCount: validObs.length, eventsMarked: unprocessedIds.length },
         });
         return validObs.length;
     }
@@ -716,11 +880,12 @@ export class MemoryGate {
         const events = this.engine.store.getUnprocessedEvents();
         if (events.length === 0)
             return 0;
-        const groups = groupEventsBySession(events, { log: false });
+        const groups = groupEventsBySession(events, { log: false, projectRoot: this.projectRoot });
         const preserveSessionIds = this.getActiveSessionIds(groups);
         const { discarded } = triageGroupsWithDisposition(groups, Number.MAX_SAFE_INTEGER, {
             log: false,
             preserveSessionIds,
+            projectRoot: this.projectRoot,
         });
         const ids = discarded.flatMap(group => group.events).map(event => event.id);
         return this.engine.store.discardEvents(ids, 'low_value_auto_triage', `auto-triage:${Date.now()}`);
@@ -753,7 +918,10 @@ export class MemoryGate {
             this.projectValid = true;
         }
         this.state = 'warming';
-        console.log('[MemoryGate] State transition', { from: 'cold', to: 'warming' });
+        emitSemanticControlLog({
+            event: 'memory.engine.transitioned', identity: { projectRoot: this.projectRoot }, level: 'info', timestamp: Date.now(),
+            data: { from: 'cold', to: 'warming' },
+        });
         this.engine = new MemoryEngine(this.projectRoot, this.embedder);
         this.setLeaseStore(new LeaseStore(this.getStoreDb()));
         // Clean up zombie sessions (>24h active)
@@ -761,7 +929,13 @@ export class MemoryGate {
             const zombies = this.engine.store.getZombieSessions(24);
             for (const z of zombies) {
                 this.engine.store.closeSession(z.id);
-                console.log('[MemoryGate] Zombie session auto-closed', { sessionId: z.id, age: `${Math.round((Date.now() - z.created_at) / 3600000)}h` });
+                emitSemanticControlLog({
+                    event: 'memory.engine.transitioned',
+                    identity: { projectRoot: this.projectRoot, sessionId: z.id },
+                    level: 'warn',
+                    timestamp: Date.now(),
+                    data: { from: 'active', to: 'closed', reason: 'zombie_session', ageHours: Math.round((Date.now() - z.created_at) / 3600000) },
+                });
             }
         }
         catch { /* non-critical */ }
@@ -771,7 +945,10 @@ export class MemoryGate {
         }
         catch { /* non-critical */ }
         this.state = 'hot';
-        console.log('[MemoryGate] State transition', { from: 'warming', to: 'hot' });
+        emitSemanticControlLog({
+            event: 'memory.engine.transitioned', identity: { projectRoot: this.projectRoot }, level: 'info', timestamp: Date.now(),
+            data: { from: 'warming', to: 'hot' },
+        });
     }
     // --- IMemoryEngine implementation (delegates to engine when hot) ---
     requireEngine() {
@@ -854,7 +1031,28 @@ export class MemoryGate {
     async saveExplicitMemoryIntent(content, sessionId, sourceEventId) {
         if (!this.engine)
             await this.forceWarmUp();
-        return this.requireEngine().saveExplicitMemoryIntent(content, sessionId, sourceEventId);
+        try {
+            const result = await this.requireEngine().saveExplicitMemoryIntent(content, sessionId, sourceEventId);
+            emitSemanticControlLog({
+                event: 'memory.write.completed',
+                identity: { projectRoot: this.projectRoot, sessionId },
+                level: 'info',
+                timestamp: Date.now(),
+                data: {
+                    stage: 'explicit_memory', memoryId: result.id, relatedMemoryIds: result.relatedMemoryIds ?? [],
+                    semanticStatus: result.semanticDecision?.status,
+                    operations: result.semanticDecision?.operations ?? [],
+                },
+            });
+            return result;
+        }
+        catch (error) {
+            emitSemanticControlLog({
+                event: 'memory.write.degraded', identity: { projectRoot: this.projectRoot, sessionId }, level: 'error', timestamp: Date.now(),
+                data: { stage: 'explicit_memory', reason: memoryErrorReason(error) },
+            });
+            throw error;
+        }
     }
     async enrichExplicitMemoryObservation(observationId) {
         if (!this.engine)
@@ -924,15 +1122,28 @@ export class MemoryGate {
             const rows = engine.store.listObservationsMissingEmbedding(batchSize, lastId);
             if (rows.length === 0)
                 break;
-            const embeddings = await this.embedder.embedBatch(rows.map(row => `${row.title}\n${row.narrative}`));
-            if (embeddings.length !== rows.length) {
-                throw new Error(`Observation vector count mismatch: ${rows.length} observations, ${embeddings.length} vectors`);
+            const texts = rows.map(row => `${row.title}\n${row.narrative}`);
+            let indexed = [];
+            try {
+                const embeddings = await this.embedder.embedBatch(texts);
+                if (embeddings.length !== rows.length) {
+                    throw new Error(`Observation vector count mismatch: ${rows.length} observations, ${embeddings.length} vectors`);
+                }
+                indexed = rows.map((row, index) => ({ id: row.id, embedding: embeddings[index] }));
+            }
+            catch {
+                for (let index = 0; index < rows.length; index++) {
+                    try {
+                        indexed.push({ id: rows[index].id, embedding: await this.embedder.embed(texts[index]) });
+                    }
+                    catch {
+                        // Keep this observation pending; deterministic retrieval remains available.
+                    }
+                }
             }
-            engine.store.insertObservationEmbeddings(rows.map((row, index) => ({
-                id: row.id,
-                embedding: embeddings[index],
-            })));
-            processed += rows.length;
+            if (indexed.length > 0)
+                engine.store.insertObservationEmbeddings(indexed);
+            processed += indexed.length;
             lastId = rows.at(-1).id;
             const coverage = engine.store.countObservationVectors();
             options.onProgress?.({ processed, remaining: coverage.missing, lastId });
@@ -964,7 +1175,38 @@ export class MemoryGate {
     async search(query, options) {
         if (!this.engine)
             await this.forceWarmUp();
-        return this.requireEngine().search(query, options);
+        const startedAt = Date.now();
+        try {
+            const results = await this.requireEngine().search(query, options);
+            emitSemanticControlLog({
+                event: 'memory.search.completed',
+                identity: {
+                    projectRoot: this.projectRoot,
+                    sessionId: options?.sessionId,
+                    executionId: options?.executionId,
+                    requestId: options?.requestId,
+                },
+                level: 'info',
+                timestamp: Date.now(),
+                data: {
+                    status: results.length > 0 ? 'selected' : 'empty', selected: results.length,
+                    verified: results.filter(result => result.entry.status === 'approved').length,
+                    durationMs: Date.now() - startedAt,
+                    vectorReady: results.filter(result => (result.scoreDetails?.semanticScore ?? 0) > 0).length,
+                },
+            });
+            return results;
+        }
+        catch (error) {
+            emitSemanticControlLog({
+                event: 'memory.search.completed',
+                identity: { projectRoot: this.projectRoot, sessionId: options?.sessionId, executionId: options?.executionId, requestId: options?.requestId },
+                level: 'error',
+                timestamp: Date.now(),
+                data: { status: 'failed', selected: 0, durationMs: Date.now() - startedAt, reason: memoryErrorReason(error) },
+            });
+            throw error;
+        }
     }
     async listMemories(filter) {
         if (!this.engine)
@@ -1070,6 +1312,24 @@ function eventTurnId(event, fallbackEventId) {
     }
     return `turn:${fallbackEventId}`;
 }
+function readCanonicalPrompt(payloadJson, userMessage) {
+    if (payloadJson) {
+        try {
+            const payload = JSON.parse(payloadJson);
+            if (typeof payload.prompt === 'string' && payload.prompt.trim())
+                return payload.prompt.trim();
+        }
+        catch { }
+    }
+    return userMessage?.trim() ?? '';
+}
+function sameGroundedClause(narrative, evidenceSpan) {
+    const left = narrative.normalize('NFKC').trim().toLocaleLowerCase('en-US');
+    const right = evidenceSpan.normalize('NFKC').trim().toLocaleLowerCase('en-US');
+    if (left.length < 4 || right.length < 4)
+        return false;
+    return left === right || (right.includes(left) && left.length / right.length >= 0.7);
+}
 // --- L1 Rule Scorer (backward compat) ---
 function scoreL1Rules(event) {
     let score = 0;
@@ -1131,5 +1391,8 @@ function scoreDialogSignals(message) {
     }
     return maxScore;
 }
+function memoryErrorReason(error) {
+    return (error instanceof Error ? error.message : String(error)).trim().slice(0, 500) || 'memory_operation_failed';
+}
 export { scoreL1Rules, scoreDialogSignals };
 //# sourceMappingURL=memory-gate.js.map