dsh-tdai-memory 0.1.0

package/vendor/tdai/core/record/l1-dedup.js

@@ -0,0 +1,318 @@
+/**
+ * L1 Memory Conflict Detection (Batch Mode): decides how to handle multiple new
+ * memories against existing records in a single LLM call.
+ *
+ * v4: Removed JSONL-based Jaccard fallback. Candidate recall now relies exclusively
+ *     on vector search (primary) and FTS5 BM25 (degraded). If neither is available,
+ *     conflict detection is skipped entirely — all memories go straight to store.
+ *
+ * Two-phase approach:
+ * 1. Candidate search per new memory — vector recall or FTS5 keyword recall (fast, no LLM)
+ * 2. Batch LLM judgment on all new memories + their candidate pools (single call)
+ */
+import { CONFLICT_DETECTION_SYSTEM_PROMPT, formatBatchConflictPrompt } from "../prompts/l1-dedup.js";
+import { CleanContextRunner } from "../../utils/clean-context-runner.js";
+import { sanitizeJsonForParse } from "../../utils/sanitize.js";
+import { buildFtsQuery } from "../store/sqlite.js";
+const TAG = "[memory-tdai][l1-dedup]";
+// ============================
+// Core function (batch mode)
+// ============================
+/**
+ * Batch conflict detection: compare all new memories against existing records
+ * in a single LLM call.
+ *
+ * Candidate recall strategy (3-tier degradation):
+ * 1. Vector recall (vectorStore + embeddingService) — cosine similarity (best)
+ * 2. FTS5 keyword recall (vectorStore with FTS available) — BM25 ranking (degraded)
+ * 3. Skip conflict detection entirely — all memories go straight to "store"
+ *
+ * The old JSONL-based Jaccard fallback has been removed. If neither vector search
+ * nor FTS is available, we skip dedup rather than paying the O(N) full-file-scan cost.
+ *
+ * @param memories - Newly extracted memories (with record_id)
+ * @param config - OpenClaw config (for LLM access)
+ * @param logger - Optional logger
+ * @param model - Optional model override
+ * @param vectorStore - Optional vector store for cosine similarity search
+ * @param embeddingService - Optional embedding service for computing query vectors
+ * @param conflictRecallTopK - Top-K candidates to recall per new memory (default: 5)
+ * @returns Array of dedup decisions, one per new memory
+ */
+export async function batchDedup(params) {
+    const { memories, config, logger, model, vectorStore, embeddingService, llmRunner } = params;
+    const topK = params.conflictRecallTopK ?? 5;
+    if (memories.length === 0) {
+        return [];
+    }
+    const storeAll = () => memories.map((m) => ({
+        record_id: m.record_id,
+        action: "store",
+        target_ids: [],
+    }));
+    // Determine what recall capabilities are available
+    const hasVectorData = vectorStore && (await vectorStore.countL1()) > 0;
+    const hasFts = vectorStore?.isFtsAvailable() ?? false;
+    // Fast path: no recall capability at all → skip dedup
+    if (!hasVectorData && !hasFts) {
+        logger?.debug?.(`${TAG} No vector data and no FTS available, skipping conflict detection for ${memories.length} memories`);
+        return storeAll();
+    }
+    // Phase 1: Find candidates
+    //
+    // Decision tree (after the fast-path guard above, vectorStore is guaranteed non-null):
+    //   hasVectorData + embeddingService → Tier 1 vector recall (FTS fallback on error)
+    //   otherwise hasFts                → Tier 2 FTS keyword recall
+    //   otherwise                       → skip dedup (defensive; shouldn't reach here)
+    let matches;
+    if (hasVectorData && embeddingService) {
+        // === Tier 1: Vector recall mode ===
+        logger?.debug?.(`${TAG} Using vector recall mode (topK=${topK})`);
+        try {
+            matches = await findCandidatesByVector(memories, vectorStore, embeddingService, topK, logger, params.embeddingTimeoutMs);
+        }
+        catch (err) {
+            logger?.warn?.(`${TAG} Vector recall failed, falling back to FTS keyword: ${err instanceof Error ? err.message : String(err)}`);
+            // Degrade to FTS keyword recall
+            if (hasFts) {
+                matches = await findCandidatesByFts(memories, vectorStore, logger);
+            }
+            else {
+                logger?.debug?.(`${TAG} FTS not available either, skipping conflict detection`);
+                return storeAll();
+            }
+        }
+    }
+    else if (hasFts) {
+        // === Tier 2: FTS keyword recall ===
+        logger?.debug?.(`${TAG} Using FTS keyword recall mode (no embedding service or no vector data)`);
+        matches = await findCandidatesByFts(memories, vectorStore, logger);
+    }
+    else {
+        // Shouldn't reach here given the fast-path check above, but be defensive
+        logger?.debug?.(`${TAG} No usable recall path, skipping conflict detection`);
+        return storeAll();
+    }
+    // Check if any memory has candidates
+    const hasAnyCandidates = matches.some((m) => m.candidates.length > 0);
+    if (!hasAnyCandidates) {
+        logger?.debug?.(`${TAG} No similar records found for any memory, all will be stored`);
+        return storeAll();
+    }
+    // Phase 2: Batch LLM judgment
+    return runLlmJudgment(matches, memories, config, logger, model, llmRunner);
+}
+/**
+ * Phase 2: Run batch LLM judgment on candidate matches.
+ */
+async function runLlmJudgment(matches, memories, config, logger, model, llmRunner) {
+    logger?.debug?.(`${TAG} Running batch conflict detection for ${memories.length} memories`);
+    try {
+        const userPrompt = formatBatchConflictPrompt(matches);
+        let result;
+        if (llmRunner) {
+            // Use the host-neutral LLMRunner interface
+            result = await llmRunner.run({
+                prompt: userPrompt,
+                systemPrompt: CONFLICT_DETECTION_SYSTEM_PROMPT,
+                taskId: "l1-conflict-detection",
+                timeoutMs: 180_000,
+            });
+        }
+        else {
+            // Fallback: create CleanContextRunner (OpenClaw path)
+            const runner = new CleanContextRunner({
+                config,
+                modelRef: model,
+                enableTools: false,
+                logger,
+            });
+            result = await runner.run({
+                prompt: userPrompt,
+                systemPrompt: CONFLICT_DETECTION_SYSTEM_PROMPT,
+                taskId: "l1-conflict-detection",
+                timeoutMs: 180_000,
+            });
+        }
+        const decisions = parseBatchResult(result, memories, logger);
+        return decisions;
+    }
+    catch (err) {
+        logger?.warn?.(`${TAG} Batch conflict detection failed, defaulting all to store: ${err instanceof Error ? err.message : String(err)}`);
+        return memories.map((m) => ({
+            record_id: m.record_id,
+            action: "store",
+            target_ids: [],
+        }));
+    }
+}
+// ============================
+// Candidate recall strategies
+// ============================
+/**
+ * Vector-based candidate recall (aligned with prototype):
+ * batch-embed new memories → cosine search in VectorStore → exclude self-batch → return candidates.
+ */
+async function findCandidatesByVector(memories, vectorStore, embeddingService, topK, logger, embeddingTimeoutMs) {
+    const newRecordIds = new Set(memories.map((m) => m.record_id));
+    // Batch-compute embeddings for all new memories
+    const texts = memories.map((m) => m.content);
+    const embeddings = await embeddingService.embedBatch(texts, embeddingTimeoutMs ? { timeoutMs: embeddingTimeoutMs } : undefined);
+    const matches = [];
+    for (let i = 0; i < memories.length; i++) {
+        const mem = memories[i];
+        const queryVec = embeddings[i];
+        // Vector search top-K (request extra to account for self-batch filtering)
+        const searchResults = await vectorStore.searchL1Vector(queryVec, topK + memories.length, mem.content);
+        // Exclude records from current batch, convert to MemoryRecord format
+        const candidates = searchResults
+            .filter((r) => !newRecordIds.has(r.record_id))
+            .slice(0, topK)
+            .map((r) => ({
+            id: r.record_id,
+            content: r.content,
+            type: r.type,
+            priority: r.priority,
+            scene_name: r.scene_name,
+            source_message_ids: [],
+            metadata: {},
+            timestamps: [r.timestamp_str].filter(Boolean),
+            createdAt: "",
+            updatedAt: "",
+            sessionKey: r.session_key,
+            sessionId: r.session_id,
+        }));
+        matches.push({ newMemory: mem, candidates });
+    }
+    logger?.debug?.(`${TAG} Vector recall: ${matches.map((m) => `${m.newMemory.record_id}→${m.candidates.length}`).join(", ")}`);
+    return matches;
+}
+/**
+ * FTS5-based candidate recall:
+ * Uses the FTS index for efficient BM25-ranked keyword matching.
+ * This replaces the old Jaccard word-overlap fallback entirely.
+ */
+async function findCandidatesByFts(memories, vectorStore, _logger) {
+    const newRecordIds = new Set(memories.map((m) => m.record_id));
+    const matches = [];
+    for (const mem of memories) {
+        const ftsQuery = buildFtsQuery(mem.content);
+        if (ftsQuery) {
+            const ftsResults = await vectorStore.searchL1Fts(ftsQuery, 10);
+            // Filter out records from the current batch
+            const candidates = ftsResults
+                .filter((r) => !newRecordIds.has(r.record_id))
+                .slice(0, 5)
+                .map((r) => ({
+                id: r.record_id,
+                content: r.content,
+                type: r.type,
+                priority: r.priority,
+                scene_name: r.scene_name,
+                source_message_ids: [],
+                metadata: r.metadata_json ? (() => { try {
+                    return JSON.parse(r.metadata_json);
+                }
+                catch {
+                    return {};
+                } })() : {},
+                timestamps: [r.timestamp_str].filter(Boolean),
+                createdAt: "",
+                updatedAt: "",
+                sessionKey: r.session_key,
+                sessionId: r.session_id,
+            }));
+            matches.push({ newMemory: mem, candidates });
+        }
+        else {
+            matches.push({ newMemory: mem, candidates: [] });
+        }
+    }
+    _logger?.debug?.(`${TAG} FTS keyword recall: ${matches.map((m) => `${m.newMemory.record_id}→${m.candidates.length}`).join(", ")}`);
+    return matches;
+}
+// ============================
+// Result parsing
+// ============================
+const VALID_TYPES = ["persona", "episodic", "instruction"];
+/**
+ * Parse the LLM's batch conflict detection JSON response.
+ *
+ * Expected format: [{record_id, action, target_ids, merged_content, merged_type, merged_priority, merged_timestamps}]
+ */
+function parseBatchResult(raw, memories, logger) {
+    try {
+        // Strip markdown code block wrappers
+        let cleaned = raw.trim();
+        if (cleaned.startsWith("```")) {
+            cleaned = cleaned.replace(/^```(?:json)?\s*\n?/, "").replace(/\n?```\s*$/, "");
+        }
+        // Extract JSON array
+        const arrayMatch = cleaned.match(/\[[\s\S]*\]/);
+        if (!arrayMatch) {
+            logger?.warn?.(`${TAG} No JSON array found in conflict detection response`);
+            return fallbackStoreAll(memories);
+        }
+        // Sanitize control characters inside JSON string literals that LLM may produce
+        const sanitized = sanitizeJsonForParse(arrayMatch[0]);
+        const parsed = JSON.parse(sanitized);
+        if (!Array.isArray(parsed)) {
+            logger?.warn?.(`${TAG} Conflict detection response is not an array`);
+            return fallbackStoreAll(memories);
+        }
+        // Build decisions from LLM output
+        const decisions = [];
+        const validActions = ["store", "update", "merge", "skip"];
+        for (const item of parsed) {
+            if (!item || typeof item !== "object")
+                continue;
+            const d = item;
+            const recordId = String(d.record_id ?? "");
+            // Skip entries with empty/missing record_id — they are LLM hallucinations
+            if (!recordId) {
+                logger?.debug?.(`${TAG} Skipping decision with empty record_id`);
+                continue;
+            }
+            const action = String(d.action ?? "store");
+            if (!validActions.includes(action)) {
+                logger?.warn?.(`${TAG} Invalid action "${action}" for record ${recordId}, defaulting to store`);
+            }
+            decisions.push({
+                record_id: recordId,
+                action: validActions.includes(action) ? action : "store",
+                target_ids: Array.isArray(d.target_ids) ? d.target_ids.map(String) : [],
+                merged_content: typeof d.merged_content === "string" ? d.merged_content : undefined,
+                merged_type: VALID_TYPES.includes(d.merged_type) ? d.merged_type : undefined,
+                merged_priority: typeof d.merged_priority === "number" ? d.merged_priority : undefined,
+                merged_timestamps: Array.isArray(d.merged_timestamps) ? d.merged_timestamps.map(String) : undefined,
+            });
+        }
+        // Ensure all memories have a decision (fill missing with "store")
+        const decidedIds = new Set(decisions.map((d) => d.record_id));
+        for (const mem of memories) {
+            if (!decidedIds.has(mem.record_id)) {
+                logger?.debug?.(`${TAG} No decision for record ${mem.record_id}, defaulting to store`);
+                decisions.push({
+                    record_id: mem.record_id,
+                    action: "store",
+                    target_ids: [],
+                });
+            }
+        }
+        return decisions;
+    }
+    catch (err) {
+        logger?.warn?.(`${TAG} Failed to parse conflict detection result: ${err instanceof Error ? err.message : String(err)}`);
+        return fallbackStoreAll(memories);
+    }
+}
+/**
+ * Fallback: store all memories when parsing fails.
+ */
+function fallbackStoreAll(memories) {
+    return memories.map((m) => ({
+        record_id: m.record_id,
+        action: "store",
+        target_ids: [],
+    }));
+}

package/vendor/tdai/core/record/l1-extractor.js

@@ -0,0 +1,379 @@
+/**
+ * L1 Memory Extractor: extracts structured memories from L0 conversation messages
+ * using a single LLM call with JSON-mode structured output.
+ *
+ * v3: Aligned with Kenty's prompt — scene segmentation + memory extraction in one call,
+ * followed by batch conflict detection.
+ *
+ * Pipeline:
+ * 1. Read recent messages from L0 (split into background + new)
+ * 2. Call LLM to extract scene-segmented memories
+ * 3. Batch conflict detection against existing records
+ * 4. Write to L1 JSONL files
+ */
+import { EXTRACT_MEMORIES_SYSTEM_PROMPT, formatExtractionPrompt } from "../prompts/l1-extraction.js";
+import { batchDedup } from "./l1-dedup.js";
+import { writeMemory, generateMemoryId } from "./l1-writer.js";
+import { CleanContextRunner } from "../../utils/clean-context-runner.js";
+import { sanitizeJsonForParse, shouldExtractL1 } from "../../utils/sanitize.js";
+import { report } from "../report/reporter.js";
+const TAG = "[memory-tdai][l1-extractor]";
+// ============================
+// Core function
+// ============================
+/**
+ * Run the full L1 extraction pipeline on conversation messages.
+ *
+ * @param messages - Filtered conversation messages (from L0 or directly from hook)
+ * @param sessionKey - The session key
+ * @param baseDir - Base data directory (~/.openclaw/memory-tdai/)
+ * @param config - OpenClaw config (for LLM access)
+ * @param options - Extraction options
+ * @param logger - Optional logger
+ */
+export async function extractL1Memories(params) {
+    const { messages, sessionKey, sessionId, baseDir, config, logger, instanceId: metricInstanceId } = params;
+    const options = params.options ?? {};
+    const maxNewMessages = options.maxMessagesPerExtraction ?? 10;
+    const maxBgMessages = options.maxBackgroundMessages ?? 5;
+    const enableDedup = options.enableDedup ?? true;
+    const maxMemoriesPerSession = options.maxMemoriesPerSession ?? 10;
+    if (messages.length === 0) {
+        logger?.debug?.(`${TAG} No messages to extract from`);
+        return { success: true, extractedCount: 0, storedCount: 0, records: [], sceneNames: [] };
+    }
+    const l1StartMs = Date.now();
+    // Quality gate: filter messages through L1 extraction rules (length, symbols,
+    // prompt injection, etc.) before sending to the LLM. L0 deliberately captures
+    // everything; the strict filtering happens here at L1 stage.
+    const qualifiedMessages = messages.filter((m) => shouldExtractL1(m.content));
+    if (qualifiedMessages.length < messages.length) {
+        logger?.debug?.(`${TAG} L1 quality filter: ${messages.length} → ${qualifiedMessages.length} messages ` +
+            `(${messages.length - qualifiedMessages.length} filtered out)`);
+    }
+    if (qualifiedMessages.length === 0) {
+        logger?.debug?.(`${TAG} All messages filtered out by L1 quality gate`);
+        return { success: true, extractedCount: 0, storedCount: 0, records: [], sceneNames: [] };
+    }
+    // Split messages into background (older) + new (recent)
+    const newMessages = qualifiedMessages.slice(-maxNewMessages);
+    const bgEndIdx = qualifiedMessages.length - newMessages.length;
+    const backgroundMessages = bgEndIdx > 0
+        ? qualifiedMessages.slice(Math.max(0, bgEndIdx - maxBgMessages), bgEndIdx)
+        : [];
+    logger?.debug?.(`${TAG} Extracting from ${newMessages.length} new messages (+ ${backgroundMessages.length} background) [${qualifiedMessages.length} qualified from ${messages.length} input]`);
+    // Step 1: LLM extraction (scene segmentation + memory extraction)
+    let scenes;
+    try {
+        scenes = await callLlmExtraction({
+            newMessages,
+            backgroundMessages,
+            previousSceneName: options.previousSceneName,
+            config,
+            logger,
+            model: options.model,
+            llmRunner: options.llmRunner,
+        });
+        logger?.debug?.(`${TAG} LLM detected ${scenes.length} scene(s)`);
+    }
+    catch (err) {
+        logger?.error(`${TAG} LLM extraction failed: ${err instanceof Error ? err.message : String(err)}`);
+        return { success: false, extractedCount: 0, storedCount: 0, records: [], sceneNames: [] };
+    }
+    // Flatten all memories across scenes
+    const allExtracted = [];
+    const sceneNames = [];
+    for (const scene of scenes) {
+        sceneNames.push(scene.scene_name);
+        for (const mem of scene.memories) {
+            const memType = normalizeType(mem.type);
+            if (!memType) {
+                logger?.warn?.(`${TAG} Skipping memory with invalid type "${mem.type}"`);
+                continue;
+            }
+            allExtracted.push({
+                content: mem.content,
+                type: memType,
+                priority: typeof mem.priority === "number" ? mem.priority : 50,
+                source_message_ids: Array.isArray(mem.source_message_ids) ? mem.source_message_ids : [],
+                metadata: mem.metadata ?? {},
+                scene_name: scene.scene_name,
+            });
+        }
+    }
+    logger?.debug?.(`${TAG} Total extracted memories: ${allExtracted.length} across ${scenes.length} scene(s)`);
+    if (allExtracted.length === 0) {
+        return {
+            success: true,
+            extractedCount: 0,
+            storedCount: 0,
+            records: [],
+            sceneNames,
+            lastSceneName: sceneNames[sceneNames.length - 1],
+        };
+    }
+    // Limit per session
+    let extracted = allExtracted;
+    if (extracted.length > maxMemoriesPerSession) {
+        logger?.debug?.(`${TAG} Limiting from ${extracted.length} to ${maxMemoriesPerSession} memories per session`);
+        extracted = extracted.slice(0, maxMemoriesPerSession);
+    }
+    // Assign temporary IDs to extracted memories (needed for batch dedup)
+    const memoriesWithIds = extracted.map((m) => ({
+        ...m,
+        record_id: generateMemoryId(),
+    }));
+    // Step 2: Batch Conflict Detection + Write
+    let storedRecords;
+    if (enableDedup) {
+        try {
+            const decisions = await batchDedup({
+                memories: memoriesWithIds,
+                config,
+                logger,
+                model: options.model,
+                vectorStore: options.vectorStore,
+                embeddingService: options.embeddingService,
+                conflictRecallTopK: options.conflictRecallTopK,
+                embeddingTimeoutMs: options.embeddingTimeoutMs,
+                llmRunner: options.llmRunner,
+            });
+            storedRecords = await applyDecisions({
+                memoriesWithIds,
+                decisions,
+                baseDir,
+                sessionKey,
+                sessionId,
+                logger,
+                vectorStore: options.vectorStore,
+                embeddingService: options.embeddingService,
+            });
+        }
+        catch (err) {
+            logger?.warn?.(`${TAG} Batch dedup failed, storing all as new: ${err instanceof Error ? err.message : String(err)}`);
+            storedRecords = await storeAllDirectly(memoriesWithIds, baseDir, sessionKey, sessionId, logger, options.vectorStore, options.embeddingService);
+        }
+    }
+    else {
+        storedRecords = await storeAllDirectly(memoriesWithIds, baseDir, sessionKey, sessionId, logger, options.vectorStore, options.embeddingService);
+    }
+    logger?.info(`${TAG} Extraction complete: extracted=${extracted.length}, stored=${storedRecords.length}`);
+    // ── l1_extraction metric ──
+    if (metricInstanceId && logger) {
+        // Build type distribution of stored memories
+        const memoriesByType = {};
+        for (const r of storedRecords) {
+            memoriesByType[r.type] = (memoriesByType[r.type] ?? 0) + 1;
+        }
+        report("l1_extraction", {
+            sessionKey,
+            inputMessageCount: messages.length,
+            memoriesExtracted: extracted.length,
+            memoriesStored: storedRecords.length,
+            memoriesStoredContent: storedRecords.map((r) => ({
+                content: r.content,
+                type: r.type,
+                scene: r.scene_name ?? null,
+            })),
+            memoriesByType,
+            totalDurationMs: Date.now() - l1StartMs,
+            success: true,
+            error: null,
+        });
+    }
+    return {
+        success: true,
+        extractedCount: extracted.length,
+        storedCount: storedRecords.length,
+        records: storedRecords,
+        sceneNames,
+        lastSceneName: sceneNames[sceneNames.length - 1],
+    };
+}
+// ============================
+// LLM call
+// ============================
+/**
+ * Call LLM to extract scene-segmented memories from conversation messages.
+ */
+async function callLlmExtraction(params) {
+    const { newMessages, backgroundMessages, previousSceneName, config, logger, model, llmRunner } = params;
+    const userPrompt = formatExtractionPrompt({
+        newMessages,
+        backgroundMessages,
+        previousSceneName,
+    });
+    // [l1-debug] ENTRY — what are we about to ask the LLM to extract?
+    logger?.debug?.(`${TAG} [l1-debug] ENTRY taskId=l1-extraction, newMsgs=${newMessages.length}, bgMsgs=${backgroundMessages.length}, userPromptLen=${userPrompt.length}, sysPromptLen=${EXTRACT_MEMORIES_SYSTEM_PROMPT.length}, model=${model ?? "(default)"}, previousSceneName=${previousSceneName ? JSON.stringify(previousSceneName) : "(none)"}, runnerKind=${llmRunner ? "llmRunner" : "CleanContextRunner"}`);
+    let result;
+    if (llmRunner) {
+        // Use the host-neutral LLMRunner interface
+        result = await llmRunner.run({
+            prompt: userPrompt,
+            systemPrompt: EXTRACT_MEMORIES_SYSTEM_PROMPT,
+            taskId: "l1-extraction",
+            timeoutMs: 180_000,
+        });
+    }
+    else {
+        // Fallback: create CleanContextRunner (OpenClaw path)
+        const runner = new CleanContextRunner({
+            config,
+            modelRef: model,
+            enableTools: false,
+            logger,
+        });
+        result = await runner.run({
+            prompt: userPrompt,
+            systemPrompt: EXTRACT_MEMORIES_SYSTEM_PROMPT,
+            taskId: "l1-extraction",
+            timeoutMs: 180_000,
+        });
+    }
+    return parseExtractionResult(result, logger);
+}
+/**
+ * Parse the LLM's JSON response into SceneSegment array.
+ * Expected format: [{scene_name, message_ids, memories: [...]}]
+ */
+function parseExtractionResult(raw, logger) {
+    try {
+        // Strip markdown code block wrappers if present
+        let cleaned = raw.trim();
+        if (cleaned.startsWith("```")) {
+            cleaned = cleaned.replace(/^```(?:json)?\s*\n?/, "").replace(/\n?```\s*$/, "");
+        }
+        // Try to extract JSON array
+        const arrayMatch = cleaned.match(/\[[\s\S]*\]/);
+        if (!arrayMatch) {
+            logger?.warn?.(`${TAG} No JSON array found in extraction response`);
+            // [l1-debug] NO_JSON — dump the full raw so we can see what the LLM actually said
+            const rawPreview = raw.slice(0, 2048);
+            logger?.warn?.(`${TAG} [l1-debug] NO_JSON taskId=l1-extraction, rawLen=${raw.length}, cleanedLen=${cleaned.length}, rawFull=${JSON.stringify(rawPreview)}${raw.length > 2048 ? `…(+${raw.length - 2048})` : ""}`);
+            return [];
+        }
+        // Sanitize control characters inside JSON string literals that LLM may produce
+        const sanitized = sanitizeJsonForParse(arrayMatch[0]);
+        const parsed = JSON.parse(sanitized);
+        if (!Array.isArray(parsed)) {
+            logger?.warn?.(`${TAG} Extraction response is not an array`);
+            return [];
+        }
+        const scenes = [];
+        for (const item of parsed) {
+            if (!item || typeof item !== "object")
+                continue;
+            const s = item;
+            scenes.push({
+                scene_name: typeof s.scene_name === "string" ? s.scene_name : "未知情境",
+                message_ids: Array.isArray(s.message_ids) ? s.message_ids.map(String) : [],
+                memories: Array.isArray(s.memories)
+                    ? s.memories
+                        .filter((m) => m && typeof m === "object" && typeof m.content === "string" && m.content.length > 0)
+                        .map((m) => ({
+                        content: String(m.content),
+                        type: String(m.type ?? "episodic"),
+                        priority: typeof m.priority === "number" ? m.priority : 50,
+                        source_message_ids: Array.isArray(m.source_message_ids) ? m.source_message_ids.map(String) : [],
+                        metadata: (m.metadata && typeof m.metadata === "object" ? m.metadata : {}),
+                    }))
+                    : [],
+            });
+        }
+        return scenes;
+    }
+    catch (err) {
+        logger?.warn?.(`${TAG} Failed to parse extraction result: ${err instanceof Error ? err.message : String(err)}`);
+        return [];
+    }
+}
+// ============================
+// Write helpers
+// ============================
+/**
+ * Apply batch dedup decisions — write memories according to their decisions.
+ */
+async function applyDecisions(params) {
+    const { memoriesWithIds, decisions, baseDir, sessionKey, sessionId, logger, vectorStore, embeddingService } = params;
+    const storedRecords = [];
+    // Build a map from record_id → decision
+    const decisionMap = new Map();
+    for (const d of decisions) {
+        decisionMap.set(d.record_id, d);
+    }
+    for (const memoryWithId of memoriesWithIds) {
+        const decision = decisionMap.get(memoryWithId.record_id) ?? {
+            record_id: memoryWithId.record_id,
+            action: "store",
+            target_ids: [],
+        };
+        try {
+            const record = await writeMemory({
+                memory: memoryWithId,
+                decision,
+                baseDir,
+                sessionKey,
+                sessionId,
+                logger,
+                vectorStore,
+                embeddingService,
+            });
+            if (record) {
+                storedRecords.push(record);
+            }
+        }
+        catch (err) {
+            logger?.warn?.(`${TAG} Write failed for memory "${memoryWithId.content.slice(0, 50)}...": ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    return storedRecords;
+}
+/**
+ * Store all memories directly (no dedup).
+ */
+async function storeAllDirectly(memoriesWithIds, baseDir, sessionKey, sessionId, logger, vectorStore, embeddingService) {
+    const storedRecords = [];
+    for (const memoryWithId of memoriesWithIds) {
+        try {
+            const record = await writeMemory({
+                memory: memoryWithId,
+                decision: {
+                    record_id: memoryWithId.record_id,
+                    action: "store",
+                    target_ids: [],
+                },
+                baseDir,
+                sessionKey,
+                sessionId,
+                logger,
+                vectorStore,
+                embeddingService,
+            });
+            if (record) {
+                storedRecords.push(record);
+            }
+        }
+        catch (err) {
+            logger?.warn?.(`${TAG} Write failed for memory "${memoryWithId.content.slice(0, 50)}...": ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    return storedRecords;
+}
+// ============================
+// Helpers
+// ============================
+const VALID_TYPES = ["persona", "episodic", "instruction"];
+function normalizeType(raw) {
+    const lower = raw.toLowerCase().trim();
+    if (VALID_TYPES.includes(lower)) {
+        return lower;
+    }
+    // Handle legacy type names
+    if (lower === "episode")
+        return "episodic";
+    if (lower === "instruct")
+        return "instruction";
+    if (lower === "preference")
+        return "persona"; // fold preference into persona
+    return null;
+}