@memtensor/memos-local-openclaw-plugin 0.1.3 → 0.1.5

package/src/ingest/task-processor.ts

@@ -0,0 +1,396 @@
+import { v4 as uuid } from "uuid";
+import type { SqliteStore } from "../storage/sqlite";
+import type { PluginContext, Task, Chunk } from "../types";
+import { DEFAULTS } from "../types";
+import { Summarizer } from "./providers";
+
+const TRIVIAL_PATTERNS = [
+  /^(test|testing|hello|hi|hey|ok|okay|yes|no|yeah|nope|sure|thanks|thank you|thx|ping|pong|哈哈|好的|嗯|是的|不是|谢谢|你好|测试)\s*[.!?。！？]*$/,
+  /^(aaa+|bbb+|xxx+|zzz+|123+|asdf+|qwer+|haha+|lol+|hmm+)\s*$/,
+  /^[\s\p{P}\p{S}]*$/u,
+];
+
+const SKIP_REASONS = {
+  noChunks: "该任务没有对话内容，已自动跳过。",
+} as const;
+
+/**
+ * Asynchronous task-level processor.
+ *
+ * After each ingestion batch, checks whether the current conversation
+ * constitutes a "new task" compared to the previous one. If so:
+ *   1. Finalizes the previous task (generates a detailed summary).
+ *   2. Creates a new active task for incoming chunks.
+ *
+ * Task boundary detection:
+ *   - Session change → always new task
+ *   - Time gap > 2h → always new task
+ *   - LLM judges whether new user message starts a different topic
+ */
+export class TaskProcessor {
+  private summarizer: Summarizer;
+  private processing = false;
+  private onTaskCompletedCallback?: (task: Task) => void;
+
+  constructor(
+    private store: SqliteStore,
+    private ctx: PluginContext,
+  ) {
+    this.summarizer = new Summarizer(ctx.config.summarizer, ctx.log);
+  }
+
+  onTaskCompleted(cb: (task: Task) => void): void {
+    this.onTaskCompletedCallback = cb;
+  }
+
+  /**
+   * Called after new chunks are ingested.
+   * Determines if a new task boundary was crossed and handles transition.
+   */
+  async onChunksIngested(sessionKey: string, latestTimestamp: number): Promise<void> {
+    this.ctx.log.debug(`TaskProcessor.onChunksIngested called session=${sessionKey} ts=${latestTimestamp} processing=${this.processing}`);
+    if (this.processing) {
+      this.ctx.log.debug("TaskProcessor.onChunksIngested skipped — already processing");
+      return;
+    }
+    this.processing = true;
+    try {
+      await this.detectAndProcess(sessionKey, latestTimestamp);
+    } catch (err) {
+      this.ctx.log.error(`TaskProcessor error: ${err}`);
+    } finally {
+      this.processing = false;
+    }
+  }
+
+  private async detectAndProcess(sessionKey: string, latestTimestamp: number): Promise<void> {
+    this.ctx.log.debug(`TaskProcessor.detectAndProcess session=${sessionKey}`);
+
+    // Finalize any active tasks from OTHER sessions (session change = task boundary)
+    const allActive = this.store.getAllActiveTasks();
+    for (const t of allActive) {
+      if (t.sessionKey !== sessionKey) {
+        this.ctx.log.info(`Session changed: finalizing task=${t.id} from session=${t.sessionKey}`);
+        await this.finalizeTask(t);
+      }
+    }
+
+    const activeTask = this.store.getActiveTask(sessionKey);
+    this.ctx.log.debug(`TaskProcessor.detectAndProcess activeTask=${activeTask?.id ?? "none"}`);
+
+    if (!activeTask) {
+      await this.createNewTask(sessionKey, latestTimestamp);
+      return;
+    }
+
+    const isNewTask = await this.isTaskBoundary(activeTask, sessionKey, latestTimestamp);
+
+    if (isNewTask) {
+      await this.finalizeTask(activeTask);
+      await this.createNewTask(sessionKey, latestTimestamp);
+    } else {
+      this.assignUnassignedChunks(sessionKey, activeTask.id);
+      this.store.updateTask(activeTask.id, { endedAt: undefined });
+    }
+  }
+
+  private async isTaskBoundary(activeTask: Task, sessionKey: string, latestTimestamp: number): Promise<boolean> {
+    if (activeTask.sessionKey !== sessionKey) return true;
+
+    const chunks = this.store.getChunksByTask(activeTask.id);
+    if (chunks.length === 0) return false;
+
+    const lastChunkTs = Math.max(...chunks.map((c) => c.createdAt));
+    const gap = latestTimestamp - lastChunkTs;
+
+    // Hard timeout: always split after 2h regardless of topic
+    if (gap > DEFAULTS.taskIdleTimeoutMs) {
+      this.ctx.log.info(
+        `Task boundary: time gap ${Math.round(gap / 60000)}min > ${Math.round(DEFAULTS.taskIdleTimeoutMs / 60000)}min`,
+      );
+      return true;
+    }
+
+    // LLM topic judgment: build context from existing task and compare with new message
+    const newUserChunks = this.store.getUnassignedChunks(sessionKey).filter((c) => c.role === "user");
+    if (newUserChunks.length === 0) return false;
+
+    const existingUserChunks = chunks.filter((c) => c.role === "user");
+    if (existingUserChunks.length === 0) return false;
+
+    const currentContext = this.buildContextSummary(chunks);
+    const newMessage = newUserChunks.map((c) => c.content).join("\n");
+
+    const isNew = await this.summarizer.judgeNewTopic(currentContext, newMessage);
+
+    if (isNew === null) {
+      this.ctx.log.debug("Topic judge unavailable (no LLM configured), keeping current task");
+      return false;
+    }
+
+    if (isNew) {
+      this.ctx.log.info(`Task boundary: LLM judged new topic. New message: "${newMessage.slice(0, 80)}..."`);
+    } else {
+      this.ctx.log.debug(`LLM judged SAME topic, continuing task=${activeTask.id}`);
+    }
+
+    return isNew;
+  }
+
+  /**
+   * Build a concise context string from existing task chunks for the LLM topic judge.
+   * Takes recent user/assistant summaries to keep token usage low.
+   */
+  private buildContextSummary(chunks: Chunk[]): string {
+    const relevant = chunks
+      .filter((c) => c.role === "user" || c.role === "assistant")
+      .slice(-6);
+
+    return relevant
+      .map((c) => `[${c.role === "user" ? "User" : "Assistant"}]: ${c.summary || c.content.slice(0, 150)}`)
+      .join("\n");
+  }
+
+  private async createNewTask(sessionKey: string, timestamp: number): Promise<void> {
+    const taskId = uuid();
+    const task: Task = {
+      id: taskId,
+      sessionKey,
+      title: "",
+      summary: "",
+      status: "active",
+      startedAt: timestamp,
+      endedAt: null,
+      updatedAt: timestamp,
+    };
+    this.store.insertTask(task);
+    this.assignUnassignedChunks(sessionKey, taskId);
+    this.ctx.log.info(`Created new task=${taskId} session=${sessionKey}`);
+  }
+
+  private assignUnassignedChunks(sessionKey: string, taskId: string): void {
+    const unassigned = this.store.getUnassignedChunks(sessionKey);
+    for (const chunk of unassigned) {
+      this.store.setChunkTaskId(chunk.id, taskId);
+    }
+    if (unassigned.length > 0) {
+      this.ctx.log.debug(`Assigned ${unassigned.length} chunks to task=${taskId}`);
+    }
+  }
+
+  async finalizeTask(task: Task): Promise<void> {
+    const chunks = this.store.getChunksByTask(task.id);
+    const fallbackTitle = chunks.length > 0 ? this.extractTitle(chunks) : "";
+
+    if (chunks.length === 0) {
+      this.ctx.log.info(`Task ${task.id} skipped: no chunks`);
+      this.store.updateTask(task.id, { title: fallbackTitle, summary: SKIP_REASONS.noChunks, status: "skipped", endedAt: Date.now() });
+      return;
+    }
+
+    const skipReason = this.shouldSkipSummary(chunks);
+
+    if (skipReason) {
+      this.ctx.log.info(`Task ${task.id} skipped: ${skipReason} (chunks=${chunks.length}, title="${fallbackTitle}")`);
+      const reason = this.humanReadableSkipReason(skipReason, chunks);
+      this.store.updateTask(task.id, { title: fallbackTitle, summary: reason, status: "skipped", endedAt: Date.now() });
+      return;
+    }
+
+    const conversationText = this.buildConversationText(chunks);
+    let summary: string;
+    try {
+      summary = await this.summarizer.summarizeTask(conversationText);
+    } catch (err) {
+      this.ctx.log.warn(`Task summary generation failed for task=${task.id}: ${err}`);
+      summary = this.fallbackSummary(chunks);
+    }
+
+    const { title: llmTitle, body } = this.parseTitleFromSummary(summary);
+    const title = llmTitle || fallbackTitle;
+
+    this.store.updateTask(task.id, {
+      title,
+      summary: body,
+      status: "completed",
+      endedAt: Date.now(),
+    });
+
+    this.ctx.log.info(
+      `Finalized task=${task.id} title="${title}" chunks=${chunks.length} summaryLen=${body.length}`,
+    );
+
+    if (this.onTaskCompletedCallback) {
+      const finalized = this.store.getTask(task.id);
+      if (finalized) {
+        try {
+          this.onTaskCompletedCallback(finalized);
+        } catch (err) {
+          this.ctx.log.warn(`TaskProcessor onTaskCompleted callback error: ${err}`);
+        }
+      }
+    }
+  }
+
+  /**
+   * Determine if a task is too trivial to warrant an LLM summary call.
+   * Returns a skip reason string, or null if summary should proceed.
+   *
+   * Skip conditions (any one triggers skip):
+   *  1. Total chunks < 4 — too few messages to form a meaningful task
+   *  2. Real conversation turns < 2 — no back-and-forth dialogue
+   *  3. No user messages — purely system/tool generated, no user intent
+   *  4. Total content < 200 chars — not enough substance
+   *  5. User content is trivial/test data — "hello", "test", "ok" etc.
+   *  6. All messages are tool results — automated output, no conversation
+   *  7. High content repetition — user repeated the same thing (debug loops)
+   */
+  private shouldSkipSummary(chunks: Chunk[]): string | null {
+    const userChunks = chunks.filter((c) => c.role === "user");
+    const assistantChunks = chunks.filter((c) => c.role === "assistant");
+    const toolChunks = chunks.filter((c) => c.role === "tool");
+
+    // 1. Too few chunks
+    if (chunks.length < 4) {
+      return `too few chunks (${chunks.length} < 4 minimum)`;
+    }
+
+    // 2. Not enough real conversation turns (need at least 2 user-assistant exchanges)
+    const turns = Math.min(userChunks.length, assistantChunks.length);
+    if (turns < 2) {
+      return `too few conversation turns (${turns} < 2 minimum)`;
+    }
+
+    // 3. No user messages at all — purely automated
+    if (userChunks.length === 0) {
+      return "no user messages — task appears to be automated/system-generated";
+    }
+
+    // 4. Total content too short
+    // CJK characters carry more info per char, so use a lower threshold
+    const totalContentLen = chunks.reduce((sum, c) => sum + c.content.length, 0);
+    const hasCJK = /[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]/.test(
+      userChunks[0]?.content ?? "",
+    );
+    const minContentLen = hasCJK ? 80 : 200;
+    if (totalContentLen < minContentLen) {
+      return `content too short (${totalContentLen} chars < ${minContentLen} minimum)`;
+    }
+
+    // 5. User content is trivial/test data
+    const userContent = userChunks.map((c) => c.content).join("\n");
+    if (this.looksLikeTrivialContent(userContent)) {
+      return "user content appears to be test/trivial data";
+    }
+
+    // 6. Assistant content is also trivial (both sides are low-value)
+    const assistantContent = assistantChunks.map((c) => c.content).join("\n");
+    if (this.looksLikeTrivialContent(userContent + "\n" + assistantContent)) {
+      return "conversation content (both user and assistant) appears trivial";
+    }
+
+    // 7. Almost all messages are tool results with minimal user interaction
+    if (toolChunks.length > 0 && toolChunks.length >= chunks.length * 0.7 && userChunks.length <= 1) {
+      return `dominated by tool results (${toolChunks.length}/${chunks.length} chunks) with minimal user input`;
+    }
+
+    // 8. High repetition — user keeps saying the same thing
+    if (userChunks.length >= 3) {
+      const uniqueUserMsgs = new Set(userChunks.map((c) => c.content.trim().toLowerCase()));
+      const uniqueRatio = uniqueUserMsgs.size / userChunks.length;
+      if (uniqueRatio < 0.4) {
+        return `high content repetition (${uniqueUserMsgs.size} unique out of ${userChunks.length} user messages)`;
+      }
+    }
+
+    return null;
+  }
+
+  private looksLikeTrivialContent(text: string): boolean {
+    const lines = text.toLowerCase().split(/\n/).map((l) => l.trim()).filter(Boolean);
+    if (lines.length === 0) return true;
+
+    const trivialCount = lines.filter((line) => {
+      if (line.length < 5) return true;
+      if (TRIVIAL_PATTERNS.some((p) => p.test(line))) return true;
+      return false;
+    }).length;
+
+    return trivialCount / lines.length > 0.7;
+  }
+
+  private buildConversationText(chunks: Chunk[]): string {
+    const lines: string[] = [];
+    for (const c of chunks) {
+      const roleLabel = c.role === "user" ? "User" : c.role === "assistant" ? "Assistant" : c.role;
+      lines.push(`[${roleLabel}]: ${c.content}`);
+    }
+    return lines.join("\n\n");
+  }
+
+  /**
+   * Extract the LLM-generated title from the summary output.
+   * The LLM is prompted to output "📌 Title\n<title text>" as the first section.
+   * Returns the title and the remaining body (with the title section stripped).
+   */
+  private parseTitleFromSummary(summary: string): { title: string; body: string } {
+    const titleMatch = summary.match(/📌\s*(?:Title|标题)\s*\n(.+)/);
+    if (titleMatch) {
+      const title = titleMatch[1].trim().slice(0, 80);
+      const body = summary.replace(/📌\s*(?:Title|标题)\s*\n.+\n?/, "").trim();
+      return { title, body };
+    }
+    return { title: "", body: summary };
+  }
+
+  private extractTitle(chunks: Chunk[]): string {
+    const firstUser = chunks.find((c) => c.role === "user");
+    if (!firstUser) return "Untitled Task";
+    const text = firstUser.content.trim();
+    if (text.length <= 60) return text;
+    return text.slice(0, 57) + "...";
+  }
+
+  private humanReadableSkipReason(reason: string, chunks: Chunk[]): string {
+    const userCount = chunks.filter((c) => c.role === "user").length;
+    const assistantCount = chunks.filter((c) => c.role === "assistant").length;
+
+    if (reason.includes("too few chunks")) {
+      return `对话内容过少（${chunks.length} 条消息），不足以生成有效摘要。至少需要 4 条消息。`;
+    }
+    if (reason.includes("too few conversation turns")) {
+      return `对话轮次不足（${Math.min(userCount, assistantCount)} 轮），需要至少 2 轮完整的问答交互才能生成摘要。`;
+    }
+    if (reason.includes("no user messages")) {
+      return "该任务没有用户消息，仅包含系统或工具自动生成的内容。";
+    }
+    if (reason.includes("content too short")) {
+      return "对话内容过短，信息量不足以生成有意义的摘要。";
+    }
+    if (reason.includes("trivial")) {
+      return "对话内容为简单问候或测试数据（如 hello、test、ok），无需生成摘要。";
+    }
+    if (reason.includes("tool results")) {
+      return "该任务主要由工具执行结果组成，缺少足够的用户交互内容。";
+    }
+    if (reason.includes("repetition")) {
+      return "对话中存在大量重复内容，无法提取有效信息生成摘要。";
+    }
+    return `对话未达到生成摘要的条件：${reason}`;
+  }
+
+  private fallbackSummary(chunks: Chunk[]): string {
+    const title = this.extractTitle(chunks);
+    const summaries = chunks
+      .filter((c) => c.summary)
+      .map((c) => `- ${c.summary}`);
+    const lines = [
+      `🎯 Goal`,
+      title,
+      ``,
+      `📋 Key Steps`,
+      ...summaries.slice(0, 20),
+    ];
+    return lines.join("\n");
+  }
+}

package/src/ingest/worker.ts

@@ -1,13 +1,15 @@
 import { v4 as uuid } from "uuid";
+import { createHash } from "crypto";
 import type { ConversationMessage, Chunk, PluginContext } from "../types";
 import type { SqliteStore } from "../storage/sqlite";
 import type { Embedder } from "../embedding";
 import { Summarizer } from "./providers";
-import { chunkText } from "./chunker";
-import { findDuplicate } from "./dedup";
+import { findDuplicate, findTopSimilar } from "./dedup";
+import { TaskProcessor } from "./task-processor";
 
 export class IngestWorker {
   private summarizer: Summarizer;
+  private taskProcessor: TaskProcessor;
   private queue: ConversationMessage[] = [];
   private processing = false;
   private flushResolvers: Array<() => void> = [];
@@ -18,8 +20,11 @@ export class IngestWorker {
     private ctx: PluginContext,
   ) {
     this.summarizer = new Summarizer(ctx.config.summarizer, ctx.log);
+    this.taskProcessor = new TaskProcessor(store, ctx);
   }
 
+  getTaskProcessor(): TaskProcessor { return this.taskProcessor; }
+
   enqueue(messages: ConversationMessage[]): void {
     this.queue.push(...messages);
     if (!this.processing) {
@@ -40,39 +45,83 @@ export class IngestWorker {
 
   private async processQueue(): Promise<void> {
     this.processing = true;
+    const t0 = performance.now();
+
+    let lastSessionKey: string | undefined;
+    let lastTimestamp = 0;
+    let stored = 0;
+    let skipped = 0;
+    let merged = 0;
+    let duplicated = 0;
+    let errors = 0;
+    const resultLines: string[] = [];
+    const inputLines: string[] = [];
+    const totalMessages = this.queue.length;
 
     while (this.queue.length > 0) {
       const msg = this.queue.shift()!;
+      inputLines.push(`[${msg.role}] ${msg.content}`);
       try {
-        await this.ingestMessage(msg);
+        const result = await this.ingestMessage(msg);
+        lastSessionKey = msg.sessionKey;
+        lastTimestamp = Math.max(lastTimestamp, msg.timestamp);
+        if (result === "skipped") {
+          skipped++;
+          resultLines.push(`[${msg.role}] ⏭ exact-dup → ${msg.content}`);
+        } else if (result.action === "stored") {
+          stored++;
+          resultLines.push(`[${msg.role}] ✅ stored → ${result.summary ?? msg.content}`);
+        } else if (result.action === "duplicate") {
+          duplicated++;
+          resultLines.push(`[${msg.role}] 🔁 dedup(${result.reason ?? "similar"}) → ${msg.content}`);
+        } else if (result.action === "merged") {
+          merged++;
+          resultLines.push(`[${msg.role}] 🔀 merged → ${msg.content}`);
+        }
       } catch (err) {
+        errors++;
+        resultLines.push(`[${msg.role}] ❌ error → ${msg.content}`);
         this.ctx.log.error(`Failed to ingest message turn=${msg.turnId}: ${err}`);
       }
     }
 
+    const dur = performance.now() - t0;
+
+    if (stored + merged > 0 || skipped > 0 || duplicated > 0) {
+      this.store.recordToolCall("memory_add", dur, errors === 0);
+      try {
+        const inputInfo = {
+          session: lastSessionKey,
+          messages: totalMessages,
+          details: inputLines,
+        };
+        const stats = [`stored=${stored}`, skipped > 0 ? `skipped=${skipped}` : null, duplicated > 0 ? `dedup=${duplicated}` : null, merged > 0 ? `merged=${merged}` : null, errors > 0 ? `errors=${errors}` : null].filter(Boolean).join(", ");
+        this.store.recordApiLog("memory_add", inputInfo, `${stats}\n${resultLines.join("\n")}`, dur, errors === 0);
+      } catch (_) { /* best-effort */ }
+    }
+
+    if (lastSessionKey) {
+      this.ctx.log.debug(`Calling TaskProcessor.onChunksIngested session=${lastSessionKey} ts=${lastTimestamp}`);
+      this.taskProcessor
+        .onChunksIngested(lastSessionKey, lastTimestamp)
+        .catch((err) => this.ctx.log.error(`TaskProcessor post-ingest error: ${err}`));
+    }
+
     this.processing = false;
     for (const resolve of this.flushResolvers) resolve();
     this.flushResolvers = [];
   }
 
-  private async ingestMessage(msg: ConversationMessage): Promise<void> {
-    if (msg.role === "tool") {
-      await this.ingestToolResult(msg);
-      return;
-    }
-
-    const rawChunks = chunkText(msg.content);
-    this.ctx.log.debug(`Chunked turn=${msg.turnId} into ${rawChunks.length} chunks`);
-
-    for (let seq = 0; seq < rawChunks.length; seq++) {
-      const raw = rawChunks[seq];
-      await this.storeChunk(msg, raw.content, raw.kind, seq);
+  private async ingestMessage(msg: ConversationMessage): Promise<
+    "skipped" | { action: "stored" | "duplicate" | "merged"; summary?: string; reason?: string }
+  > {
+    if (this.store.chunkExistsByContent(msg.sessionKey, msg.role, msg.content)) {
+      this.ctx.log.debug(`Exact-dup (same session+role+hash), skipping: session=${msg.sessionKey} role=${msg.role} len=${msg.content.length}`);
+      return "skipped";
     }
-  }
 
-  private async ingestToolResult(msg: ConversationMessage): Promise<void> {
-    this.ctx.log.debug(`Ingesting tool result turn=${msg.turnId} tool=${msg.toolName ?? "unknown"} len=${msg.content.length}`);
-    await this.storeChunk(msg, msg.content, "tool_result", 0);
+    const kind = msg.role === "tool" ? "tool_result" : "paragraph";
+    return await this.storeChunk(msg, msg.content, kind, 0);
   }
 
   private async storeChunk(
@@ -80,7 +129,7 @@ export class IngestWorker {
     content: string,
     kind: Chunk["kind"],
     seq: number,
-  ): Promise<void> {
+  ): Promise<{ action: "stored" | "duplicate" | "merged"; chunkId?: string; summary?: string; targetChunkId?: string; reason?: string }> {
     const chunkId = uuid();
     const summary = await this.summarizer.summarize(content);
 
@@ -91,19 +140,65 @@ export class IngestWorker {
       this.ctx.log.warn(`Embedding failed for chunk=${chunkId}, storing without vector: ${err}`);
     }
 
+    let dedupStatus: "active" | "duplicate" | "merged" = "active";
+    let dedupTarget: string | null = null;
+    let dedupReason: string | null = null;
+
+    // Smart dedup: find Top-5 similar chunks, then ask LLM to judge
     if (embedding) {
-      const dupId = findDuplicate(
-        this.store,
-        embedding,
-        this.ctx.config.dedup?.similarityThreshold ?? 0.93,
-        this.ctx.log,
-      );
-
-      if (dupId) {
-        this.store.updateSummary(dupId, summary);
-        this.store.upsertEmbedding(dupId, embedding);
-        this.ctx.log.debug(`Dedup-merged into existing chunk=${dupId}`);
-        return;
+      const similarThreshold = this.ctx.config.dedup?.similarityThreshold ?? 0.75;
+      const topSimilar = findTopSimilar(this.store, embedding, similarThreshold, 5, this.ctx.log);
+
+      if (topSimilar.length > 0) {
+        const candidates = topSimilar.map((s, i) => {
+          const chunk = this.store.getChunk(s.chunkId);
+          return {
+            index: i + 1,
+            summary: chunk?.summary ?? "",
+            chunkId: s.chunkId,
+          };
+        }).filter(c => c.summary);
+
+        if (candidates.length > 0) {
+          const dedupResult = await this.summarizer.judgeDedup(summary, candidates);
+
+          if (dedupResult && dedupResult.action === "DUPLICATE" && dedupResult.targetIndex) {
+            const targetChunkId = candidates[dedupResult.targetIndex - 1]?.chunkId;
+            if (targetChunkId) {
+              this.store.recordMergeHit(targetChunkId, "DUPLICATE", dedupResult.reason);
+              dedupStatus = "duplicate";
+              dedupTarget = targetChunkId;
+              dedupReason = dedupResult.reason;
+              this.ctx.log.debug(`Smart dedup: DUPLICATE → target=${targetChunkId}, storing with status=duplicate, reason: ${dedupResult.reason}`);
+            }
+          }
+
+          if (dedupStatus === "active" && dedupResult && dedupResult.action === "UPDATE" && dedupResult.targetIndex && dedupResult.mergedSummary) {
+            const targetChunkId = candidates[dedupResult.targetIndex - 1]?.chunkId;
+            if (targetChunkId) {
+              const oldChunk = this.store.getChunk(targetChunkId);
+              const oldSummary = oldChunk?.summary ?? "";
+              this.store.recordMergeHit(targetChunkId, "UPDATE", dedupResult.reason, oldSummary, dedupResult.mergedSummary);
+              this.store.updateChunkSummaryAndContent(targetChunkId, dedupResult.mergedSummary, content);
+
+              try {
+                const [newEmb] = await this.embedder.embed([dedupResult.mergedSummary]);
+                if (newEmb) this.store.upsertEmbedding(targetChunkId, newEmb);
+              } catch (err) {
+                this.ctx.log.warn(`Re-embed after UPDATE failed: ${err}`);
+              }
+
+              dedupStatus = "merged";
+              dedupTarget = targetChunkId;
+              dedupReason = dedupResult.reason;
+              this.ctx.log.debug(`Smart dedup: UPDATE → merged into chunk=${targetChunkId}, storing with status=merged, reason: ${dedupResult.reason}`);
+            }
+          }
+
+          if (dedupStatus === "active") {
+            this.ctx.log.debug(`Smart dedup: NEW — creating active chunk (reason: ${dedupResult?.reason ?? "no_result"})`);
+          }
+        }
       }
     }
 
@@ -117,14 +212,30 @@ export class IngestWorker {
       kind,
       summary,
       embedding: null,
+      taskId: null,
+      skillId: null,
+      dedupStatus,
+      dedupTarget,
+      dedupReason,
+      mergeCount: 0,
+      lastHitAt: null,
+      mergeHistory: "[]",
       createdAt: msg.timestamp,
       updatedAt: msg.timestamp,
     };
 
     this.store.insertChunk(chunk);
-    if (embedding) {
+    if (embedding && dedupStatus === "active") {
       this.store.upsertEmbedding(chunkId, embedding);
     }
-    this.ctx.log.debug(`Stored chunk=${chunkId} kind=${kind} role=${msg.role} len=${content.length} hasVec=${!!embedding}`);
+    this.ctx.log.debug(`Stored chunk=${chunkId} kind=${kind} role=${msg.role} dedup=${dedupStatus} len=${content.length} hasVec=${!!embedding && dedupStatus === "active"}`);
+
+    if (dedupStatus === "duplicate") {
+      return { action: "duplicate", summary, targetChunkId: dedupTarget ?? undefined, reason: dedupReason ?? undefined };
+    }
+    if (dedupStatus === "merged") {
+      return { action: "merged", summary, targetChunkId: dedupTarget ?? undefined, reason: dedupReason ?? undefined };
+    }
+    return { action: "stored", chunkId, summary };
   }
 }