@memtensor/memos-local-openclaw-plugin 0.1.3 → 0.1.5

package/dist/ingest/task-processor.js

@@ -0,0 +1,354 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TaskProcessor = void 0;
+const uuid_1 = require("uuid");
+const types_1 = require("../types");
+const providers_1 = require("./providers");
+const TRIVIAL_PATTERNS = [
+    /^(test|testing|hello|hi|hey|ok|okay|yes|no|yeah|nope|sure|thanks|thank you|thx|ping|pong|哈哈|好的|嗯|是的|不是|谢谢|你好|测试)\s*[.!?。！？]*$/,
+    /^(aaa+|bbb+|xxx+|zzz+|123+|asdf+|qwer+|haha+|lol+|hmm+)\s*$/,
+    /^[\s\p{P}\p{S}]*$/u,
+];
+const SKIP_REASONS = {
+    noChunks: "该任务没有对话内容，已自动跳过。",
+};
+/**
+ * Asynchronous task-level processor.
+ *
+ * After each ingestion batch, checks whether the current conversation
+ * constitutes a "new task" compared to the previous one. If so:
+ *   1. Finalizes the previous task (generates a detailed summary).
+ *   2. Creates a new active task for incoming chunks.
+ *
+ * Task boundary detection:
+ *   - Session change → always new task
+ *   - Time gap > 2h → always new task
+ *   - LLM judges whether new user message starts a different topic
+ */
+class TaskProcessor {
+    store;
+    ctx;
+    summarizer;
+    processing = false;
+    onTaskCompletedCallback;
+    constructor(store, ctx) {
+        this.store = store;
+        this.ctx = ctx;
+        this.summarizer = new providers_1.Summarizer(ctx.config.summarizer, ctx.log);
+    }
+    onTaskCompleted(cb) {
+        this.onTaskCompletedCallback = cb;
+    }
+    /**
+     * Called after new chunks are ingested.
+     * Determines if a new task boundary was crossed and handles transition.
+     */
+    async onChunksIngested(sessionKey, latestTimestamp) {
+        this.ctx.log.debug(`TaskProcessor.onChunksIngested called session=${sessionKey} ts=${latestTimestamp} processing=${this.processing}`);
+        if (this.processing) {
+            this.ctx.log.debug("TaskProcessor.onChunksIngested skipped — already processing");
+            return;
+        }
+        this.processing = true;
+        try {
+            await this.detectAndProcess(sessionKey, latestTimestamp);
+        }
+        catch (err) {
+            this.ctx.log.error(`TaskProcessor error: ${err}`);
+        }
+        finally {
+            this.processing = false;
+        }
+    }
+    async detectAndProcess(sessionKey, latestTimestamp) {
+        this.ctx.log.debug(`TaskProcessor.detectAndProcess session=${sessionKey}`);
+        // Finalize any active tasks from OTHER sessions (session change = task boundary)
+        const allActive = this.store.getAllActiveTasks();
+        for (const t of allActive) {
+            if (t.sessionKey !== sessionKey) {
+                this.ctx.log.info(`Session changed: finalizing task=${t.id} from session=${t.sessionKey}`);
+                await this.finalizeTask(t);
+            }
+        }
+        const activeTask = this.store.getActiveTask(sessionKey);
+        this.ctx.log.debug(`TaskProcessor.detectAndProcess activeTask=${activeTask?.id ?? "none"}`);
+        if (!activeTask) {
+            await this.createNewTask(sessionKey, latestTimestamp);
+            return;
+        }
+        const isNewTask = await this.isTaskBoundary(activeTask, sessionKey, latestTimestamp);
+        if (isNewTask) {
+            await this.finalizeTask(activeTask);
+            await this.createNewTask(sessionKey, latestTimestamp);
+        }
+        else {
+            this.assignUnassignedChunks(sessionKey, activeTask.id);
+            this.store.updateTask(activeTask.id, { endedAt: undefined });
+        }
+    }
+    async isTaskBoundary(activeTask, sessionKey, latestTimestamp) {
+        if (activeTask.sessionKey !== sessionKey)
+            return true;
+        const chunks = this.store.getChunksByTask(activeTask.id);
+        if (chunks.length === 0)
+            return false;
+        const lastChunkTs = Math.max(...chunks.map((c) => c.createdAt));
+        const gap = latestTimestamp - lastChunkTs;
+        // Hard timeout: always split after 2h regardless of topic
+        if (gap > types_1.DEFAULTS.taskIdleTimeoutMs) {
+            this.ctx.log.info(`Task boundary: time gap ${Math.round(gap / 60000)}min > ${Math.round(types_1.DEFAULTS.taskIdleTimeoutMs / 60000)}min`);
+            return true;
+        }
+        // LLM topic judgment: build context from existing task and compare with new message
+        const newUserChunks = this.store.getUnassignedChunks(sessionKey).filter((c) => c.role === "user");
+        if (newUserChunks.length === 0)
+            return false;
+        const existingUserChunks = chunks.filter((c) => c.role === "user");
+        if (existingUserChunks.length === 0)
+            return false;
+        const currentContext = this.buildContextSummary(chunks);
+        const newMessage = newUserChunks.map((c) => c.content).join("\n");
+        const isNew = await this.summarizer.judgeNewTopic(currentContext, newMessage);
+        if (isNew === null) {
+            this.ctx.log.debug("Topic judge unavailable (no LLM configured), keeping current task");
+            return false;
+        }
+        if (isNew) {
+            this.ctx.log.info(`Task boundary: LLM judged new topic. New message: "${newMessage.slice(0, 80)}..."`);
+        }
+        else {
+            this.ctx.log.debug(`LLM judged SAME topic, continuing task=${activeTask.id}`);
+        }
+        return isNew;
+    }
+    /**
+     * Build a concise context string from existing task chunks for the LLM topic judge.
+     * Takes recent user/assistant summaries to keep token usage low.
+     */
+    buildContextSummary(chunks) {
+        const relevant = chunks
+            .filter((c) => c.role === "user" || c.role === "assistant")
+            .slice(-6);
+        return relevant
+            .map((c) => `[${c.role === "user" ? "User" : "Assistant"}]: ${c.summary || c.content.slice(0, 150)}`)
+            .join("\n");
+    }
+    async createNewTask(sessionKey, timestamp) {
+        const taskId = (0, uuid_1.v4)();
+        const task = {
+            id: taskId,
+            sessionKey,
+            title: "",
+            summary: "",
+            status: "active",
+            startedAt: timestamp,
+            endedAt: null,
+            updatedAt: timestamp,
+        };
+        this.store.insertTask(task);
+        this.assignUnassignedChunks(sessionKey, taskId);
+        this.ctx.log.info(`Created new task=${taskId} session=${sessionKey}`);
+    }
+    assignUnassignedChunks(sessionKey, taskId) {
+        const unassigned = this.store.getUnassignedChunks(sessionKey);
+        for (const chunk of unassigned) {
+            this.store.setChunkTaskId(chunk.id, taskId);
+        }
+        if (unassigned.length > 0) {
+            this.ctx.log.debug(`Assigned ${unassigned.length} chunks to task=${taskId}`);
+        }
+    }
+    async finalizeTask(task) {
+        const chunks = this.store.getChunksByTask(task.id);
+        const fallbackTitle = chunks.length > 0 ? this.extractTitle(chunks) : "";
+        if (chunks.length === 0) {
+            this.ctx.log.info(`Task ${task.id} skipped: no chunks`);
+            this.store.updateTask(task.id, { title: fallbackTitle, summary: SKIP_REASONS.noChunks, status: "skipped", endedAt: Date.now() });
+            return;
+        }
+        const skipReason = this.shouldSkipSummary(chunks);
+        if (skipReason) {
+            this.ctx.log.info(`Task ${task.id} skipped: ${skipReason} (chunks=${chunks.length}, title="${fallbackTitle}")`);
+            const reason = this.humanReadableSkipReason(skipReason, chunks);
+            this.store.updateTask(task.id, { title: fallbackTitle, summary: reason, status: "skipped", endedAt: Date.now() });
+            return;
+        }
+        const conversationText = this.buildConversationText(chunks);
+        let summary;
+        try {
+            summary = await this.summarizer.summarizeTask(conversationText);
+        }
+        catch (err) {
+            this.ctx.log.warn(`Task summary generation failed for task=${task.id}: ${err}`);
+            summary = this.fallbackSummary(chunks);
+        }
+        const { title: llmTitle, body } = this.parseTitleFromSummary(summary);
+        const title = llmTitle || fallbackTitle;
+        this.store.updateTask(task.id, {
+            title,
+            summary: body,
+            status: "completed",
+            endedAt: Date.now(),
+        });
+        this.ctx.log.info(`Finalized task=${task.id} title="${title}" chunks=${chunks.length} summaryLen=${body.length}`);
+        if (this.onTaskCompletedCallback) {
+            const finalized = this.store.getTask(task.id);
+            if (finalized) {
+                try {
+                    this.onTaskCompletedCallback(finalized);
+                }
+                catch (err) {
+                    this.ctx.log.warn(`TaskProcessor onTaskCompleted callback error: ${err}`);
+                }
+            }
+        }
+    }
+    /**
+     * Determine if a task is too trivial to warrant an LLM summary call.
+     * Returns a skip reason string, or null if summary should proceed.
+     *
+     * Skip conditions (any one triggers skip):
+     *  1. Total chunks < 4 — too few messages to form a meaningful task
+     *  2. Real conversation turns < 2 — no back-and-forth dialogue
+     *  3. No user messages — purely system/tool generated, no user intent
+     *  4. Total content < 200 chars — not enough substance
+     *  5. User content is trivial/test data — "hello", "test", "ok" etc.
+     *  6. All messages are tool results — automated output, no conversation
+     *  7. High content repetition — user repeated the same thing (debug loops)
+     */
+    shouldSkipSummary(chunks) {
+        const userChunks = chunks.filter((c) => c.role === "user");
+        const assistantChunks = chunks.filter((c) => c.role === "assistant");
+        const toolChunks = chunks.filter((c) => c.role === "tool");
+        // 1. Too few chunks
+        if (chunks.length < 4) {
+            return `too few chunks (${chunks.length} < 4 minimum)`;
+        }
+        // 2. Not enough real conversation turns (need at least 2 user-assistant exchanges)
+        const turns = Math.min(userChunks.length, assistantChunks.length);
+        if (turns < 2) {
+            return `too few conversation turns (${turns} < 2 minimum)`;
+        }
+        // 3. No user messages at all — purely automated
+        if (userChunks.length === 0) {
+            return "no user messages — task appears to be automated/system-generated";
+        }
+        // 4. Total content too short
+        // CJK characters carry more info per char, so use a lower threshold
+        const totalContentLen = chunks.reduce((sum, c) => sum + c.content.length, 0);
+        const hasCJK = /[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]/.test(userChunks[0]?.content ?? "");
+        const minContentLen = hasCJK ? 80 : 200;
+        if (totalContentLen < minContentLen) {
+            return `content too short (${totalContentLen} chars < ${minContentLen} minimum)`;
+        }
+        // 5. User content is trivial/test data
+        const userContent = userChunks.map((c) => c.content).join("\n");
+        if (this.looksLikeTrivialContent(userContent)) {
+            return "user content appears to be test/trivial data";
+        }
+        // 6. Assistant content is also trivial (both sides are low-value)
+        const assistantContent = assistantChunks.map((c) => c.content).join("\n");
+        if (this.looksLikeTrivialContent(userContent + "\n" + assistantContent)) {
+            return "conversation content (both user and assistant) appears trivial";
+        }
+        // 7. Almost all messages are tool results with minimal user interaction
+        if (toolChunks.length > 0 && toolChunks.length >= chunks.length * 0.7 && userChunks.length <= 1) {
+            return `dominated by tool results (${toolChunks.length}/${chunks.length} chunks) with minimal user input`;
+        }
+        // 8. High repetition — user keeps saying the same thing
+        if (userChunks.length >= 3) {
+            const uniqueUserMsgs = new Set(userChunks.map((c) => c.content.trim().toLowerCase()));
+            const uniqueRatio = uniqueUserMsgs.size / userChunks.length;
+            if (uniqueRatio < 0.4) {
+                return `high content repetition (${uniqueUserMsgs.size} unique out of ${userChunks.length} user messages)`;
+            }
+        }
+        return null;
+    }
+    looksLikeTrivialContent(text) {
+        const lines = text.toLowerCase().split(/\n/).map((l) => l.trim()).filter(Boolean);
+        if (lines.length === 0)
+            return true;
+        const trivialCount = lines.filter((line) => {
+            if (line.length < 5)
+                return true;
+            if (TRIVIAL_PATTERNS.some((p) => p.test(line)))
+                return true;
+            return false;
+        }).length;
+        return trivialCount / lines.length > 0.7;
+    }
+    buildConversationText(chunks) {
+        const lines = [];
+        for (const c of chunks) {
+            const roleLabel = c.role === "user" ? "User" : c.role === "assistant" ? "Assistant" : c.role;
+            lines.push(`[${roleLabel}]: ${c.content}`);
+        }
+        return lines.join("\n\n");
+    }
+    /**
+     * Extract the LLM-generated title from the summary output.
+     * The LLM is prompted to output "📌 Title\n<title text>" as the first section.
+     * Returns the title and the remaining body (with the title section stripped).
+     */
+    parseTitleFromSummary(summary) {
+        const titleMatch = summary.match(/📌\s*(?:Title|标题)\s*\n(.+)/);
+        if (titleMatch) {
+            const title = titleMatch[1].trim().slice(0, 80);
+            const body = summary.replace(/📌\s*(?:Title|标题)\s*\n.+\n?/, "").trim();
+            return { title, body };
+        }
+        return { title: "", body: summary };
+    }
+    extractTitle(chunks) {
+        const firstUser = chunks.find((c) => c.role === "user");
+        if (!firstUser)
+            return "Untitled Task";
+        const text = firstUser.content.trim();
+        if (text.length <= 60)
+            return text;
+        return text.slice(0, 57) + "...";
+    }
+    humanReadableSkipReason(reason, chunks) {
+        const userCount = chunks.filter((c) => c.role === "user").length;
+        const assistantCount = chunks.filter((c) => c.role === "assistant").length;
+        if (reason.includes("too few chunks")) {
+            return `对话内容过少（${chunks.length} 条消息），不足以生成有效摘要。至少需要 4 条消息。`;
+        }
+        if (reason.includes("too few conversation turns")) {
+            return `对话轮次不足（${Math.min(userCount, assistantCount)} 轮），需要至少 2 轮完整的问答交互才能生成摘要。`;
+        }
+        if (reason.includes("no user messages")) {
+            return "该任务没有用户消息，仅包含系统或工具自动生成的内容。";
+        }
+        if (reason.includes("content too short")) {
+            return "对话内容过短，信息量不足以生成有意义的摘要。";
+        }
+        if (reason.includes("trivial")) {
+            return "对话内容为简单问候或测试数据（如 hello、test、ok），无需生成摘要。";
+        }
+        if (reason.includes("tool results")) {
+            return "该任务主要由工具执行结果组成，缺少足够的用户交互内容。";
+        }
+        if (reason.includes("repetition")) {
+            return "对话中存在大量重复内容，无法提取有效信息生成摘要。";
+        }
+        return `对话未达到生成摘要的条件：${reason}`;
+    }
+    fallbackSummary(chunks) {
+        const title = this.extractTitle(chunks);
+        const summaries = chunks
+            .filter((c) => c.summary)
+            .map((c) => `- ${c.summary}`);
+        const lines = [
+            `🎯 Goal`,
+            title,
+            ``,
+            `📋 Key Steps`,
+            ...summaries.slice(0, 20),
+        ];
+        return lines.join("\n");
+    }
+}
+exports.TaskProcessor = TaskProcessor;
+//# sourceMappingURL=task-processor.js.map

package/dist/ingest/task-processor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"task-processor.js","sourceRoot":"","sources":["../../src/ingest/task-processor.ts"],"names":[],"mappings":";;;AAAA,+BAAkC;AAGlC,oCAAoC;AACpC,2CAAyC;AAEzC,MAAM,gBAAgB,GAAG;IACvB,+HAA+H;IAC/H,6DAA6D;IAC7D,oBAAoB;CACrB,CAAC;AAEF,MAAM,YAAY,GAAG;IACnB,QAAQ,EAAE,kBAAkB;CACpB,CAAC;AAEX;;;;;;;;;;;;GAYG;AACH,MAAa,aAAa;IAMd;IACA;IANF,UAAU,CAAa;IACvB,UAAU,GAAG,KAAK,CAAC;IACnB,uBAAuB,CAAwB;IAEvD,YACU,KAAkB,EAClB,GAAkB;QADlB,UAAK,GAAL,KAAK,CAAa;QAClB,QAAG,GAAH,GAAG,CAAe;QAE1B,IAAI,CAAC,UAAU,GAAG,IAAI,sBAAU,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IACnE,CAAC;IAED,eAAe,CAAC,EAAwB;QACtC,IAAI,CAAC,uBAAuB,GAAG,EAAE,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,gBAAgB,CAAC,UAAkB,EAAE,eAAuB;QAChE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,iDAAiD,UAAU,OAAO,eAAe,eAAe,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QACtI,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,6DAA6D,CAAC,CAAC;YAClF,OAAO;QACT,CAAC;QACD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;QAC3D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,GAAG,EAAE,CAAC,CAAC;QACpD,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QAC1B,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,UAAkB,EAAE,eAAuB;QACxE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,0CAA0C,UAAU,EAAE,CAAC,CAAC;QAE3E,iFAAiF;QACjF,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,iBAAiB,EAAE,CAAC;QACjD,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;gBAChC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;gBAC3F,MAAM,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QACxD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,6CAA6C,UAAU,EAAE,EAAE,IAAI,MAAM,EAAE,CAAC,CAAC;QAE5F,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;YACtD,OAAO;QACT,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,EAAE,UAAU,EAAE,eAAe,CAAC,CAAC;QAErF,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;YACpC,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;QACxD,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,sBAAsB,CAAC,UAAU,EAAE,UAAU,CAAC,EAAE,CAAC,CAAC;YACvD,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,UAAgB,EAAE,UAAkB,EAAE,eAAuB;QACxF,IAAI,UAAU,CAAC,UAAU,KAAK,UAAU;YAAE,OAAO,IAAI,CAAC;QAEtD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACzD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QAEtC,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;QAChE,MAAM,GAAG,GAAG,eAAe,GAAG,WAAW,CAAC;QAE1C,0DAA0D;QAC1D,IAAI,GAAG,GAAG,gBAAQ,CAAC,iBAAiB,EAAE,CAAC;YACrC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CACf,2BAA2B,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,SAAS,IAAI,CAAC,KAAK,CAAC,gBAAQ,CAAC,iBAAiB,GAAG,KAAK,CAAC,KAAK,CAC/G,CAAC;YACF,OAAO,IAAI,CAAC;QACd,CAAC;QAED,oFAAoF;QACpF,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;QAClG,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QAE7C,MAAM,kBAAkB,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;QACnE,IAAI,kBAAkB,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QAElD,MAAM,cAAc,GAAG,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;QACxD,MAAM,UAAU,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAElE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,cAAc,EAAE,UAAU,CAAC,CAAC;QAE9E,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACnB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,mEAAmE,CAAC,CAAC;YACxF,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,sDAAsD,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QACzG,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,0CAA0C,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAChF,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;;OAGG;IACK,mBAAmB,CAAC,MAAe;QACzC,MAAM,QAAQ,GAAG,MAAM;aACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC;aAC1D,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAEb,OAAO,QAAQ;aACZ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,MAAM,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;aACpG,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,UAAkB,EAAE,SAAiB;QAC/D,MAAM,MAAM,GAAG,IAAA,SAAI,GAAE,CAAC;QACtB,MAAM,IAAI,GAAS;YACjB,EAAE,EAAE,MAAM;YACV,UAAU;YACV,KAAK,EAAE,EAAE;YACT,OAAO,EAAE,EAAE;YACX,MAAM,EAAE,QAAQ;YAChB,SAAS,EAAE,SAAS;YACpB,OAAO,EAAE,IAAI;YACb,SAAS,EAAE,SAAS;SACrB,CAAC;QACF,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAC5B,IAAI,CAAC,sBAAsB,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAChD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,oBAAoB,MAAM,YAAY,UAAU,EAAE,CAAC,CAAC;IACxE,CAAC;IAEO,sBAAsB,CAAC,UAAkB,EAAE,MAAc;QAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;QAC9D,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YAC/B,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QAC9C,CAAC;QACD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,UAAU,CAAC,MAAM,mBAAmB,MAAM,EAAE,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,IAAU;QAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACnD,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAEzE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,EAAE,qBAAqB,CAAC,CAAC;YACxD,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACjI,OAAO;QACT,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAElD,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,EAAE,aAAa,UAAU,YAAY,MAAM,CAAC,MAAM,YAAY,aAAa,IAAI,CAAC,CAAC;YAChH,MAAM,MAAM,GAAG,IAAI,CAAC,uBAAuB,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YAChE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAClH,OAAO;QACT,CAAC;QAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC;QAC5D,IAAI,OAAe,CAAC;QACpB,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAC;QAClE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,2CAA2C,IAAI,CAAC,EAAE,KAAK,GAAG,EAAE,CAAC,CAAC;YAChF,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;QACzC,CAAC;QAED,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;QACtE,MAAM,KAAK,GAAG,QAAQ,IAAI,aAAa,CAAC;QAExC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE;YAC7B,KAAK;YACL,OAAO,EAAE,IAAI;YACb,MAAM,EAAE,WAAW;YACnB,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE;SACpB,CAAC,CAAC;QAEH,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CACf,kBAAkB,IAAI,CAAC,EAAE,WAAW,KAAK,YAAY,MAAM,CAAC,MAAM,eAAe,IAAI,CAAC,MAAM,EAAE,CAC/F,CAAC;QAEF,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YACjC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC9C,IAAI,SAAS,EAAE,CAAC;gBACd,IAAI,CAAC;oBACH,IAAI,CAAC,uBAAuB,CAAC,SAAS,CAAC,CAAC;gBAC1C,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,iDAAiD,GAAG,EAAE,CAAC,CAAC;gBAC5E,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;;;;;;;;OAYG;IACK,iBAAiB,CAAC,MAAe;QACvC,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;QAC3D,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC;QACrE,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;QAE3D,oBAAoB;QACpB,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,mBAAmB,MAAM,CAAC,MAAM,eAAe,CAAC;QACzD,CAAC;QAED,mFAAmF;QACnF,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC;QAClE,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,OAAO,+BAA+B,KAAK,eAAe,CAAC;QAC7D,CAAC;QAED,gDAAgD;QAChD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,kEAAkE,CAAC;QAC5E,CAAC;QAED,6BAA6B;QAC7B,oEAAoE;QACpE,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC7E,MAAM,MAAM,GAAG,2CAA2C,CAAC,IAAI,CAC7D,UAAU,CAAC,CAAC,CAAC,EAAE,OAAO,IAAI,EAAE,CAC7B,CAAC;QACF,MAAM,aAAa,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QACxC,IAAI,eAAe,GAAG,aAAa,EAAE,CAAC;YACpC,OAAO,sBAAsB,eAAe,YAAY,aAAa,WAAW,CAAC;QACnF,CAAC;QAED,uCAAuC;QACvC,MAAM,WAAW,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChE,IAAI,IAAI,CAAC,uBAAuB,CAAC,WAAW,CAAC,EAAE,CAAC;YAC9C,OAAO,8CAA8C,CAAC;QACxD,CAAC;QAED,kEAAkE;QAClE,MAAM,gBAAgB,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1E,IAAI,IAAI,CAAC,uBAAuB,CAAC,WAAW,GAAG,IAAI,GAAG,gBAAgB,CAAC,EAAE,CAAC;YACxE,OAAO,gEAAgE,CAAC;QAC1E,CAAC;QAED,wEAAwE;QACxE,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAChG,OAAO,8BAA8B,UAAU,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,kCAAkC,CAAC;QAC5G,CAAC;QAED,wDAAwD;QACxD,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;YACtF,MAAM,WAAW,GAAG,cAAc,CAAC,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC;YAC5D,IAAI,WAAW,GAAG,GAAG,EAAE,CAAC;gBACtB,OAAO,4BAA4B,cAAc,CAAC,IAAI,kBAAkB,UAAU,CAAC,MAAM,iBAAiB,CAAC;YAC7G,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,uBAAuB,CAAC,IAAY;QAC1C,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAClF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAEpC,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;YACzC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,IAAI,CAAC;YACjC,IAAI,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC5D,OAAO,KAAK,CAAC;QACf,CAAC,CAAC,CAAC,MAAM,CAAC;QAEV,OAAO,YAAY,GAAG,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC;IAC3C,CAAC;IAEO,qBAAqB,CAAC,MAAe;QAC3C,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,MAAM,SAAS,GAAG,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAC7F,KAAK,CAAC,IAAI,CAAC,IAAI,SAAS,MAAM,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7C,CAAC;QACD,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;IAED;;;;OAIG;IACK,qBAAqB,CAAC,OAAe;QAC3C,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAC/D,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAChD,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,6BAA6B,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YACvE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;QACzB,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IACtC,CAAC;IAEO,YAAY,CAAC,MAAe;QAClC,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;QACxD,IAAI,CAAC,SAAS;YAAE,OAAO,eAAe,CAAC;QACvC,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QACtC,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE;YAAE,OAAO,IAAI,CAAC;QACnC,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC;IACnC,CAAC;IAEO,uBAAuB,CAAC,MAAc,EAAE,MAAe;QAC7D,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;QACjE,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,MAAM,CAAC;QAE3E,IAAI,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;YACtC,OAAO,UAAU,MAAM,CAAC,MAAM,6BAA6B,CAAC;QAC9D,CAAC;QACD,IAAI,MAAM,CAAC,QAAQ,CAAC,4BAA4B,CAAC,EAAE,CAAC;YAClD,OAAO,UAAU,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,cAAc,CAAC,4BAA4B,CAAC;QACnF,CAAC;QACD,IAAI,MAAM,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACxC,OAAO,4BAA4B,CAAC;QACtC,CAAC;QACD,IAAI,MAAM,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC;YACzC,OAAO,wBAAwB,CAAC;QAClC,CAAC;QACD,IAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YAC/B,OAAO,yCAAyC,CAAC;QACnD,CAAC;QACD,IAAI,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;YACpC,OAAO,6BAA6B,CAAC;QACvC,CAAC;QACD,IAAI,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;YAClC,OAAO,2BAA2B,CAAC;QACrC,CAAC;QACD,OAAO,gBAAgB,MAAM,EAAE,CAAC;IAClC,CAAC;IAEO,eAAe,CAAC,MAAe;QACrC,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,MAAM;aACrB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;aACxB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAChC,MAAM,KAAK,GAAG;YACZ,SAAS;YACT,KAAK;YACL,EAAE;YACF,cAAc;YACd,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;SAC1B,CAAC;QACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;CACF;AA9WD,sCA8WC"}

package/dist/ingest/worker.d.ts

@@ -1,21 +1,23 @@
 import type { ConversationMessage, PluginContext } from "../types";
 import type { SqliteStore } from "../storage/sqlite";
 import type { Embedder } from "../embedding";
+import { TaskProcessor } from "./task-processor";
 export declare class IngestWorker {
     private store;
     private embedder;
     private ctx;
     private summarizer;
+    private taskProcessor;
     private queue;
     private processing;
     private flushResolvers;
     constructor(store: SqliteStore, embedder: Embedder, ctx: PluginContext);
+    getTaskProcessor(): TaskProcessor;
     enqueue(messages: ConversationMessage[]): void;
     /** Wait until all queued messages have been processed. */
     flush(): Promise<void>;
     private processQueue;
     private ingestMessage;
-    private ingestToolResult;
     private storeChunk;
 }
 //# sourceMappingURL=worker.d.ts.map

package/dist/ingest/worker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../../src/ingest/worker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,mBAAmB,EAAS,aAAa,EAAE,MAAM,UAAU,CAAC;AAC1E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAK7C,qBAAa,YAAY;IAOrB,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,QAAQ;IAChB,OAAO,CAAC,GAAG;IARb,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,KAAK,CAA6B;IAC1C,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,cAAc,CAAyB;gBAGrC,KAAK,EAAE,WAAW,EAClB,QAAQ,EAAE,QAAQ,EAClB,GAAG,EAAE,aAAa;IAK5B,OAAO,CAAC,QAAQ,EAAE,mBAAmB,EAAE,GAAG,IAAI;IAU9C,0DAA0D;IACpD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAOd,YAAY;YAiBZ,aAAa;YAeb,gBAAgB;YAKhB,UAAU;CAoDzB"}
+{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../../src/ingest/worker.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,mBAAmB,EAAS,aAAa,EAAE,MAAM,UAAU,CAAC;AAC1E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAG7C,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,qBAAa,YAAY;IAQrB,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,QAAQ;IAChB,OAAO,CAAC,GAAG;IATb,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,aAAa,CAAgB;IACrC,OAAO,CAAC,KAAK,CAA6B;IAC1C,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,cAAc,CAAyB;gBAGrC,KAAK,EAAE,WAAW,EAClB,QAAQ,EAAE,QAAQ,EAClB,GAAG,EAAE,aAAa;IAM5B,gBAAgB,IAAI,aAAa;IAEjC,OAAO,CAAC,QAAQ,EAAE,mBAAmB,EAAE,GAAG,IAAI;IAU9C,0DAA0D;IACpD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAOd,YAAY;YAqEZ,aAAa;YAYb,UAAU;CAkHzB"}

package/dist/ingest/worker.js

@@ -3,13 +3,14 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.IngestWorker = void 0;
 const uuid_1 = require("uuid");
 const providers_1 = require("./providers");
-const chunker_1 = require("./chunker");
 const dedup_1 = require("./dedup");
+const task_processor_1 = require("./task-processor");
 class IngestWorker {
     store;
     embedder;
     ctx;
     summarizer;
+    taskProcessor;
     queue = [];
     processing = false;
     flushResolvers = [];
@@ -18,7 +19,9 @@ class IngestWorker {
         this.embedder = embedder;
         this.ctx = ctx;
         this.summarizer = new providers_1.Summarizer(ctx.config.summarizer, ctx.log);
+        this.taskProcessor = new task_processor_1.TaskProcessor(store, ctx);
     }
+    getTaskProcessor() { return this.taskProcessor; }
     enqueue(messages) {
         this.queue.push(...messages);
         if (!this.processing) {
@@ -38,35 +41,79 @@ class IngestWorker {
     }
     async processQueue() {
         this.processing = true;
+        const t0 = performance.now();
+        let lastSessionKey;
+        let lastTimestamp = 0;
+        let stored = 0;
+        let skipped = 0;
+        let merged = 0;
+        let duplicated = 0;
+        let errors = 0;
+        const resultLines = [];
+        const inputLines = [];
+        const totalMessages = this.queue.length;
         while (this.queue.length > 0) {
             const msg = this.queue.shift();
+            inputLines.push(`[${msg.role}] ${msg.content}`);
             try {
-                await this.ingestMessage(msg);
+                const result = await this.ingestMessage(msg);
+                lastSessionKey = msg.sessionKey;
+                lastTimestamp = Math.max(lastTimestamp, msg.timestamp);
+                if (result === "skipped") {
+                    skipped++;
+                    resultLines.push(`[${msg.role}] ⏭ exact-dup → ${msg.content}`);
+                }
+                else if (result.action === "stored") {
+                    stored++;
+                    resultLines.push(`[${msg.role}] ✅ stored → ${result.summary ?? msg.content}`);
+                }
+                else if (result.action === "duplicate") {
+                    duplicated++;
+                    resultLines.push(`[${msg.role}] 🔁 dedup(${result.reason ?? "similar"}) → ${msg.content}`);
+                }
+                else if (result.action === "merged") {
+                    merged++;
+                    resultLines.push(`[${msg.role}] 🔀 merged → ${msg.content}`);
+                }
             }
             catch (err) {
+                errors++;
+                resultLines.push(`[${msg.role}] ❌ error → ${msg.content}`);
                 this.ctx.log.error(`Failed to ingest message turn=${msg.turnId}: ${err}`);
             }
         }
+        const dur = performance.now() - t0;
+        if (stored + merged > 0 || skipped > 0 || duplicated > 0) {
+            this.store.recordToolCall("memory_add", dur, errors === 0);
+            try {
+                const inputInfo = {
+                    session: lastSessionKey,
+                    messages: totalMessages,
+                    details: inputLines,
+                };
+                const stats = [`stored=${stored}`, skipped > 0 ? `skipped=${skipped}` : null, duplicated > 0 ? `dedup=${duplicated}` : null, merged > 0 ? `merged=${merged}` : null, errors > 0 ? `errors=${errors}` : null].filter(Boolean).join(", ");
+                this.store.recordApiLog("memory_add", inputInfo, `${stats}\n${resultLines.join("\n")}`, dur, errors === 0);
+            }
+            catch (_) { /* best-effort */ }
+        }
+        if (lastSessionKey) {
+            this.ctx.log.debug(`Calling TaskProcessor.onChunksIngested session=${lastSessionKey} ts=${lastTimestamp}`);
+            this.taskProcessor
+                .onChunksIngested(lastSessionKey, lastTimestamp)
+                .catch((err) => this.ctx.log.error(`TaskProcessor post-ingest error: ${err}`));
+        }
         this.processing = false;
         for (const resolve of this.flushResolvers)
             resolve();
         this.flushResolvers = [];
     }
     async ingestMessage(msg) {
-        if (msg.role === "tool") {
-            await this.ingestToolResult(msg);
-            return;
+        if (this.store.chunkExistsByContent(msg.sessionKey, msg.role, msg.content)) {
+            this.ctx.log.debug(`Exact-dup (same session+role+hash), skipping: session=${msg.sessionKey} role=${msg.role} len=${msg.content.length}`);
+            return "skipped";
         }
-        const rawChunks = (0, chunker_1.chunkText)(msg.content);
-        this.ctx.log.debug(`Chunked turn=${msg.turnId} into ${rawChunks.length} chunks`);
-        for (let seq = 0; seq < rawChunks.length; seq++) {
-            const raw = rawChunks[seq];
-            await this.storeChunk(msg, raw.content, raw.kind, seq);
-        }
-    }
-    async ingestToolResult(msg) {
-        this.ctx.log.debug(`Ingesting tool result turn=${msg.turnId} tool=${msg.toolName ?? "unknown"} len=${msg.content.length}`);
-        await this.storeChunk(msg, msg.content, "tool_result", 0);
+        const kind = msg.role === "tool" ? "tool_result" : "paragraph";
+        return await this.storeChunk(msg, msg.content, kind, 0);
     }
     async storeChunk(msg, content, kind, seq) {
         const chunkId = (0, uuid_1.v4)();
@@ -78,13 +125,59 @@ class IngestWorker {
         catch (err) {
             this.ctx.log.warn(`Embedding failed for chunk=${chunkId}, storing without vector: ${err}`);
         }
+        let dedupStatus = "active";
+        let dedupTarget = null;
+        let dedupReason = null;
+        // Smart dedup: find Top-5 similar chunks, then ask LLM to judge
         if (embedding) {
-            const dupId = (0, dedup_1.findDuplicate)(this.store, embedding, this.ctx.config.dedup?.similarityThreshold ?? 0.93, this.ctx.log);
-            if (dupId) {
-                this.store.updateSummary(dupId, summary);
-                this.store.upsertEmbedding(dupId, embedding);
-                this.ctx.log.debug(`Dedup-merged into existing chunk=${dupId}`);
-                return;
+            const similarThreshold = this.ctx.config.dedup?.similarityThreshold ?? 0.75;
+            const topSimilar = (0, dedup_1.findTopSimilar)(this.store, embedding, similarThreshold, 5, this.ctx.log);
+            if (topSimilar.length > 0) {
+                const candidates = topSimilar.map((s, i) => {
+                    const chunk = this.store.getChunk(s.chunkId);
+                    return {
+                        index: i + 1,
+                        summary: chunk?.summary ?? "",
+                        chunkId: s.chunkId,
+                    };
+                }).filter(c => c.summary);
+                if (candidates.length > 0) {
+                    const dedupResult = await this.summarizer.judgeDedup(summary, candidates);
+                    if (dedupResult && dedupResult.action === "DUPLICATE" && dedupResult.targetIndex) {
+                        const targetChunkId = candidates[dedupResult.targetIndex - 1]?.chunkId;
+                        if (targetChunkId) {
+                            this.store.recordMergeHit(targetChunkId, "DUPLICATE", dedupResult.reason);
+                            dedupStatus = "duplicate";
+                            dedupTarget = targetChunkId;
+                            dedupReason = dedupResult.reason;
+                            this.ctx.log.debug(`Smart dedup: DUPLICATE → target=${targetChunkId}, storing with status=duplicate, reason: ${dedupResult.reason}`);
+                        }
+                    }
+                    if (dedupStatus === "active" && dedupResult && dedupResult.action === "UPDATE" && dedupResult.targetIndex && dedupResult.mergedSummary) {
+                        const targetChunkId = candidates[dedupResult.targetIndex - 1]?.chunkId;
+                        if (targetChunkId) {
+                            const oldChunk = this.store.getChunk(targetChunkId);
+                            const oldSummary = oldChunk?.summary ?? "";
+                            this.store.recordMergeHit(targetChunkId, "UPDATE", dedupResult.reason, oldSummary, dedupResult.mergedSummary);
+                            this.store.updateChunkSummaryAndContent(targetChunkId, dedupResult.mergedSummary, content);
+                            try {
+                                const [newEmb] = await this.embedder.embed([dedupResult.mergedSummary]);
+                                if (newEmb)
+                                    this.store.upsertEmbedding(targetChunkId, newEmb);
+                            }
+                            catch (err) {
+                                this.ctx.log.warn(`Re-embed after UPDATE failed: ${err}`);
+                            }
+                            dedupStatus = "merged";
+                            dedupTarget = targetChunkId;
+                            dedupReason = dedupResult.reason;
+                            this.ctx.log.debug(`Smart dedup: UPDATE → merged into chunk=${targetChunkId}, storing with status=merged, reason: ${dedupResult.reason}`);
+                        }
+                    }
+                    if (dedupStatus === "active") {
+                        this.ctx.log.debug(`Smart dedup: NEW — creating active chunk (reason: ${dedupResult?.reason ?? "no_result"})`);
+                    }
+                }
             }
         }
         const chunk = {
@@ -97,14 +190,29 @@ class IngestWorker {
             kind,
             summary,
             embedding: null,
+            taskId: null,
+            skillId: null,
+            dedupStatus,
+            dedupTarget,
+            dedupReason,
+            mergeCount: 0,
+            lastHitAt: null,
+            mergeHistory: "[]",
             createdAt: msg.timestamp,
             updatedAt: msg.timestamp,
         };
         this.store.insertChunk(chunk);
-        if (embedding) {
+        if (embedding && dedupStatus === "active") {
             this.store.upsertEmbedding(chunkId, embedding);
         }
-        this.ctx.log.debug(`Stored chunk=${chunkId} kind=${kind} role=${msg.role} len=${content.length} hasVec=${!!embedding}`);
+        this.ctx.log.debug(`Stored chunk=${chunkId} kind=${kind} role=${msg.role} dedup=${dedupStatus} len=${content.length} hasVec=${!!embedding && dedupStatus === "active"}`);
+        if (dedupStatus === "duplicate") {
+            return { action: "duplicate", summary, targetChunkId: dedupTarget ?? undefined, reason: dedupReason ?? undefined };
+        }
+        if (dedupStatus === "merged") {
+            return { action: "merged", summary, targetChunkId: dedupTarget ?? undefined, reason: dedupReason ?? undefined };
+        }
+        return { action: "stored", chunkId, summary };
     }
 }
 exports.IngestWorker = IngestWorker;