@memtensor/memos-local-openclaw-plugin 1.0.7 → 1.0.8-beta.10

package/src/ingest/providers/openai.ts

@@ -188,19 +188,26 @@ SAME — the new message:
 - Reports a result, error, or feedback about the current task
 - Discusses different tools or approaches for the SAME goal (e.g., learning English via BBC → via ChatGPT = SAME)
 - Is a short acknowledgment (ok, thanks, 好的) in response to the current flow
+- Is a follow-up, update, or different angle on the same news event, person, or story
+- Shares the same core entity (person, company, event) even if the specific detail or angle differs
+- Contains pronouns or references (那, 这, 它, 其中, 哪些, those, which, what about, etc.) pointing to items from the current conversation
+- Asks about a sub-topic, tool, detail, dimension, or aspect of the current discussion topic
 
 NEW — the new message:
-- Introduces a subject from a DIFFERENT domain than the current task (e.g., tech → cooking, work → personal life, database → travel)
-- Has NO logical connection to what was being discussed
+- Introduces a subject from a COMPLETELY DIFFERENT domain than the current task (e.g., tech → cooking, work → personal life, database → travel)
+- Has NO logical connection to what was being discussed — no shared entities, events, or themes
 - Starts a request about a different project, system, or life area
 - Begins with a new greeting/reset followed by a different topic
 
 Key principles:
-- If the topic domain clearly changed (e.g., server config → recipe, code review → vacation plan), choose NEW
+- Default to SAME unless the topic domain CLEARLY changed. When in doubt, choose SAME.
+- CRITICAL: Short messages (under ~30 characters) that use pronouns or ask "what about X" / "哪些" / "那XX呢" are almost always follow-ups referring to the current topic. Only mark them NEW if they explicitly name a completely unrelated domain.
+- If the new message mentions the same person, event, product, or entity as the current task, it is SAME regardless of the angle
 - Different aspects of the SAME project/system are SAME (e.g., Nginx SSL → Nginx gzip = SAME)
-- Different unrelated technologies discussed independently are NEW (e.g., Redis config → cooking recipe = NEW)
-- When unsure, lean toward SAME for closely related topics, but do NOT hesitate to mark NEW for obvious domain shifts
-- Examples: "配置Nginx" → "加gzip压缩" = SAME; "配置Nginx" → "做红烧肉" = NEW; "MySQL配置" → "K8s部署" in same infra project = SAME; "部署服务器" → "年会安排" = NEW
+- Asking about tools, systems, or methods for the current topic is SAME (e.g., "港股调研" → "那处理系统有哪些" = SAME; "数据分析" → "用什么工具" = SAME)
+- Follow-up news about the same event is SAME (e.g., "博士失联" → "博士遗体被找到" = SAME; "产品发布" → "产品销量" = SAME)
+- Different unrelated domains discussed independently are NEW (e.g., Redis config → cooking recipe = NEW)
+- Examples: "配置Nginx" → "加gzip压缩" = SAME; "配置Nginx" → "做红烧肉" = NEW; "港股调研" → "那处理系统有哪些" = SAME; "部署服务器" → "年会安排" = NEW
 
 Output exactly one word: NEW or SAME`;
 
@@ -246,6 +253,134 @@ export async function judgeNewTopicOpenAI(
   return answer.startsWith("NEW");
 }
 
+// ─── Structured Topic Classifier ───
+
+export interface TopicClassifyResult {
+  decision: "NEW" | "SAME";
+  confidence: number;
+  boundaryType: string;
+  reason: string; // may be empty for compact responses
+}
+
+const TOPIC_CLASSIFIER_PROMPT = `Classify if NEW MESSAGE continues current task or starts an unrelated one.
+Output ONLY JSON: {"d":"S"|"N","c":0.0-1.0}
+d=S(same) or N(new). c=confidence. Default S. Only N if completely unrelated domain.
+Sub-questions, tools, methods, details of current topic = S.`;
+
+export async function classifyTopicOpenAI(
+  taskState: string,
+  newMessage: string,
+  cfg: SummarizerConfig,
+  log: Logger,
+): Promise<TopicClassifyResult> {
+  const endpoint = normalizeChatEndpoint(cfg.endpoint ?? "https://api.openai.com/v1/chat/completions");
+  const model = cfg.model ?? "gpt-4o-mini";
+  const headers: Record<string, string> = {
+    "Content-Type": "application/json",
+    Authorization: `Bearer ${cfg.apiKey}`,
+    ...cfg.headers,
+  };
+
+  const userContent = `TASK:\n${taskState}\n\nMSG:\n${newMessage}`;
+
+  const resp = await fetch(endpoint, {
+    method: "POST",
+    headers,
+    body: JSON.stringify(buildRequestBody(cfg, {
+      model,
+      temperature: 0,
+      max_tokens: 60,
+      messages: [
+        { role: "system", content: TOPIC_CLASSIFIER_PROMPT },
+        { role: "user", content: userContent },
+      ],
+    })),
+    signal: AbortSignal.timeout(cfg.timeoutMs ?? 15_000),
+  });
+
+  if (!resp.ok) {
+    const body = await resp.text();
+    throw new Error(`OpenAI topic-classifier failed (${resp.status}): ${body}`);
+  }
+
+  const json = (await resp.json()) as { choices: Array<{ message: { content: string } }> };
+  const raw = json.choices[0]?.message?.content?.trim() ?? "";
+  log.debug(`Topic classifier raw: "${raw}"`);
+
+  return parseTopicClassifyResult(raw, log);
+}
+
+const TOPIC_ARBITRATION_PROMPT = `A classifier flagged this message as possibly new topic (low confidence). Is it truly UNRELATED, or a sub-question/follow-up?
+Tools/methods/details of current task = SAME. Shared entity/theme = SAME. Entirely different domain = NEW.
+Reply one word: NEW or SAME`;
+
+export async function arbitrateTopicSplitOpenAI(
+  taskState: string,
+  newMessage: string,
+  cfg: SummarizerConfig,
+  log: Logger,
+): Promise<string> {
+  const endpoint = normalizeChatEndpoint(cfg.endpoint ?? "https://api.openai.com/v1/chat/completions");
+  const model = cfg.model ?? "gpt-4o-mini";
+  const headers: Record<string, string> = {
+    "Content-Type": "application/json",
+    Authorization: `Bearer ${cfg.apiKey}`,
+    ...cfg.headers,
+  };
+
+  const userContent = `TASK:\n${taskState}\n\nMSG:\n${newMessage}`;
+
+  const resp = await fetch(endpoint, {
+    method: "POST",
+    headers,
+    body: JSON.stringify(buildRequestBody(cfg, {
+      model,
+      temperature: 0,
+      max_tokens: 10,
+      messages: [
+        { role: "system", content: TOPIC_ARBITRATION_PROMPT },
+        { role: "user", content: userContent },
+      ],
+    })),
+    signal: AbortSignal.timeout(cfg.timeoutMs ?? 15_000),
+  });
+
+  if (!resp.ok) {
+    const body = await resp.text();
+    throw new Error(`OpenAI topic-arbitration failed (${resp.status}): ${body}`);
+  }
+
+  const json = (await resp.json()) as { choices: Array<{ message: { content: string } }> };
+  const answer = json.choices[0]?.message?.content?.trim().toUpperCase() ?? "";
+  log.debug(`Topic arbitration result: "${answer}"`);
+  return answer.startsWith("NEW") ? "NEW" : "SAME";
+}
+
+export function parseTopicClassifyResult(raw: string, log: Logger): TopicClassifyResult {
+  try {
+    const jsonMatch = raw.match(/\{[\s\S]*\}/);
+    if (jsonMatch) {
+      const p = JSON.parse(jsonMatch[0]);
+      const decision: "NEW" | "SAME" =
+        (p.d === "N" || p.decision === "NEW") ? "NEW" : "SAME";
+      const confidence: number =
+        typeof p.c === "number" ? p.c : typeof p.confidence === "number" ? p.confidence : 0.5;
+      return {
+        decision,
+        confidence,
+        boundaryType: p.boundaryType || "",
+        reason: p.reason || "",
+      };
+    }
+  } catch (err) {
+    log.debug(`Failed to parse topic classify JSON: ${err}`);
+  }
+  const upper = raw.toUpperCase();
+  if (upper.startsWith("NEW") || upper.startsWith("N"))
+    return { decision: "NEW", confidence: 0.5, boundaryType: "", reason: "parse fallback" };
+  return { decision: "SAME", confidence: 0.5, boundaryType: "", reason: "parse fallback" };
+}
+
 const FILTER_RELEVANT_PROMPT = `You are a memory relevance judge.
 
 Given a QUERY and CANDIDATE memories, decide: does each candidate's content contain information that would HELP ANSWER the query?

package/src/ingest/task-processor.ts

@@ -51,6 +51,9 @@ export class TaskProcessor {
    * Determines if a new task boundary was crossed and handles transition.
    */
   async onChunksIngested(sessionKey: string, latestTimestamp: number, owner?: string): Promise<void> {
+    if (sessionKey.startsWith("temp:") || sessionKey.startsWith("internal:") || sessionKey.startsWith("system:")) {
+      return;
+    }
     const resolvedOwner = owner ?? "agent:main";
     this.ctx.log.debug(`TaskProcessor.onChunksIngested called session=${sessionKey} ts=${latestTimestamp} owner=${resolvedOwner} processing=${this.processing}`);
     this.pendingEvents.push({ sessionKey, latestTimestamp, owner: resolvedOwner });
@@ -79,13 +82,19 @@ export class TaskProcessor {
     }
   }
 
+  private static extractAgentPrefix(sessionKey: string): string {
+    const parts = sessionKey.split(":");
+    return parts.length >= 3 ? parts.slice(0, 3).join(":") : sessionKey;
+  }
+
   private async detectAndProcess(sessionKey: string, latestTimestamp: number, owner: string): Promise<void> {
     this.ctx.log.debug(`TaskProcessor.detectAndProcess session=${sessionKey} owner=${owner}`);
 
+    const currentAgentPrefix = TaskProcessor.extractAgentPrefix(sessionKey);
     const allActive = this.store.getAllActiveTasks(owner);
     for (const t of allActive) {
-      if (t.sessionKey !== sessionKey) {
-        this.ctx.log.info(`Session changed: finalizing task=${t.id} from session=${t.sessionKey} (owner=${owner})`);
+      if (t.sessionKey !== sessionKey && TaskProcessor.extractAgentPrefix(t.sessionKey) === currentAgentPrefix) {
+        this.ctx.log.info(`Session changed within agent: finalizing task=${t.id} from session=${t.sessionKey} (owner=${owner})`);
         await this.finalizeTask(t);
       }
     }
@@ -179,26 +188,36 @@ export class TaskProcessor {
         continue;
       }
 
-      // LLM topic judgment — check this single user message against full task context
-      const context = this.buildContextSummary(currentTaskChunks);
+      // Structured topic classification
+      const taskState = this.buildTopicJudgeState(currentTaskChunks, userChunk);
       const newMsg = userChunk.content.slice(0, 500);
-      this.ctx.log.info(`Topic judge: "${newMsg.slice(0, 60)}" vs ${existingUserCount} user turns`);
-      const isNew = await this.summarizer.judgeNewTopic(context, newMsg);
-      this.ctx.log.info(`Topic judge result: ${isNew === null ? "null(fallback)" : isNew ? "NEW" : "SAME"}`);
+      this.ctx.log.info(`Topic classify: "${newMsg.slice(0, 60)}" vs ${existingUserCount} user turns`);
+      const result = await this.summarizer.classifyTopic(taskState, newMsg);
+      this.ctx.log.info(`Topic classify: decision=${result?.decision ?? "null"} confidence=${result?.confidence ?? "?"} type=${result?.boundaryType ?? "?"} reason=${result?.reason ?? ""}`);
 
-      if (isNew === null) {
+      if (!result || result.decision === "SAME") {
         this.assignChunksToTask(turn, currentTask.id);
         currentTaskChunks = currentTaskChunks.concat(turn);
         continue;
       }
 
-      if (isNew) {
-        this.ctx.log.info(`Task boundary at turn ${i}: LLM judged new topic. Msg: "${newMsg.slice(0, 80)}..."`);
-        await this.finalizeTask(currentTask);
-        currentTask = await this.createNewTaskReturn(sessionKey, userChunk.createdAt, owner);
-        currentTaskChunks = [];
+      // Low-confidence NEW: second-pass arbitration
+      if (result.confidence < 0.65) {
+        this.ctx.log.info(`Low confidence NEW (${result.confidence}), running second-pass arbitration...`);
+        const secondResult = await this.summarizer.arbitrateTopicSplit(taskState, newMsg);
+        this.ctx.log.info(`Second-pass result: ${secondResult ?? "null(fallback->SAME)"}`);
+        if (!secondResult || secondResult !== "NEW") {
+          this.assignChunksToTask(turn, currentTask.id);
+          currentTaskChunks = currentTaskChunks.concat(turn);
+          continue;
+        }
       }
 
+      this.ctx.log.info(`Task boundary at turn ${i}: classifier judged NEW (confidence=${result.confidence}). Msg: "${newMsg.slice(0, 80)}..."`);
+      await this.finalizeTask(currentTask);
+      currentTask = await this.createNewTaskReturn(sessionKey, userChunk.createdAt, owner);
+      currentTaskChunks = [];
+
       this.assignChunksToTask(turn, currentTask.id);
       currentTaskChunks = currentTaskChunks.concat(turn);
     }
@@ -226,38 +245,39 @@ export class TaskProcessor {
   }
 
   /**
-   * Build context from existing task chunks for the LLM topic judge.
-   * Includes both the task's opening topic and recent exchanges,
-   * so the LLM understands both what the task was originally about
-   * and where the conversation currently is.
-   *
-   * For user messages, include full content (up to 500 chars) since
-   * they carry the topic signal. For assistant messages, use summary
-   * or truncated content since they mostly elaborate.
+   * Build compact task state for the LLM topic classifier.
+   * Includes: topic (first user msg), last 3 turn summaries,
+   * and optional assistant snippet for short/ambiguous messages.
    */
-  private buildContextSummary(chunks: Chunk[]): string {
-    const conversational = chunks.filter((c) => c.role === "user" || c.role === "assistant");
-    if (conversational.length === 0) return "";
-
-    const formatChunk = (c: Chunk) => {
-      const label = c.role === "user" ? "User" : "Assistant";
-      const maxLen = c.role === "user" ? 500 : 200;
-      const text = c.summary || c.content.slice(0, maxLen);
-      return `[${label}]: ${text}`;
-    };
+  private buildTopicJudgeState(chunks: Chunk[], newUserChunk: Chunk): string {
+    const conv = chunks.filter((c) => c.role === "user" || c.role === "assistant");
+    if (conv.length === 0) return "";
+
+    const firstUser = conv.find((c) => c.role === "user");
+    const topic = firstUser?.summary || firstUser?.content.slice(0, 80) || "";
+
+    const turns: Array<{ u: string; a: string }> = [];
+    for (let j = 0; j < conv.length; j++) {
+      if (conv[j].role === "user") {
+        const u = conv[j].summary || conv[j].content.slice(0, 60);
+        const nextA = conv[j + 1]?.role === "assistant" ? conv[j + 1] : null;
+        const a = nextA ? (nextA.summary || nextA.content.slice(0, 60)) : "";
+        turns.push({ u, a });
+      }
+    }
+
+    const recent = turns.slice(-3);
+    const turnLines = recent.map((t, i) => `${i + 1}. U:${t.u} A:${t.a}`);
 
-    if (conversational.length <= 10) {
-      return conversational.map(formatChunk).join("\n");
+    let snippet = "";
+    if (newUserChunk.content.length < 30 || /^[那这它其还哪啥]/.test(newUserChunk.content.trim())) {
+      const lastA = [...conv].reverse().find((c) => c.role === "assistant");
+      if (lastA) snippet = lastA.content.slice(0, 200);
     }
 
-    const opening = conversational.slice(0, 6).map(formatChunk);
-    const recent = conversational.slice(-4).map(formatChunk);
-    return [
-      "--- Task opening ---",
-      ...opening,
-      "--- Recent exchanges ---",
-      ...recent,
-    ].join("\n");
+    const parts = [`topic:${topic}`, ...turnLines];
+    if (snippet) parts.push(`lastA:${snippet}`);
+    return parts.join("\n");
   }
 
   private async createNewTaskReturn(sessionKey: string, timestamp: number, owner: string = "agent:main"): Promise<Task> {

package/src/ingest/worker.ts

@@ -25,8 +25,14 @@ export class IngestWorker {
 
   getTaskProcessor(): TaskProcessor { return this.taskProcessor; }
 
+  private static isEphemeralSession(sessionKey: string): boolean {
+    return sessionKey.startsWith("temp:") || sessionKey.startsWith("internal:") || sessionKey.startsWith("system:");
+  }
+
   enqueue(messages: ConversationMessage[]): void {
-    this.queue.push(...messages);
+    const filtered = messages.filter((m) => !IngestWorker.isEphemeralSession(m.sessionKey));
+    if (filtered.length === 0) return;
+    this.queue.push(...filtered);
     if (!this.processing) {
       this.processQueue().catch((err) => {
         this.ctx.log.error(`Ingest worker error: ${err}`);
@@ -150,14 +156,23 @@ export class IngestWorker {
     let mergeHistory = "[]";
 
     // Fast path: exact content_hash match within same owner (agent dimension)
+    // Strategy: retire the OLD chunk, keep the NEW one active (latest wins)
     const chunkOwner = msg.owner ?? "agent:main";
     const existingByHash = this.store.findActiveChunkByHash(content, chunkOwner);
     if (existingByHash) {
-      this.ctx.log.debug(`Exact-dup (owner=${chunkOwner}): hash match → existing=${existingByHash}`);
+      this.ctx.log.debug(`Exact-dup (owner=${chunkOwner}): hash match → retiring old=${existingByHash}, keeping new=${chunkId}`);
       this.store.recordMergeHit(existingByHash, "DUPLICATE", "exact content hash match");
-      dedupStatus = "duplicate";
-      dedupTarget = existingByHash;
+      const oldChunk = this.store.getChunk(existingByHash);
+      this.store.markDedupStatus(existingByHash, "duplicate", chunkId, "exact content hash match");
+      this.store.deleteEmbedding(existingByHash);
+      mergedFromOld = existingByHash;
       dedupReason = "exact content hash match";
+      if (oldChunk) {
+        const oldHistory = JSON.parse(oldChunk.mergeHistory || "[]");
+        oldHistory.push({ action: "duplicate_superseded", at: Date.now(), reason: "exact content hash match", sourceChunkId: existingByHash });
+        mergeHistory = JSON.stringify(oldHistory);
+        mergeCount = (oldChunk.mergeCount || 0) + 1;
+      }
     }
 
     // Smart dedup: find Top-5 similar chunks, then ask LLM to judge
@@ -173,8 +188,9 @@ export class IngestWorker {
             index: i + 1,
             summary: chunk?.summary ?? "",
             chunkId: s.chunkId,
+            role: chunk?.role,
           };
-        }).filter(c => c.summary);
+        }).filter(c => c.summary && c.role === msg.role);
 
         if (candidates.length > 0) {
           const dedupResult = await this.summarizer.judgeDedup(summary, candidates);
@@ -183,10 +199,18 @@ export class IngestWorker {
             const targetChunkId = candidates[dedupResult.targetIndex - 1]?.chunkId;
             if (targetChunkId) {
               this.store.recordMergeHit(targetChunkId, "DUPLICATE", dedupResult.reason);
-              dedupStatus = "duplicate";
-              dedupTarget = targetChunkId;
+              const oldChunk = this.store.getChunk(targetChunkId);
+              this.store.markDedupStatus(targetChunkId, "duplicate", chunkId, dedupResult.reason);
+              this.store.deleteEmbedding(targetChunkId);
+              mergedFromOld = targetChunkId;
               dedupReason = dedupResult.reason;
-              this.ctx.log.debug(`Smart dedup: DUPLICATE → target=${targetChunkId}, storing with status=duplicate, reason: ${dedupResult.reason}`);
+              if (oldChunk) {
+                const oldHistory = JSON.parse(oldChunk.mergeHistory || "[]");
+                oldHistory.push({ action: "duplicate_superseded", at: Date.now(), reason: dedupResult.reason, sourceChunkId: targetChunkId });
+                mergeHistory = JSON.stringify(oldHistory);
+                mergeCount = (oldChunk.mergeCount || 0) + 1;
+              }
+              this.ctx.log.debug(`Smart dedup: DUPLICATE → retiring old=${targetChunkId}, keeping new=${chunkId} active, reason: ${dedupResult.reason}`);
             }
           }
 
@@ -266,9 +290,6 @@ export class IngestWorker {
     }
     this.ctx.log.debug(`Stored chunk=${chunkId} kind=${kind} role=${msg.role} dedup=${dedupStatus} len=${content.length} hasVec=${!!embedding && dedupStatus === "active"}`);
 
-    if (dedupStatus === "duplicate") {
-      return { action: "duplicate", summary, targetChunkId: dedupTarget ?? undefined, reason: dedupReason ?? undefined };
-    }
     if (mergedFromOld) {
       return { action: "merged", chunkId, summary, targetChunkId: mergedFromOld, reason: dedupReason ?? undefined };
     }

package/src/recall/engine.ts

@@ -77,7 +77,7 @@ export class RecallEngine {
     }
     const shortTerms = [...new Set([...spaceSplit, ...cjkBigrams])];
     const patternHits = shortTerms.length > 0
-      ? this.store.patternSearch(shortTerms, { limit: candidatePool })
+      ? this.store.patternSearch(shortTerms, { limit: candidatePool, ownerFilter })
       : [];
     const patternRanked = patternHits.map((h, i) => ({
       id: h.chunkId,
@@ -234,6 +234,7 @@ export class RecallEngine {
         score: Math.round(candidate.score * 1000) / 1000,
         taskId: chunk.taskId,
         skillId: chunk.skillId,
+        owner: chunk.owner,
         origin: chunk.owner === "public" ? "local-shared" : "local",
         source: {
           ts: chunk.createdAt,

package/src/shared/llm-call.ts

@@ -2,6 +2,19 @@ import * as fs from "fs";
 import * as path from "path";
 import type { SummarizerConfig, SummaryProvider, Logger, PluginContext, OpenClawAPI } from "../types";
 
+/**
+ * Resolve a SecretInput (string | SecretRef) to a plain string.
+ * Supports env-sourced SecretRef from OpenClaw's credential system.
+ */
+function resolveApiKey(
+  input: string | { source: string; provider?: string; id: string } | undefined,
+): string | undefined {
+  if (!input) return undefined;
+  if (typeof input === "string") return input;
+  if (input.source === "env") return process.env[input.id];
+  return undefined;
+}
+
 /**
  * Detect provider type from provider key name or base URL.
  */
@@ -56,7 +69,7 @@ export function loadOpenClawFallbackConfig(log: Logger): SummarizerConfig | unde
     if (!providerCfg) return undefined;
 
     const baseUrl: string | undefined = providerCfg.baseUrl;
-    const apiKey: string | undefined = providerCfg.apiKey;
+    const apiKey = resolveApiKey(providerCfg.apiKey);
     if (!baseUrl || !apiKey) return undefined;
 
     const provider = detectProvider(providerKey, baseUrl);

package/src/sharing/types.ts

@@ -38,6 +38,7 @@ export interface HubSearchHit {
   hubRank: number;
   taskTitle: string | null;
   ownerName: string;
+  sourceAgent: string;
   groupName: string | null;
   visibility: SharedVisibility;
   source: {