@heixui/dsh-session-suite 0.3.3 → 0.4.0

package/lib/memory-host.js

@@ -209,9 +209,17 @@ export function installMemory({ ctx, root, config, log }) {
           const sessionId = session?.id
           if (typeof sessionId !== 'string') return
           const cwd = resolveCwd(ctx, session)
-          void extractAfterTurn({ ctx, engine, sessionId, cwd, log, config }).catch(error =>
-            log('implicit memory extraction failed', error),
-          )
+          void extractAfterTurn({ ctx, engine, sessionId, cwd, log, config })
+            // The trace is published in a finally so a thrown error still leaves a
+            // readable cause on the settings page instead of an empty queue with
+            // no explanation.
+            .finally(() => publishExtractionTrace(engine))
+            .catch(error => {
+              extractionTrace.stage = 'threw'
+              extractionTrace.reason = errorMessage(error)
+              publishExtractionTrace(engine)
+              log('implicit memory extraction failed', error)
+            })
         }),
       'session-suite: implicit memory',
     ),
@@ -372,6 +380,10 @@ export function installMemory({ ctx, root, config, log }) {
       hasText: Boolean(lastRecall) || recallByScope.size > 0,
       scopes: recallByScope.size,
     }),
+    // Why the last implicit extraction produced (or did not produce) candidates.
+    // "The queue is empty" needs a cause, or the user cannot tell a working
+    // feature from a broken one.
+    extractionStatus: () => ({ ...extractionTrace }),
   }
 }
 
@@ -393,38 +405,50 @@ async function buildRecallText({ engine, cwd, question, config }) {
   // actually justifies, and records the reason for every candidate it dropped so
   // the settings page can explain a recall instead of leaving it a mystery.
   const { items: matched } = await engine.select({ cwd, query: question ?? '', limit })
-  const usable = matched.filter(item => !item.archived)
-
-  // Then a small resident layer: the most important memories ride along whatever
-  // the question happened to be. Without it, a follow-up like "继续" shares no
-  // characters with any memory, so a strict relevance filter would inject nothing
-  // at all — and a feature that vanishes on ordinary follow-ups reads as broken,
-  // not as a relevance decision. This is the same shape as the resident layer in
-  // dsh-fish-memory's design: durable facts get their own small quota instead of
-  // competing with question-matched events.
-  if (residentLimit > 0 && usable.length < limit) {
-    const ranked = await engine.recall({ cwd, query: '', limit, touch: false })
-    let residents = 0
-    for (const item of ranked) {
-      if (residents >= residentLimit || usable.length >= limit) break
-      if (item.archived) continue
-      if (usable.some(existing => existing.id === item.id)) continue
-      usable.push(item)
-      residents += 1
-    }
+  // A secret never reaches a prompt. It is not context — it is a credential — and
+  // injecting it would copy it into every provider request and every transcript.
+  const usable = matched.filter(item => !item.archived && item.privacy !== 'secret')
+
+  // The resident layer is an OVERVIEW, not more search results.
+  //
+  // It used to append the N highest-scoring entries as if they were relevant,
+  // which produced two problems: the injected block mixed "matched your question"
+  // with "just important in general" under one heading, and a user reading the
+  // settings probe reasonably concluded that a memory which DID match had been
+  // missed — it was sitting three lines below, unrelated to the query.
+  //
+  // Topic cards fix both: the block now says what the library knows in aggregate,
+  // and the two halves are labelled so nobody has to guess which is which.
+  let cards = []
+  if (residentLimit > 0) {
+    cards = await engine.residentCards({ cwd, limit: residentLimit }).catch(() => [])
   }
 
-  if (usable.length === 0) return ''
-  const lines = usable.map(item => `- ${item.scope === 'global' ? '[全局]' : '[本工作区]'} ${item.text}`)
-  return [
-    '## 长期记忆（跨会话）',
-    '',
-    '以下条目来自长期记忆库，与本轮问题相关。它们是稳定事实或约定，不是本轮对话内容；冲突时以用户当前的话为准。',
-    '',
-    ...lines,
+  if (usable.length === 0 && cards.length === 0) return ''
+
+  const sections = ['## 长期记忆（跨会话）', '']
+  if (usable.length > 0) {
+    sections.push(
+      '### 与本次提问相关',
+      '',
+      ...usable.map(item => `- ${item.scope === 'global' ? '[全局]' : '[本工作区]'} ${item.text}`),
+      '',
+    )
+  }
+  if (cards.length > 0) {
+    sections.push(
+      '### 库内概览（与本次提问无关，供你了解已积累的内容）',
+      '',
+      ...MemoryEngine.formatCards(cards),
+      '',
+    )
+  }
+  sections.push(
+    '以上是稳定事实或约定，不是本轮对话内容；冲突时以用户当前的话为准。',
     '',
     '如果用户说【记住：…】或【新增全局记忆：…】，立即照做；如果这些记忆已经过时，用 memory 工具更正。',
-  ].join('\n')
+  )
+  return sections.join('\n')
 }
 
 /** @param {readonly any[]} messages step messages @returns {string} last user text */
@@ -449,27 +473,103 @@ function lastUserText(messages) {
  * @param {(message: string, error?: unknown) => void} input.log logger
  * @param {any} input.config plugin config
  */
+/**
+ * Why the last implicit-extraction attempt produced what it did.
+ *
+ * Kept as module state rather than only a log line because the plugin has no log
+ * surface the user can reach: the settings page and the status endpoint read this
+ * so "the queue is empty" comes with a cause.
+ */
+const extractionTrace = {
+  sessionId: null,
+  at: 0,
+  ok: false,
+  stage: 'idle',
+  reason: '',
+  candidates: [],
+  produced: 0,
+  queued: 0,
+  rejected: 0,
+}
+
+/** @param {string} sessionId session being processed */
+function resetExtractionTrace(sessionId) {
+  extractionTrace.sessionId = sessionId
+  extractionTrace.at = Date.now()
+  extractionTrace.ok = false
+  extractionTrace.stage = 'start'
+  extractionTrace.reason = ''
+  extractionTrace.candidates = []
+  extractionTrace.produced = 0
+  extractionTrace.queued = 0
+  extractionTrace.rejected = 0
+}
+
+/**
+ * Record why extraction stopped before it could queue anything.
+ *
+ * @param {string} reason human-readable cause
+ * @returns {undefined} always undefined, so callers can `return failExtraction(...)`
+ */
+function failExtraction(reason) {
+  extractionTrace.ok = true
+  extractionTrace.stage = 'stopped'
+  extractionTrace.reason = reason
+  return undefined
+}
+
+/**
+ * Publish the trace to the engine so the settings page can read it.
+ *
+ * @param {any} engine memory engine
+ */
+function publishExtractionTrace(engine) {
+  try {
+    engine.lastExtraction = { ...extractionTrace }
+  } catch {
+    /* a diagnostic must never break extraction itself */
+  }
+}
+
 async function extractAfterTurn({ ctx, engine, sessionId, cwd, log, config }) {
+  // Every early return records why. An empty pending queue previously carried no
+  // explanation at all: the user could see "nothing waiting" but not whether the
+  // model found nothing, no model answered, or a filter dropped the result. That
+  // is the difference between a feature that works and one that looks broken.
+  resetExtractionTrace(sessionId)
   const settings = await engine.settings()
-  if (settings.implicit === false) return
+  if (settings.implicit === false) return failExtraction('implicit extraction is switched off in settings')
   const llm = ctx.get('llm')
-  if (!llm?.stream) return
+  if (!llm?.stream) return failExtraction('the Host exposes no llm.stream')
 
   const sessionController = ctx.get('sessionController')
   const inspection = await sessionController?.inspect?.(sessionId).catch(() => undefined)
   const events = inspection?.events ?? []
   const turn = lastCompletedTurn(events)
-  if (typeof turn !== 'number') return
+  if (typeof turn !== 'number') return failExtraction('this session has no completed turn yet')
   const transcript = turnTranscript(events, turn)
-  if (!transcript) return
+  if (!transcript) return failExtraction('the last turn produced no readable transcript')
 
   // The memory model is a fallback chain: a free provider that is rate limited
   // (429) or unreachable must not silently disable extraction for the session,
   // so every candidate is tried in order until one answers.
   const candidates = memoryCandidates(config)
-  if (candidates.length === 0) return
-
-  const prompt = `下面是一轮对话。找出其中**长期有效**的信息：稳定偏好、固定技术约定、长期策略、用户固有习惯。\n单次任务、一次性查询、临时调试、闲聊都不要。\n只输出 JSON：{"memories":[{"text":"...","scope":"workspace|global","affect":0.0-1.0,"tags":["..."]}]}\n没有就输出 {"memories":[]}。\n\n对话：\n${transcript}`
+  if (candidates.length === 0) return failExtraction('no memoryModels chain is configured')
+  extractionTrace.candidates = candidates.map(candidate => `${candidate.provider}/${candidate.model}`)
+
+  // The prompt asks for two different shapes, because the library stores two
+  // different things: durable truths, and dated events with an outcome. Asking only
+  // for the former is why episodic knowledge — "that approach failed last time" —
+  // was never captured, even though it is the kind that prevents repeating a
+  // mistake. The `kind` field lets the reply say which it means.
+  const prompt =
+    '下面是一轮对话。找出其中**值得长期记住**的信息，分两类：\n' +
+    '1. durable：稳定偏好、固定技术约定、长期策略、用户固有习惯，跨会话仍然成立的事实。\n' +
+    '2. episode：这一轮里实际**发生了什么**——尝试了什么、结果如何、踩了什么坑、为什么失败。带具体情境，不是泛泛结论。\n' +
+    '单次闲聊、情绪流水账、没有依据的猜测都不要。\n' +
+    '只输出 JSON：{"memories":[{"kind":"durable|episode","text":"...","scope":"workspace|global","affect":0.0-1.0,"tags":["..."],"context":"仅episode:当时在做什么","outcome":"仅episode:结果或教训"}]}\n' +
+    '没有就输出 {"memories":[]}。\n\n' +
+    `对话：\n${transcript}`
 
   let answer = ''
   let used
@@ -495,24 +595,80 @@ async function extractAfterTurn({ ctx, engine, sessionId, cwd, log, config }) {
       log(`memory extraction failed on ${candidate.provider}/${candidate.model}: ${errorMessage(error)}`)
     }
   }
-  if (!answer.trim()) return
+  if (!answer.trim()) {
+    return failExtraction(
+      `no memory model answered (tried ${candidates.length}: ${candidates
+        .map(candidate => `${candidate.provider}/${candidate.model}`)
+        .join(', ')})`,
+    )
+  }
   log(`memory extraction answered by ${used?.provider}/${used?.model}`)
+  extractionTrace.used = `${used?.provider}/${used?.model}`
 
   const parsed = parseJsonBlock(answer)
-  if (!parsed?.memories?.length) return
-  for (const item of parsed.memories.slice(0, 4)) {
+  // Record what happened even when nothing survives, so "the queue is empty" has
+  // a cause attached instead of being an unexplained absence.
+  if (!parsed) {
+    extractionTrace.parsed = false
+    extractionTrace.reason = 'the model answered but the reply was not usable JSON'
+    log(`memory extraction: no usable JSON in the reply`)
+    return
+  }
+  const produced = Array.isArray(parsed.memories) ? parsed.memories : []
+  extractionTrace.parsed = true
+  extractionTrace.produced = produced.length
+  if (produced.length === 0) {
+    extractionTrace.reason = 'the model judged this turn to hold nothing durable'
+    return
+  }
+
+  let queued = 0
+  let rejected = 0
+  for (const item of produced.slice(0, 4)) {
     const text = String(item.text ?? '').trim()
-    if (!text || !DURABLE_HINTS.test(text)) continue
-    if (TRANSIENT_HINTS.test(text)) continue
+    // The extraction prompt already asked the model for long-lived information
+    // only, so a second keyword gate here was acting as a filter on the model's
+    // own paraphrase rather than on the user's words. Measured cost: 7 of 10
+    // realistic candidates were dropped ("本项目使用 pnpm 管理依赖" carries none
+    // of 以后/一律/总是), which made the pending queue look broken. Keep only the
+    // checks that describe a malformed candidate, not a stylistic one.
+    if (!text) {
+      rejected += 1
+      continue
+    }
+    if (!isMeaningfulText(text)) {
+      rejected += 1
+      continue
+    }
+    if (TRANSIENT_HINTS.test(text)) {
+      rejected += 1
+      continue
+    }
     await engine.suggest({
       text,
       scope: item.scope === 'global' ? 'global' : 'workspace',
       cwd,
+      // The prompt asks the model to label an event as an episode. Anything else —
+      // including a reply from a model that ignored the new field — stays a fact,
+      // so an older prompt shape cannot corrupt the library.
+      kind: item.kind === 'episode' ? 'episode' : 'fact',
+      ...(item.kind === 'episode'
+        ? {
+            context: typeof item.context === 'string' ? item.context : undefined,
+            outcome: typeof item.outcome === 'string' ? item.outcome : undefined,
+          }
+        : {}),
       tags: Array.isArray(item.tags) ? item.tags : inferTags(text),
       affect: typeof item.affect === 'number' ? item.affect : 0.6,
       sessionId,
     })
+    queued += 1
   }
+  extractionTrace.queued = queued
+  extractionTrace.rejected = rejected
+  extractionTrace.reason =
+    queued > 0 ? `queued ${queued} suggestion(s) for confirmation` : 'every candidate was rejected as unusable'
+  log(`memory extraction: ${queued} queued, ${rejected} rejected`)
 }
 
 /** @param {string} text model output @returns {any | undefined} parsed JSON */

package/lib/retrieval.js

@@ -215,3 +215,78 @@ export function reciprocalRankFusion(rankings, k = 60) {
 export function rankIds(ids, scores) {
   return [...ids].sort((a, b) => (scores.get(b) ?? 0) - (scores.get(a) ?? 0) || String(a).localeCompare(String(b)))
 }
+
+/**
+ * Cosine similarity between two equal-length vectors.
+ *
+ * Kept here rather than in the embedding provider so the arithmetic is testable
+ * without a model: everything that decides ranking order is a pure function.
+ *
+ * @param {ArrayLike<number>} a first vector
+ * @param {ArrayLike<number>} b second vector
+ * @returns {number} similarity, 0 when either vector is degenerate
+ */
+export function cosineSimilarity(a, b) {
+  const length = Math.min(a?.length ?? 0, b?.length ?? 0)
+  if (length === 0) return 0
+  let dot = 0
+  let normA = 0
+  let normB = 0
+  for (let index = 0; index < length; index += 1) {
+    const x = Number(a[index]) || 0
+    const y = Number(b[index]) || 0
+    dot += x * y
+    normA += x * x
+    normB += y * y
+  }
+  if (normA === 0 || normB === 0) return 0
+  return dot / Math.sqrt(normA * normB)
+}
+
+/**
+ * Re-rank a candidate list by semantic similarity, as a second stage.
+ *
+ * This is the "retrieve then rerank" shape: the lexical routes decide *which*
+ * memories are candidates (cheap, offline, no model), and a dense model decides
+ * the *order* among them (expensive, so it only sees a short list). Doing it the
+ * other way round — embedding the whole library per query — would cost far more
+ * and would still need the lexical routes to catch exact tokens.
+ *
+ * The function is deliberately pure and provider-agnostic: it takes the vectors
+ * it should compare. If a caller passes no vectors, the input order is returned
+ * untouched, so a missing or failed model degrades to "no reranking" rather than
+ * to an exception. That matters because reranking is an optimisation, not a
+ * correctness requirement — a broken model must not break recall.
+ *
+ * Interpolation rather than replacement: the fused score already encodes lexical
+ * agreement, and letting the dense score overwrite it would discard signal. The
+ * weight defaults to 0.5 because with no held-out set to tune against, a claim of
+ * a better constant would be unfounded.
+ *
+ * @param {Array<{ id: string | number }>} candidates ordered by the lexical stage
+ * @param {ArrayLike<number> | undefined} queryVector query embedding
+ * @param {Map<string | number, ArrayLike<number>>} vectorsById candidate embeddings
+ * @param {{ lexical?: Map<string | number, number>, weight?: number }} [options]
+ *   lexical scores to interpolate with, and how much the dense score counts
+ * @returns {Array<{ id: string | number }>} the same candidates, reordered
+ */
+export function rerankBySimilarity(candidates, queryVector, vectorsById, options = {}) {
+  const list = Array.isArray(candidates) ? candidates : []
+  if (!queryVector || !(vectorsById instanceof Map) || vectorsById.size === 0) return list
+  const weight = typeof options.weight === 'number' ? Math.min(1, Math.max(0, options.weight)) : 0.5
+  const lexical = options.lexical instanceof Map ? options.lexical : undefined
+
+  const scored = list.map(candidate => {
+    const vector = vectorsById.get(candidate.id)
+    const dense = vector ? cosineSimilarity(queryVector, vector) : 0
+    const prior = lexical ? (lexical.get(candidate.id) ?? 0) : 0
+    // A candidate with no vector keeps its lexical standing instead of being
+    // pushed to the bottom: an embedding failure for one row must not delete it
+    // from the results.
+    const combined = vector ? (1 - weight) * prior + weight * dense : prior
+    return { candidate, combined, dense, hadVector: Boolean(vector) }
+  })
+
+  scored.sort((a, b) => b.combined - a.combined || String(a.candidate.id).localeCompare(String(b.candidate.id)))
+  return scored.map(entry => entry.candidate)
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@heixui/dsh-session-suite",
-  "version": "0.3.3",
+  "version": "0.4.0",
   "description": "Session checkpoints for DSH: rewind a conversation with its files and memory, permanent session deletion, and shareable exports",
   "type": "module",
   "main": "./index.js",