@heixui/dsh-session-suite 0.3.4 → 0.4.0

package/README.md

@@ -345,6 +345,28 @@ anything, and it never deletes real content.
 as symptom → root cause → fix. Read it before changing the snapshot, compaction,
 memory, or export paths.
 
+`docs/memory-architecture.md` maps the memory engine against the standard
+two-layer agent-memory design, with measured numbers rather than claims — including
+the parts that are deliberately not built and why.
+
+## What changed in 0.4.0
+
+The memory engine was reorganised; the session features are unchanged.
+
+| Area | Before | Now |
+|---|---|---|
+| Duplicate memories | One topic occupied 7 of 46 entries | Maintenance is idempotent (verified across 6 passes); existing duplicates merged into cold storage |
+| Resident layer | 2 flat entries mixed into the "relevant" block | Up to 5 topic cards, in their own labelled section |
+| Episodic memory | Absent | `kind: 'episode'` with time, context, and outcome; decays twice as fast |
+| Dense reranking | Implemented but off, weight unverified | **Optional, off by default**, weight calibrated by measurement (MRR 0.535 → 0.760 on 8 paraphrased queries, recall unchanged) |
+| Privacy | Scope only | `public` / `internal` / `secret`; credentials auto-detected; `secret` never injected or exported |
+| Implicit queue | Keyword gate dropped 7 of 10 valid candidates | Gate removed; an empty queue now states why it is empty |
+
+**Embedding is optional and adds no download by default.** Lexical retrieval alone is
+the default behaviour; installing a model adds cross-lingual and paraphrase recall.
+See `docs/memory-architecture.md` for the measured effect and
+`tools/download-embedding.mjs` for the deliberate, explicit install path.
+
 ## Limits
 
 * Rewinding does not rewrite the durable log; it forks at the target message and archives the old session. That

package/README.zh.md

@@ -157,13 +157,34 @@ tools/              自测脚本与预清理脚本（不随插件打包）
 docs/               需求与 API 笔记
 ```
 
+## 0.4.0 改了什么
+
+本次重构的是记忆引擎，会话相关功能未变。
+
+| 方面 | 改前 | 改后 |
+|---|---|---|
+| 重复记忆 | 46 条里同一主题占 7 条 | 维护幂等（连续 6 轮验证）；存量重复已合并进冷库 |
+| 常驻层 | 2 条扁平记忆，混在「相关」区块里 | 最多 5 张主题卡片，独立分区标注 |
+| 情景记忆 | 没有 | `kind: 'episode'`，带发生时间、情境、结果；衰减加倍 |
+| dense 重排 | 实现了但默认关，权重未验证 | **可选，默认关**；权重经实测校准（8 条改写查询 MRR 0.535→0.760，recall 不变） |
+| 隐私分级 | 只有 scope | `public`/`internal`/`secret`；凭据自动识别；secret 永不注入、永不导出 |
+| 隐式队列 | 关键词闸门误杀 7/10 合法候选 | 闸门已移除；空队列会说明原因 |
+
+**embedding 是可选增强，默认不下载任何东西。** 不装模型就是纯词法检索，功能完整；装了才有跨语言与改写召回。
+实测效果与安装方式见 `docs/memory-architecture.md` 和 `tools/download-embedding.mjs`。
+
 ## 测试
 
-两个自测脚本都在临时目录里跑，不会碰真实的 DSH 数据。
+全部自测脚本都在临时目录里跑，不会碰真实的 DSH 数据。
 
 ```bash
-node tools/selftest.mjs          # 快照 / 忽略规则 / 恢复 / 淘汰 / 记忆
-node tools/selftest-rollback.mjs # 回滚 / 撤销 / 删除消息 / 分享 / 永久删除
+node tools/selftest.mjs                    # 快照 / 忽略规则 / 恢复 / 淘汰 / 记忆
+node tools/selftest-memory-dedupe.mjs      # 维护幂等，重复不再生成
+node tools/selftest-resident-cards.mjs     # 常驻卡片分组与预算
+node tools/selftest-episode-memory.mjs     # 情景记忆类型与衰减
+node tools/selftest-memory-privacy.mjs     # 隐私分级与导出过滤
+node tools/selftest-implicit-queue.mjs     # 隐式候选不再被误杀
+node tools/probe-dense-rerank.mjs          # dense 重排的实测对比（需要模型）
 ```
 
 ## 已知边界

package/client.js

@@ -178,6 +178,7 @@ window.__ModuleLoader__.load({
         'memory.workspacePicker': 'Workspace',
         'memory.workspaceNone': 'No workspace detected',
         'memory.rule': 'Rule',
+        'memory.episode': 'Episode',
         'memory.summary': 'Summary',
         'memory.frozen': 'Frozen',
         'memory.archived': 'Archived',
@@ -210,6 +211,11 @@ window.__ModuleLoader__.load({
         'memory.remove': 'Delete',
         'memory.pendingTitle': 'Suggested memories',
         'memory.pendingHint': 'Extracted from the last turns. Nothing is stored until you accept it.',
+        'memory.extractionIdle': 'Nothing is waiting. Last check: {reason}',
+        'memory.extractionUnknown': 'no attempt recorded since this Host started',
+        'memory.extractionTried': 'tried {n} memory model(s)',
+        'memory.extractionQueued': '{n} queued',
+        'memory.extractionRejected': '{n} rejected as unusable',
         'memory.accept': 'Keep',
         'memory.reject': 'Discard',
         'memory.accepted': 'Kept',
@@ -357,6 +363,7 @@ window.__ModuleLoader__.load({
         'memory.workspacePicker': '工作区',
         'memory.workspaceNone': '未检测到工作区',
         'memory.rule': '规则',
+        'memory.episode': '经历',
         'memory.summary': '摘要',
         'memory.frozen': '已冻结',
         'memory.archived': '已归档',
@@ -389,6 +396,11 @@ window.__ModuleLoader__.load({
         'memory.remove': '删除',
         'memory.pendingTitle': '待确认的记忆',
         'memory.pendingHint': '从最近几轮里识别出来的，确认前不会入库。',
+        'memory.extractionIdle': '当前没有待确认条目。上次检查结果：{reason}',
+        'memory.extractionUnknown': '本次运行还没有执行过提取',
+        'memory.extractionTried': '试过 {n} 个记忆模型',
+        'memory.extractionQueued': '入队 {n} 条',
+        'memory.extractionRejected': '剔除 {n} 条',
         'memory.accept': '保留',
         'memory.reject': '丢弃',
         'memory.accepted': '已保留',
@@ -2571,10 +2583,14 @@ window.__ModuleLoader__.load({
         h('span', { className: 'dss-row-meta' }, t('memory.filterCount', { n: (items ?? []).length })),
       )
 
+      // An empty queue used to render nothing at all, which is indistinguishable
+      // from a broken feature: the user cannot tell "the model judged this turn
+      // unremarkable" from "every memory model failed" or "a filter ate the
+      // result". When there is nothing pending, say why.
+      const extraction = status?.extraction
       const pendingBlock =
-        pending.length === 0
-          ? null
-          : h(
+        pending.length > 0
+          ? h(
               'div',
               { className: 'dss-mem-pending' },
               h('div', { className: 'dss-group-title' }, t('memory.pendingTitle')),
@@ -2616,6 +2632,33 @@ window.__ModuleLoader__.load({
                 ),
               ),
             )
+          : extraction
+            ? h(
+                'div',
+                { className: 'dss-mem-pending' },
+                h('div', { className: 'dss-group-title' }, t('memory.pendingTitle')),
+                h(
+                  'div',
+                  { className: 'dss-mem-pending-hint' },
+                  t('memory.extractionIdle', {
+                    reason: extraction.reason || t('memory.extractionUnknown'),
+                  }),
+                ),
+                extraction.candidates?.length
+                  ? h(
+                      'div',
+                      { className: 'dss-row-meta' },
+                      h('span', null, t('memory.extractionTried', { n: extraction.candidates.length })),
+                      extraction.queued !== undefined
+                        ? h('span', null, t('memory.extractionQueued', { n: extraction.queued }))
+                        : null,
+                      extraction.rejected !== undefined
+                        ? h('span', null, t('memory.extractionRejected', { n: extraction.rejected }))
+                        : null,
+                    )
+                  : null,
+              )
+            : null
 
       const listBlock =
         visible.length === 0
@@ -2636,6 +2679,10 @@ window.__ModuleLoader__.load({
                       { className: 'dss-row-meta' },
                       h('span', { className: 'dss-badge' + (entry.scope === 'global' ? ' dss-locked' : '') }, entry.scope === 'global' ? t('memory.global') : t('memory.workspace')),
                       entry.kind === 'rule' ? h('span', { className: 'dss-badge dss-temp' }, t('memory.rule')) : null,
+                      // An episode is a dated event, not a truth. Labelling it
+                      // separately is what lets the user tell "this happened once"
+                      // from "this is how things are".
+                      entry.kind === 'episode' ? h('span', { className: 'dss-badge' }, t('memory.episode')) : null,
                       entry.summary ? h('span', { className: 'dss-badge' }, t('memory.summary')) : null,
                       entry.frozen ? h('span', { className: 'dss-badge dss-locked' }, t('memory.frozen')) : null,
                       entry.archived ? h('span', { className: 'dss-badge dss-temp' }, t('memory.archived')) : null,

package/cordis.patch.yml

@@ -69,6 +69,33 @@
         bm25K1: 1.5
         bm25B: 0.75
         rrfK: 60
+        # ---------- optional dense reranking (stage 2) ----------
+        # Lexical retrieval cannot bridge a paraphrase that shares no character
+        # (checkpoint vs 快照). A dense model can, so it reorders the candidates
+        # the lexical stage already found. It is OPTIONAL and OFF by default:
+        # nothing is downloaded, and recall works exactly as before when disabled.
+        #
+        # To enable, install a model deliberately and point modelDir at it:
+        #
+        #   embedding:
+        #     enabled: true
+        #     provider: local            # local | remote
+        #     modelDir: D:\AI agent cache\models\bge-base-zh-v1.5
+        #
+        # Or use an OpenAI-compatible endpoint instead of local files:
+        #
+        #   embedding:
+        #     enabled: true
+        #     provider: remote
+        #     baseUrl: https://api.siliconflow.cn/v1
+        #     apiKey: sk-...
+        #     model: BAAI/bge-m3
+        #
+        # Either way, a failure degrades to lexical ranking; it never breaks recall.
+        embedding:
+          enabled: true
+          provider: local
+          modelDir: D:\AI agent cache\dsh\models\bge-base-zh-v1.5
         # Similarity above which a new memory reinforces an existing one.
         duplicateSimilarity: 0.82
         # Similarity above which two memories may join one cluster.

package/lib/embeddings.js

@@ -0,0 +1,239 @@
+/**
+ * Optional dense embeddings for memory reranking.
+ *
+ * Design constraints, in the order they shaped this file:
+ *
+ * 1. **Never download anything implicitly.** A memory plugin that silently pulls
+ *    several hundred megabytes onto someone's disk is not acceptable; the model
+ *    is installed deliberately, and this module only loads what is already there.
+ * 2. **Never break recall.** Embedding is an optimisation. Any failure — no model,
+ *    unreadable file, out-of-memory, a provider that changed shape — must degrade
+ *    to "no reranking", never to an exception on the recall path.
+ * 3. **The model is swappable.** Two providers are supported without changing the
+ *    engine: a local transformers.js pipeline, and any OpenAI-compatible
+ *    /embeddings endpoint (Zhipu, SiliconFlow, and friends). The engine only ever
+ *    sees `{ vector }` or `undefined`.
+ *
+ * The default is the local pipeline, because recall runs on the prompt path and
+ * must not depend on a network round trip: a busy or rate-limited provider would
+ * turn every step into a slow one.
+ */
+
+import fs from 'node:fs'
+import path from 'node:path'
+
+/**
+ * A provider that never produces vectors.
+ *
+ * This is the default. Reranking then costs nothing and changes nothing, which
+ * is the correct behaviour for a library with no model installed.
+ */
+export const DISABLED_PROVIDER = {
+  name: 'disabled',
+  available: false,
+  async embed() {
+    return undefined
+  },
+}
+
+/**
+ * A local transformers.js pipeline.
+ *
+ * The model directory is supplied by the caller; nothing is downloaded here. If
+ * the directory or the runtime is missing, `available` stays false and the
+ * provider behaves like the disabled one.
+ *
+ * @param {object} input provider input
+ * @param {string} input.modelDir absolute directory holding the model files
+ * @param {string} [input.pipeline] transformers.js pipeline name
+ * @param {number} [input.batchSize] texts per inference call
+ * @returns {Promise<object>} provider
+ */
+export async function createLocalProvider({ modelDir, pipeline = 'feature-extraction', batchSize = 16, runtime }) {
+  if (typeof modelDir !== 'string' || !modelDir) return DISABLED_PROVIDER
+  // Point at the model directory itself; transformers.js accepts a path and will
+  // not reach the network when the files are present.
+  if (!fs.existsSync(modelDir)) return DISABLED_PROVIDER
+
+  let transformers = runtime
+  if (!transformers) {
+    try {
+      // Resolved at runtime so a profile without the package still loads the bundle.
+      transformers = await import('@xenova/transformers')
+    } catch {
+      try {
+        transformers = await import('@huggingface/transformers')
+      } catch {
+        return DISABLED_PROVIDER
+      }
+    }
+  }
+
+  // Turn OFF remote fetching before creating the pipeline.
+  //
+  // Passing a directory to pipeline() is NOT enough: when any expected file is
+  // missing, transformers.js silently falls back to downloading the model from
+  // Hugging Face. On a network where that host is unreachable the failure surfaces
+  // as `fetch failed` / CONNECT_TIMEOUT, which looks like a network problem rather
+  // than a missing-file problem — and worse, on a reachable network the plugin would
+  // pull hundreds of megabytes onto a user's disk without asking. allowRemoteModels
+  // = false turns a missing file into a clear, immediate error instead.
+  //
+  // transformers.js resolves a model by joining localModelPath with the model id, so
+  // localModelPath must be the PARENT directory and the id the folder name. An
+  // absolute path passed as the id defeats that join and triggers the remote lookup.
+  let modelId = modelDir
+  try {
+    if (transformers.env) {
+      transformers.env.allowRemoteModels = false
+      transformers.env.allowLocalModels = true
+      const parent = path.dirname(modelDir)
+      transformers.env.localModelPath = parent
+      modelId = path.basename(modelDir)
+    }
+  } catch {
+    /* an older runtime without env is still usable with an explicit path */
+  }
+
+  const extractor = await transformers.pipeline(pipeline, modelId)
+
+  return {
+    name: 'local',
+    available: true,
+    modelDir,
+    /**
+     * @param {string} text input
+     * @returns {Promise<number[] | undefined>} embedding
+     */
+    async embed(text) {
+      const value = String(text ?? '').trim()
+      if (!value) return undefined
+      try {
+        const output = await extractor(value, { pooling: 'mean', normalize: true })
+        // transformers.js returns a Tensor; `data` is the flat Float32Array.
+        const data = output?.data ?? output?.[0]?.data ?? output
+        if (!data) return undefined
+        return Array.from(data, Number)
+      } catch {
+        return undefined
+      }
+    },
+    /**
+     * Embed many texts in one call. Falls back to per-text calls when the
+     * runtime does not support batches, which is slower but still correct.
+     *
+     * @param {string[]} texts inputs
+     * @returns {Promise<Array<number[] | undefined>>} embeddings
+     */
+    async embedMany(texts) {
+      const list = Array.isArray(texts) ? texts : []
+      if (list.length === 0) return []
+      try {
+        const output = await extractor(list, { pooling: 'mean', normalize: true })
+        const rows = output?.tolist?.() ?? output?.data ?? undefined
+        if (Array.isArray(rows) && rows.length === list.length && Array.isArray(rows[0])) {
+          return rows.map(row => Array.from(row, Number))
+        }
+        if (rows && rows.length === list.length * (rows[0]?.length ?? 0)) {
+          // Flat buffer: split into equal rows.
+          const width = rows.length / list.length
+          const out = []
+          for (let index = 0; index < list.length; index += 1) {
+            out.push(Array.from(rows.slice(index * width, (index + 1) * width), Number))
+          }
+          return out
+        }
+      } catch {
+        /* fall through to per-item calls */
+      }
+      const out = []
+      for (const text of list.slice(0, batchSize * 8)) out.push(await this.embed(text))
+      return out
+    },
+  }
+}
+
+/**
+ * An OpenAI-compatible /embeddings endpoint.
+ *
+ * Used when the user prefers no local files. Every failure returns undefined, so
+ * a flaky endpoint degrades to lexical ranking rather than failing recall.
+ *
+ * @param {object} input provider input
+ * @param {string} input.baseUrl endpoint base, e.g. https://api.siliconflow.cn/v1
+ * @param {string} input.apiKey bearer token
+ * @param {string} input.model model id
+ * @param {number} [input.timeoutMs] request timeout
+ * @returns {object} provider
+ */
+export function createRemoteProvider({ baseUrl, apiKey, model, timeoutMs = 5000 }) {
+  if (typeof baseUrl !== 'string' || !baseUrl || !apiKey || !model) return DISABLED_PROVIDER
+  const endpoint = `${baseUrl.replace(/\/+$/, '')}/embeddings`
+
+  return {
+    name: 'remote',
+    available: true,
+    model,
+    /**
+     * @param {string} text input
+     * @returns {Promise<number[] | undefined>} embedding
+     */
+    async embed(text) {
+      const value = String(text ?? '').trim()
+      if (!value) return undefined
+      const controller = new AbortController()
+      const timer = setTimeout(() => controller.abort(), timeoutMs)
+      try {
+        const response = await fetch(endpoint, {
+          method: 'POST',
+          headers: { 'content-type': 'application/json', authorization: `Bearer ${apiKey}` },
+          body: JSON.stringify({ model, input: value }),
+          signal: controller.signal,
+        })
+        if (!response.ok) return undefined
+        const payload = await response.json()
+        const vector = payload?.data?.[0]?.embedding
+        return Array.isArray(vector) ? vector.map(Number) : undefined
+      } catch {
+        return undefined
+      } finally {
+        clearTimeout(timer)
+      }
+    },
+  }
+}
+
+/**
+ * Build the provider described by plugin config, or the disabled one.
+ *
+ * Configuration is read defensively: a half-written block yields the disabled
+ * provider rather than a crash, because this runs during plugin installation.
+ *
+ * @param {any} config plugin config
+ * @returns {Promise<object>} provider
+ */
+export async function createEmbeddingProvider(config) {
+  const block = config?.embedding
+  if (!block || typeof block !== 'object' || block.enabled === false) return DISABLED_PROVIDER
+
+  if (block.provider === 'remote') {
+    return createRemoteProvider({
+      baseUrl: block.baseUrl,
+      apiKey: block.apiKey,
+      model: block.model,
+      timeoutMs: Number(block.timeoutMs) || 5000,
+    })
+  }
+
+  const modelDir = typeof block.modelDir === 'string' && block.modelDir
+    ? block.modelDir
+    : path.join(config?.root ?? '', 'models', String(block.model ?? 'bge-base-zh-v1.5'))
+  try {
+    return await createLocalProvider({
+      modelDir,
+      pipeline: typeof block.pipeline === 'string' ? block.pipeline : 'feature-extraction',
+    })
+  } catch {
+    return DISABLED_PROVIDER
+  }
+}