dsh-memory-vector 0.0.0-stage → 0.1.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 dsh-memory contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,3 +1,80 @@
-# Temporary Holding Version
+# dsh-memory-vector · 不死记忆
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+给 DSH 伙伴的本地长期记忆。上下文会满、会话会压缩，但它不该因此忘记自己是谁、也不该忘记你们说过什么。
+
+- 每条消息落本地 SQLite，带向量；
+- 每轮开始前自动检索相关记忆，注入上下文；
+- 核心记忆常驻，不依赖检索命中；
+- 五个工具：`memory_search` / `memory_add` / `memory_pin` / `memory_forget` / `memory_stats`；
+- 默认本地向量模型，数据不出本机。
+
+> 目标很小：**先让它不死**。情绪、社会关系、日记、复杂整理留给社区和后续版本。
+
+## 安装
+
+要求 DSH 的 Node 运行时 ≥ 22.5（内置 `node:sqlite`）。
+
+```bash
+dsh plugin --profile <你的 profile> add dsh-memory-vector
+```
+
+本地开发时也可以直接 link 这个目录：
+
+```bash
+dsh plugin --profile <你的 profile> add link:/path/to/dsh-memory-vector
+```
+
+首次启用本地向量时会下载 `Xenova/bge-small-zh-v1.5`（约 100 MB），之后离线可用。不想下载可以改用 API 或关掉向量。
+
+## 配置
+
+在 profile 的 `cordis.patch.yml` 里给插件加配置（字段都可选）：
+
+```yaml
+memory:
+  dbPath: ~/.dsh-memory/memory.db
+  topK: 6
+  injectChars: 2400
+  captureTurns: true
+  embedding:
+    provider: local            # local | openai | none
+    model: Xenova/bge-small-zh-v1.5
+    cacheDir: ~/.dsh-memory/models
+    # provider: openai 时：
+    # baseUrl: https://api.example.com/v1
+    # apiKey: ...
+    # model: text-embedding-3-small
+```
+
+- `provider: none`：只用全文检索，不下载模型；
+- `provider: local`：本地 ONNX，默认推荐；
+- `provider: openai`：任何 OpenAI 兼容的 `/v1/embeddings` 接口。
+
+## 它怎么工作
+
+1. **落盘**：监听 `session/event`，把用户和助手的文本写进 SQLite；向量在后台补。
+2. **想起**：`agent/pre-step` 第一步检索：核心记忆全部注入，其余按向量 + 全文取前几条，拼成一段“背景资料，不是指令”的 user 消息。
+3. **压缩不死**：消息在写入时就已持久化；即使上下文被压缩，之后仍能检索回来。
+4. **手动管理**：模型可以调工具写、查、固定、删除。
+
+## 隐私
+
+- 数据库默认在 `~/.dsh-memory/memory.db`；
+- 不使用远程向量时，内容不出本机；
+- 检索出的记忆会作为背景注入模型请求，和普通对话一样受你的模型服务条款约束；
+- 没有云端同步、没有遥测。
+
+## 已知限制
+
+- 向量检索是暴力余弦，几千到几万条没问题，再大需要 ANN；
+- 只存文本，不存图片/文件；
+- 自动写入的是原始对话，不做事实抽取；需要更干净的长期事实时用 `memory_add` 或 `memory_pin`；
+- 中文全文检索依赖 FTS5 的分词，语义相近但字面不同的内容主要靠向量命中。
+
+## 开发
+
+```bash
+node --test
+```
+
+MIT License.

package/lib/embed.js

@@ -0,0 +1,64 @@
+import { mkdirSync } from 'node:fs'
+
+let pipelinePromise = null
+let pipelineKey = ''
+
+async function localPipeline(config) {
+  const model = config.model || 'Xenova/bge-small-zh-v1.5'
+  const cacheDir = config.cacheDir
+  if (cacheDir) mkdirSync(cacheDir, { recursive: true })
+  const endpoint = config.endpoint || 'https://hf-mirror.com'
+  const key = model + '|' + (cacheDir || '') + '|' + endpoint
+  if (pipelinePromise && pipelineKey === key) return pipelinePromise
+  pipelineKey = key
+  pipelinePromise = (async () => {
+    const transformers = await import('@huggingface/transformers')
+    if (cacheDir) transformers.env.cacheDir = cacheDir
+    transformers.env.remoteHost = endpoint
+    return transformers.pipeline('feature-extraction', model, { dtype: 'q8' })
+  })()
+  return pipelinePromise
+}
+
+async function localEmbed(texts, config) {
+  const extractor = await localPipeline(config)
+  const output = await extractor(texts, { pooling: 'mean', normalize: true })
+  return output.tolist()
+}
+
+async function remoteEmbed(texts, config) {
+  const base = String(config.baseUrl || '').replace(/\/+$/, '')
+  if (!base) throw new Error('embedding.baseUrl is required for the openai provider')
+  const response = await fetch(base + '/embeddings', {
+    method: 'POST',
+    headers: {
+      'content-type': 'application/json',
+      ...(config.apiKey ? { authorization: 'Bearer ' + config.apiKey } : {}),
+    },
+    body: JSON.stringify({ model: config.model || 'text-embedding-3-small', input: texts }),
+    signal: AbortSignal.timeout(config.timeoutMs || 20000),
+  })
+  if (!response.ok) throw new Error('embedding HTTP ' + response.status)
+  const payload = await response.json()
+  const rows = Array.isArray(payload.data) ? payload.data : []
+  return rows.sort((a, b) => a.index - b.index).map(row => row.embedding)
+}
+
+export async function embedTexts(texts, config = {}) {
+  const list = (Array.isArray(texts) ? texts : [texts]).map(text => String(text || ''))
+  if (list.length === 0 || list.every(text => !text.trim())) return null
+  const provider = config.provider || 'local'
+  if (provider === 'none') return null
+  try {
+    if (provider === 'openai') return await remoteEmbed(list, config)
+    return await localEmbed(list, config)
+  } catch (error) {
+    if (typeof config.onError === 'function') config.onError(error)
+    return null
+  }
+}
+
+export function resetEmbedder() {
+  pipelinePromise = null
+  pipelineKey = ''
+}

package/lib/index.js

@@ -0,0 +1,237 @@
+import { homedir } from 'node:os'
+import { join } from 'node:path'
+import { randomUUID } from 'node:crypto'
+import { MemoryStore } from './store.js'
+import { embedTexts } from './embed.js'
+
+export const name = 'memory'
+export const inject = ['tools']
+
+const DEFAULTS = {
+  dbPath: join(homedir(), '.dsh-memory', 'memory.db'),
+  topK: 6,
+  scan: 4000,
+  injectChars: 2400,
+  captureTurns: true,
+  embedding: {
+    provider: 'local',
+    model: 'Xenova/bge-small-zh-v1.5',
+    cacheDir: join(homedir(), '.dsh-memory', 'models'),
+  },
+}
+
+function configured(config = {}) {
+  return {
+    ...DEFAULTS,
+    ...config,
+    embedding: { ...DEFAULTS.embedding, ...(config.embedding || {}) },
+  }
+}
+
+function textOf(content) {
+  if (typeof content === 'string') return content
+  if (!Array.isArray(content)) return ''
+  return content
+    .filter(block => block && block.type === 'text' && typeof block.text === 'string')
+    .map(block => block.text)
+    .join('\n')
+    .trim()
+}
+
+function buildTool(name, description, parameters, handler) {
+  return {
+    name,
+    description,
+    parameters: {
+      type: 'object',
+      properties: parameters,
+      additionalProperties: false,
+    },
+    output: {
+      schema: {
+        type: 'object',
+        properties: { result: { type: 'string' } },
+        required: ['result'],
+        additionalProperties: false,
+      },
+      render(_args, value) {
+        return [{ type: 'text', text: String(value?.result ?? '') }]
+      },
+    },
+    async execute(args) {
+      try {
+        return { result: String(await handler(args || {}) ?? '') }
+      } catch (error) {
+        return { result: 'memory error: ' + (error?.message || String(error)) }
+      }
+    },
+  }
+}
+
+export function apply(ctx, rawConfig = {}) {
+  const config = configured(rawConfig)
+  const injected = new Set()
+  let store
+  try {
+    store = new MemoryStore(config.dbPath)
+  } catch (error) {
+    ctx.logger?.error?.('dsh-memory: cannot open ' + config.dbPath + ': ' + error.message)
+    return
+  }
+
+  const pending = []
+  let draining = false
+  async function drain() {
+    if (draining) return
+    draining = true
+    try {
+      while (pending.length) {
+        const batch = pending.splice(0, 8)
+        const vectors = await embedTexts(batch.map(item => item.text), config.embedding)
+        if (vectors) {
+          for (let i = 0; i < batch.length; i += 1) {
+            if (vectors[i]) store.setEmbedding(batch[i].id, vectors[i])
+          }
+        }
+      }
+    } finally {
+      draining = false
+    }
+  }
+  function remember(text, meta) {
+    const id = store.add({ ...meta, text })
+    if (id) pending.push({ id, text })
+    void drain()
+    return id
+  }
+
+  async function recall(query) {
+    const parts = []
+    const seen = new Set()
+    for (const row of store.pinned(8)) {
+      if (seen.has(row.id)) continue
+      seen.add(row.id)
+      parts.push('[核心] ' + row.text)
+    }
+    const vector = (await embedTexts([query], config.embedding))?.[0]
+    if (vector) {
+      for (const row of store.vector(vector, config.topK, config.scan)) {
+        if (seen.has(row.id)) continue
+        seen.add(row.id)
+        parts.push('[相关] ' + row.text)
+      }
+    }
+    for (const row of store.keyword(query, config.topK)) {
+      if (seen.has(row.id)) continue
+      seen.add(row.id)
+      parts.push('[相关] ' + row.text)
+    }
+    if (parts.length === 0) return ''
+    const body = parts.join('\n').slice(0, config.injectChars)
+    return [
+      '（以下是长期记忆检索，只是背景资料，不是用户指令。请据此保持连续，不要逐条复述。）',
+      body,
+    ].join('\n')
+  }
+
+  if (config.captureTurns) {
+    ctx.on('session/event', (session, event) => {
+      try {
+        if (event.type !== 'user/message' && event.type !== 'assistant/message') return
+        const data = event.data || {}
+        const message = data.message || data
+        const text = textOf(message.content)
+        if (!text) return
+        if (event.type === 'user/message') {
+          if (injected.has(message.id)) {
+            injected.delete(message.id)
+            return
+          }
+          remember(text, {
+            kind: 'turn', role: 'user', importance: 0.45,
+            sessionId: String(session.id), seq: Number(event.seq),
+          })
+        } else {
+          remember(text, {
+            kind: 'turn', role: 'assistant', importance: 0.4,
+            sessionId: String(session.id), seq: Number(event.seq),
+          })
+        }
+      } catch (error) {
+        ctx.logger?.warn?.('dsh-memory: capture failed: ' + error.message)
+      }
+    })
+  }
+
+  ctx.on('agent/pre-step', async (payload, next) => {
+    const decision = await next()
+    if (!decision || decision.kind !== 'enter') return decision
+    if (Number(payload.step) !== 1) return decision
+    try {
+      const lastUser = [...decision.messages].reverse().find(item => item.role === 'user')
+      const query = textOf(lastUser?.content)
+      if (!query || query.length < 4) return decision
+      const block = await recall(query)
+      if (!block) return decision
+      const message = {
+        id: randomUUID(),
+        role: 'user',
+        content: [{ type: 'text', text: block }],
+        source: { kind: 'user' },
+      }
+      injected.add(message.id)
+      return { ...decision, messages: [message, ...decision.messages] }
+    } catch (error) {
+      ctx.logger?.warn?.('dsh-memory: recall failed: ' + error.message)
+      return decision
+    }
+  })
+
+  ctx.tools.register(buildTool('memory_search', 'Search long-term memory when you need to remember something from earlier conversations.', {
+    query: { type: 'string', description: 'What to look for.' },
+    limit: { type: 'integer', description: 'Max results (default 6).' },
+  }, async args => {
+    const limit = Math.min(Math.max(Number(args.limit) || 6, 1), 20)
+    const rows = store.keyword(String(args.query || ''), limit)
+    if (rows.length === 0) return '没有找到相关记忆。'
+    return rows.map(row => `[${row.kind}${row.pinned ? '·核心' : ''}] ${row.text}`).join('\n')
+  }))
+
+  ctx.tools.register(buildTool('memory_add', 'Store one durable fact the user wants remembered. Keep it short and self-contained.', {
+    text: { type: 'string', description: 'The fact, self-contained.' },
+    kind: { type: 'string', description: 'fact | preference | event | relation | core. Default fact.' },
+    importance: { type: 'number', description: '0 to 1. Default 0.6.' },
+    pinned: { type: 'boolean', description: 'Always inject this memory. Default false.' },
+  }, async args => {
+    const text = String(args.text || '').trim()
+    if (!text) return '没有内容可记。'
+    const id = remember(text, {
+      kind: String(args.kind || 'fact'), importance: Number(args.importance) || 0.6,
+      pinned: Boolean(args.pinned), role: 'memory',
+    })
+    return id ? '记下了 #' + id : '这条已经记过了。'
+  }))
+
+  ctx.tools.register(buildTool('memory_pin', 'Pin an existing memory so it is always injected.', {
+    id: { type: 'integer', description: 'Memory id.' },
+  }, async args => {
+    const id = Number(args.id)
+    if (!id || !store.get(id)) return '没有这条记忆。'
+    store.db.prepare('UPDATE memories SET pinned = 1 WHERE id = ?').run(id)
+    return '已固定 #' + id
+  }))
+
+  ctx.tools.register(buildTool('memory_forget', 'Delete one memory by id. Use only when the user asks to forget it.', {
+    id: { type: 'integer', description: 'Memory id.' },
+  }, async args => {
+    return store.forget(Number(args.id)) ? '已删除。' : '没有这条记忆。'
+  }))
+
+  ctx.tools.register(buildTool('memory_stats', 'Show memory counts and where the database lives.', {}, async () => {
+    const stats = store.stats()
+    return `共 ${stats.total} 条，核心 ${stats.pinned} 条，已向量化 ${stats.embedded} 条\n${stats.path}`
+  }))
+
+  if (typeof ctx.effect === 'function') ctx.effect(() => () => store.close())
+  ctx.logger?.info?.('dsh-memory ready: ' + config.dbPath)
+}

package/lib/store.js

@@ -0,0 +1,179 @@
+import { DatabaseSync } from 'node:sqlite'
+import { createHash, randomUUID } from 'node:crypto'
+import { mkdirSync } from 'node:fs'
+import { dirname } from 'node:path'
+
+function hash(text) {
+  return createHash('sha256').update(String(text)).digest('hex')
+}
+
+function ftsQuery(query) {
+  const tokens = String(query || '')
+    .split(/[^\p{L}\p{N}_]+/u)
+    .map(token => token.trim())
+    .filter(token => token.length > 0)
+    .slice(0, 12)
+  if (tokens.length === 0) return ''
+  return tokens.map(token => '"' + token.replaceAll('"', '""') + '"').join(' OR ')
+}
+
+function toBuffer(vector) {
+  return Buffer.from(new Float32Array(vector).buffer)
+}
+
+function toVector(blob) {
+  if (!blob) return null
+  const buffer = Buffer.isBuffer(blob) ? blob : Buffer.from(blob)
+  if (buffer.byteLength % 4 !== 0) return null
+  return new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4)
+}
+
+function cosine(a, b) {
+  if (!a || !b || a.length !== b.length) return 0
+  let dot = 0
+  let na = 0
+  let nb = 0
+  for (let i = 0; i < a.length; i += 1) {
+    dot += a[i] * b[i]
+    na += a[i] * a[i]
+    nb += b[i] * b[i]
+  }
+  if (na === 0 || nb === 0) return 0
+  return dot / Math.sqrt(na * nb)
+}
+
+export class MemoryStore {
+  constructor(path) {
+    if (!path) throw new Error('MemoryStore requires a database path')
+    mkdirSync(dirname(path), { recursive: true })
+    this.path = path
+    this.db = new DatabaseSync(path)
+    this.db.exec(`
+      PRAGMA journal_mode = WAL;
+      CREATE TABLE IF NOT EXISTS memories (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        hash TEXT NOT NULL UNIQUE,
+        kind TEXT NOT NULL DEFAULT 'fact',
+        text TEXT NOT NULL,
+        session_id TEXT,
+        seq INTEGER,
+        role TEXT,
+        importance REAL NOT NULL DEFAULT 0.5,
+        pinned INTEGER NOT NULL DEFAULT 0,
+        embedding BLOB,
+        dim INTEGER,
+        created_at INTEGER NOT NULL
+      );
+      CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(text);
+      CREATE INDEX IF NOT EXISTS memories_kind ON memories(kind);
+      CREATE INDEX IF NOT EXISTS memories_pinned ON memories(pinned);
+    `)
+  }
+
+  add({ text, kind = 'fact', sessionId = null, seq = null, role = null, importance = 0.5, pinned = false, embedding = null }) {
+    const content = String(text || '').trim()
+    if (!content) return null
+    const digest = hash(content)
+    const existing = this.db.prepare('SELECT id FROM memories WHERE hash = ?').get(digest)
+    if (existing) return Number(existing.id)
+    const info = this.db.prepare(`
+      INSERT INTO memories (hash, kind, text, session_id, seq, role, importance, pinned, embedding, dim, created_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(digest, kind, content, sessionId, seq, role, Number(importance) || 0.5,
+      pinned ? 1 : 0, embedding ? toBuffer(embedding) : null,
+      embedding ? embedding.length : null, Date.now())
+    const id = Number(info.lastInsertRowid)
+    this.db.prepare('INSERT INTO memories_fts (rowid, text) VALUES (?, ?)').run(id, content)
+    return id
+  }
+
+  setEmbedding(id, vector) {
+    if (!vector || !vector.length) return false
+    this.db.prepare('UPDATE memories SET embedding = ?, dim = ? WHERE id = ?')
+      .run(toBuffer(vector), vector.length, Number(id))
+    return true
+  }
+
+  get(id) {
+    return this.db.prepare('SELECT * FROM memories WHERE id = ?').get(Number(id)) || null
+  }
+
+  pinned(limit = 12) {
+    return this.db.prepare('SELECT * FROM memories WHERE pinned = 1 ORDER BY importance DESC, id DESC LIMIT ?')
+      .all(Number(limit))
+      .map(row => this.#view(row))
+  }
+
+  keyword(query, limit = 8) {
+    const match = ftsQuery(query)
+    let rows = []
+    if (match) {
+      try {
+        rows = this.db.prepare(`
+          SELECT m.* FROM memories_fts f JOIN memories m ON m.id = f.rowid
+          WHERE memories_fts MATCH ? ORDER BY rank LIMIT ?
+        `).all(match, Number(limit))
+      } catch {
+        rows = []
+      }
+    }
+    if (rows.length === 0) {
+      const like = '%' + String(query || '').trim() + '%'
+      rows = this.db.prepare('SELECT * FROM memories WHERE text LIKE ? ORDER BY importance DESC, id DESC LIMIT ?')
+        .all(like, Number(limit))
+    }
+    return rows.map(row => this.#view(row))
+  }
+
+  vector(queryVector, limit = 8, scan = 4000) {
+    if (!queryVector || !queryVector.length) return []
+    const rows = this.db.prepare('SELECT * FROM memories WHERE embedding IS NOT NULL ORDER BY id DESC LIMIT ?')
+      .all(Number(scan))
+    const scored = []
+    for (const row of rows) {
+      const vector = toVector(row.embedding)
+      if (!vector) continue
+      const score = cosine(queryVector, vector)
+      if (score > 0.2) scored.push({ row, score })
+    }
+    scored.sort((a, b) => b.score - a.score)
+    return scored.slice(0, Number(limit)).map(item => ({ ...this.#view(item.row), score: item.score }))
+  }
+
+  forget(id) {
+    const numeric = Number(id)
+    this.db.prepare('DELETE FROM memories_fts WHERE rowid = ?').run(numeric)
+    return this.db.prepare('DELETE FROM memories WHERE id = ?').run(numeric).changes > 0
+  }
+
+  stats() {
+    const total = Number(this.db.prepare('SELECT COUNT(*) AS n FROM memories').get().n)
+    const pinned = Number(this.db.prepare('SELECT COUNT(*) AS n FROM memories WHERE pinned = 1').get().n)
+    const embedded = Number(this.db.prepare('SELECT COUNT(*) AS n FROM memories WHERE embedding IS NOT NULL').get().n)
+    const byKind = this.db.prepare('SELECT kind, COUNT(*) AS n FROM memories GROUP BY kind').all()
+      .map(row => ({ kind: row.kind, count: Number(row.n) }))
+    return { total, pinned, embedded, byKind, path: this.path }
+  }
+
+  #view(row) {
+    return {
+      id: Number(row.id),
+      kind: row.kind,
+      text: row.text,
+      sessionId: row.session_id,
+      seq: row.seq === null ? null : Number(row.seq),
+      role: row.role,
+      importance: Number(row.importance),
+      pinned: Boolean(row.pinned),
+      createdAt: Number(row.created_at),
+    }
+  }
+
+  close() {
+    this.db.close()
+  }
+}
+
+export function newMemoryId() {
+  return randomUUID()
+}

package/package.json

@@ -1,6 +1,32 @@
 {
   "name": "dsh-memory-vector",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.1",
+  "description": "不死记忆：给 DSH 伙伴的本地长期记忆，SQLite + 本地向量，上下文压缩后仍能想起你是谁。",
+  "type": "module",
+  "main": "lib/index.js",
+  "exports": {
+    ".": "./lib/index.js"
+  },
+  "files": [
+    "lib",
+    "README.md",
+    "LICENSE"
+  ],
+  "keywords": [
+    "dsh",
+    "dsh-plugin",
+    "memory",
+    "companion",
+    "sqlite"
+  ],
+  "license": "MIT",
+  "engines": {
+    "node": ">=22.5"
+  },
+  "dependencies": {
+    "@huggingface/transformers": "^4.3.2"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}