dsh-lpipe-memory 0.0.0-stage → 0.1.0

package/package.json

@@ -1,6 +1,59 @@
 {
   "name": "dsh-lpipe-memory",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "type": "module",
+  "description": "把会话原文逐字投影到长期文件：按会话分文件夹，按天分卷，哈希链可校验。",
+  "license": "AGPL-3.0-or-later",
+  "main": "src/index.js",
+  "exports": {
+    ".": "./src/index.js",
+    "./reader": "./src/reader.mjs",
+    "./paths": "./src/paths.mjs",
+    "./client": "./lib/client.js",
+    "./locale/en.json": "./locale/en.json",
+    "./locale/zh.json": "./locale/zh.json",
+    "./package.json": "./package.json",
+    "./icon": "./assets/icon.png"
+  },
+  "dependencies": {
+    "dsh-life-pipline": "^0.7.1"
+  },
+  "scripts": {
+    "test": "node --test \"test/*.test.mjs\"",
+    "build-client": "node scripts/build-client.mjs"
+  },
+  "files": [
+    "src",
+    "lib",
+    "client",
+    "assets",
+    "locale",
+    "cordis.patch.yml",
+    "LICENSE",
+    "README.md"
+  ],
+  "keywords": [
+    "dsh",
+    "dsh-plugin",
+    "deepseek-harness",
+    "memory"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Kaczev/dsh-life-pipline.git",
+    "directory": "components/dsh-lpipe-memory"
+  },
+  "dsh": {
+    "bundle": {
+      "patch": "./cordis.patch.yml"
+    },
+    "client": {
+      "inject": [
+        "@deepseek-ai/dsh-client-connection",
+        "@deepseek-ai/dsh-client-ui-slots"
+      ],
+      "platform": "web"
+    }
+  },
+  "icon": "assets/icon.png"
+}

package/src/chain.mjs

@@ -0,0 +1,28 @@
+// 哈希链：每一行的 hash 由上一行的 hash 与本行正文算出，跨文件、跨天相连。
+import { createHash } from 'node:crypto'
+
+export const GENESIS = 'sha256:genesis'
+
+/** 正文：一行除了 `prev` 与 `hash` 之外的全部字段，按插入顺序序列化。 */
+export function bodyOf(row) {
+  const { prev: _prev, hash: _hash, ...rest } = row
+  return JSON.stringify(rest)
+}
+
+export function hashOf(prev, body) {
+  return 'sha256:' + createHash('sha256').update(`${prev}\n${body}`).digest('hex')
+}
+
+/** 给一行接上链：写入前调用，`prev` 是上一行的 hash。 */
+export function seal(row, prev) {
+  const body = bodyOf(row)
+  return { ...row, prev, hash: hashOf(prev, body) }
+}
+
+/** 校验一行：返回 null 表示完好，否则返回人话说的原因。 */
+export function checkRow(row, prev) {
+  if (typeof row?.hash !== 'string' || typeof row?.prev !== 'string') return '缺少 prev/hash'
+  if (row.prev !== prev) return `prev 对不上：期望 ${prev}，实际 ${row.prev}`
+  const want = hashOf(row.prev, bodyOf(row))
+  return want === row.hash ? null : 'hash 与正文不符'
+}

package/src/compact.mjs

@@ -0,0 +1,167 @@
+// 迁移：把旧的记忆库重建成「不再重复存 parts」的格式，哈希链整条重算。
+// 只在库的 version 落后时跑一次；跑完把旧目录改名留作备份。
+import { cpSync, existsSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+import { GENESIS } from './chain.mjs'
+import { MemoryReader } from './reader.mjs'
+import { metaFile, stateFile } from './paths.mjs'
+import { carriedParts } from './project.mjs'
+import { MemoryWriter } from './writer.mjs'
+
+/**
+ * 2 = 行里不再存 text/reasoning 的副本。
+ * 3 = 每条行都带轮号：早先 `user/message` 事件本身不带 turn，那些行是 null，
+ *     星图按轮分组时会把它们整条跳过，于是你自己的话从来没有参与抽星。
+ */
+export const FORMAT_VERSION = 2
+
+const STRUCTURAL = new Set(['seq', 'day', 'at', 'session', 'turn', 'step', 'kind', 'prev', 'hash'])
+
+function readJson(file, fallback) {
+  try {
+    return JSON.parse(readFileSync(file, 'utf8'))
+  } catch {
+    return fallback
+  }
+}
+
+function writeJson(file, value) {
+  writeFileSync(file, JSON.stringify(value, null, 2) + '\n', 'utf8')
+}
+
+/** 这一条旧行还带着重复的 parts 副本吗？ */
+export function hasDuplicateParts(row) {
+  const parts = row?.parts
+  if (!Array.isArray(parts)) return false
+  return parts.some((part) => part?.type === 'text' || part?.type === 'reasoning')
+}
+
+function payloadOf(row) {
+  const payload = {}
+  for (const [key, value] of Object.entries(row)) {
+    if (STRUCTURAL.has(key)) continue
+    payload[key] = value
+  }
+  if (Array.isArray(payload.parts)) {
+    const carried = carriedParts(payload.parts)
+    if (carried.length > 0) payload.parts = carried
+    else delete payload.parts
+  }
+  return payload
+}
+
+export function needsMigration(root) {
+  const state = readJson(stateFile(root), null)
+  if (!state) return false
+  return (state.version ?? 1) < FORMAT_VERSION
+}
+
+/**
+ * 整库重建：行照旧、seq 照旧、顺序照旧，只去掉重复的 parts，并重新焊哈希链。
+ * 旧目录改名成 `<root>.bak-<时间>`，不删除。
+ */
+export function compact({ root, log = () => {}, now = () => Date.now() } = {}) {
+  const oldState = readJson(stateFile(root), { version: 1, sessions: {} })
+  const oldMetas = new Map()
+  for (const sessionId of Object.keys(oldState.sessions ?? {})) {
+    const meta = readJson(metaFile(root, sessionId), null)
+    if (meta) oldMetas.set(sessionId, meta)
+  }
+
+  const reader = new MemoryReader({ root }).reload()
+  const next = `${root}.next`
+  rmSync(next, { recursive: true, force: true })
+  const writer = new MemoryWriter({ root: next, now })
+
+  let maxSeq = -1
+  for (const session of reader.sessions()) maxSeq = Math.max(maxSeq, session.lastSeq)
+  const all = []
+  for (let seq = 0; seq <= maxSeq; seq += 1) {
+    const row = reader.read(seq)
+    if (row) all.push(row)
+  }
+  // 缺轮号的行向后取最近一个有轮号的行：一是这段行为只追加，
+  // 二是「一轮的第一条 user/message」后面紧跟的正是同一轮的行。
+  const ahead = new Map()
+  const turns = new Array(all.length).fill(null)
+  for (let index = all.length - 1; index >= 0; index -= 1) {
+    const row = all[index]
+    if (Number.isInteger(row.turn)) ahead.set(row.session, row.turn)
+    turns[index] = Number.isInteger(row.turn) ? row.turn : (ahead.get(row.session) ?? null)
+  }
+  let rows = 0
+  for (const [index, row] of all.entries()) {
+    writer.append({
+      sessionId: row.session,
+      at: row.at,
+      kind: row.kind,
+      turn: turns[index],
+      step: row.step ?? null,
+      payload: payloadOf(row),
+    })
+    rows += 1
+  }
+  writer.close()
+
+  // 会话的事件游标与标题不能丢：前者决定幂等，后者是目录。
+  const state = readJson(stateFile(next), { version: 1, seq: -1, hash: GENESIS, sessions: {} })
+  for (const [sessionId, info] of Object.entries(oldState.sessions ?? {})) {
+    const target = state.sessions?.[sessionId]
+    if (!target) continue
+    if (Number.isFinite(info.lastEventSeq)) target.lastEventSeq = info.lastEventSeq
+  }
+  state.version = FORMAT_VERSION
+  writeJson(stateFile(next), state)
+  for (const [sessionId, meta] of oldMetas) {
+    const file = metaFile(next, sessionId)
+    const nextMeta = readJson(file, null)
+    if (!nextMeta) continue
+    if (meta.title) nextMeta.title = meta.title
+    writeJson(file, nextMeta)
+  }
+
+  const mediaDir = join(root, 'media')
+  if (existsSync(mediaDir)) cpSync(mediaDir, join(next, 'media'), { recursive: true, force: true })
+
+  const backup = `${root}.bak-${new Date(now()).toISOString().replace(/[:.]/g, '-')}`
+  // Windows 上目录里只要有一个文件被别人打开着，改名就会被拒（EPERM/EBUSY）。
+  // 启动时星图那边可能正在读记忆，所以这里重试几次，别让整次迁移白做。
+  swapWithRetry(root, backup, log)
+  log(`[dsh-lpipe-memory] 已重建 ${rows} 行（旧库备份在 ${backup}）`)
+  return { rows, backup }
+}
+
+/** 把旧目录挪开、新目录就位；被占用时等一会儿再试。 */
+function swapWithRetry(root, backup, log, attempts = 8, waitMs = 400) {
+  for (let attempt = 1; attempt <= attempts; attempt += 1) {
+    try {
+      renameSync(root, backup)
+      renameSync(`${root}.next`, root)
+      return true
+    } catch (error) {
+      const code = error?.code ?? ''
+      const transient = code === 'EPERM' || code === 'EBUSY' || code === 'EACCES'
+      // 第一次改名成了、第二次没成：把旧目录放回去，下次再来。
+      try {
+        if (existsSync(backup) && !existsSync(root)) renameSync(backup, root)
+      } catch {}
+      if (!transient || attempt === attempts) throw error
+      log(`[dsh-lpipe-memory] 迁移第 ${attempt} 次改名被占用（${code}），稍后重试`)
+      Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, waitMs)
+    }
+  }
+  return false
+}
+
+/** 启动时按需迁移；失败只报一句，绝不挡住写入。 */
+export function migrateIfNeeded({ root, log = () => {} } = {}) {
+  if (!existsSync(stateFile(root))) return null
+  if (!needsMigration(root)) return null
+  try {
+    return compact({ root, log })
+  } catch (error) {
+    log(`[dsh-lpipe-memory] 迁移失败，继续用旧格式：${error?.message ?? error}`)
+    return null
+  }
+}

package/src/day.mjs

@@ -0,0 +1,15 @@
+// 天界：凌晨 4 点换天，按本地时间算。
+export const DAY_BOUNDARY_HOURS = 4
+
+/**
+ * `at` 落在哪一天。先把时间往回拨 4 小时，再取本地日期——
+ * 于是 04:00 到次日 03:59 之间发生的一切都归到同一天。
+ */
+export function dayOf(at) {
+  if (!Number.isFinite(at)) throw new TypeError('dayOf 需要一个毫秒时间戳')
+  const shifted = new Date(at - DAY_BOUNDARY_HOURS * 3600_000)
+  const year = shifted.getFullYear()
+  const month = String(shifted.getMonth() + 1).padStart(2, '0')
+  const day = String(shifted.getDate()).padStart(2, '0')
+  return `${year}-${month}-${day}`
+}

package/src/frames.mjs

@@ -0,0 +1,12 @@
+// 帧：写盘与读回都按 8 KB 一帧压缩，每帧独立可解。
+import { zstdCompressSync, zstdDecompressSync } from 'node:zlib'
+
+export const FRAME_BYTES = 8 * 1024
+
+export function compressFrame(text) {
+  return zstdCompressSync(Buffer.from(text, 'utf8'))
+}
+
+export function decompressFrame(bytes) {
+  return zstdDecompressSync(bytes).toString('utf8')
+}

package/src/index.js

@@ -0,0 +1,183 @@
+// 宿主半边：订阅 session/event，把白名单事件逐条追加进记忆。
+import { readSessionTitle } from 'dsh-life-pipline/session-title'
+
+import { captureMedia } from './media.mjs'
+import { migrateIfNeeded } from './compact.mjs'
+import { packColdDays } from './pack.mjs'
+import { memoryRoot } from './paths.mjs'
+import { project } from './project.mjs'
+import { MemoryReader } from './reader.mjs'
+import { createMemoryRpcHandler, registerMemoryRpc } from './rpc.mjs'
+import { createStatsCache } from './stats.mjs'
+import { MemoryWriter } from './writer.mjs'
+
+export const name = 'dsh-lpipe-memory'
+
+export const inject = []
+
+/**
+ * 一轮结束才落盘：一轮之内的行先攒在内存里，收到 turn/end 才一次性写进去。
+ * 盘上不会出现半截的轮次；没说完的轮次宁可不存，也不存一半。
+ */
+const TURN_END = 'turn/end'
+
+export function apply(ctx, options = {}) {
+  const root = options.root ?? memoryRoot()
+  // 宿主没给 logger 时别把失败吞掉：迁移、落盘出了问题要能看见。
+  const warn = (message) => {
+    if (typeof ctx.logger?.warn === 'function') ctx.logger.warn(message)
+    else console.warn(message)
+  }
+  // 旧格式（行里重复存 parts）在开写之前先重建，哈希链整条重算；失败就继续用旧格式。
+  if (!options.writer) migrateIfNeeded({ root, log: warn })
+  const writer = options.writer ?? new MemoryWriter({ root })
+  const reader = new MemoryReader({ root })
+  const stats = createStatsCache({ reader, root })
+  let attachments = null
+  /** sessionId -> 这一轮已经攒下的行 */
+  const buffered = new Map()
+
+  /** 把某个会话正在攒的轮次落盘；没有就什么也不做。 */
+  const commit = (sessionId, turn = null) => {
+    const rows = buffered.get(sessionId)
+    buffered.delete(sessionId)
+    if (!rows || rows.length === 0) return 0
+    // user/message 事件本身不带轮号：用 turn/end 递过来的那个补上，
+    // 否则这一行在星图里会被当成「没有轮次」整条跳过。
+    const known = Number.isInteger(turn)
+      ? turn
+      : (rows.find((row) => Number.isInteger(row.turn))?.turn ?? null)
+    const filled = known === null
+      ? rows
+      : rows.map((row) => (Number.isInteger(row.turn) ? row : { ...row, turn: known }))
+    try {
+      for (const row of filled) writer.append(row)
+      writer.flush()
+      refresh()
+      return rows.length
+    } catch (error) {
+      warn(`[dsh-lpipe-memory] 落盘失败：${error?.message ?? error}`)
+      return 0
+    }
+  }
+
+  // 落盘之后游标变了：读端与统计都得重新读一遍，页面才会看见这一轮。
+  const refresh = () => {
+    try {
+      reader.reload()
+    } catch (error) {
+      warn(`[dsh-lpipe-memory] 重读失败：${error?.message ?? error}`)
+    }
+    stats.invalidate()
+  }
+
+  ctx.effect(
+    () => () => {
+      // 卸载时还没说完的轮次不落盘，只把已落盘的部分收尾。
+      buffered.clear()
+      writer.close()
+    },
+    'dsh-lpipe-memory: close on teardown',
+  )
+
+  ctx.inject(['connection'], (connectionCtx) => {
+    try {
+      registerMemoryRpc(
+        connectionCtx,
+        createMemoryRpcHandler({ stats: (query) => stats.get(query) }),
+      )
+    } catch (error) {
+      warn(`[dsh-lpipe-memory] RPC 注册失败：${error?.message ?? error}`)
+    }
+  })
+
+  ctx.inject(['attachments'], (attachmentCtx) => {
+    attachments = attachmentCtx.attachments ?? null
+  })
+
+  // 冷日打包：启动后补做一次，之后每 6 小时一次；只碰「今天以外」的日子，打过的直接跳过。
+  if (options.pack !== false) {
+    // 必须写成 ctx.logger.info()：Cordis 的 logger 是可调用对象，方法取出来再调用会丢 this。
+    // 日志本身再出错也不能往外抛——宿主会把插件的异常当成致命加载失败。
+    const note = (message) => {
+      try {
+        if (typeof ctx.logger?.info === 'function') ctx.logger.info(message)
+        else if (typeof ctx.logger?.warn === 'function') ctx.logger.warn(message)
+      } catch {
+        // 日志写不进去就算了，打包已经完成
+      }
+    }
+    const sweep = () => {
+      try {
+        const result = packColdDays(root, { log: warn })
+        if (result.packed > 0) {
+          stats.invalidate()
+          note(`[dsh-lpipe-memory] 冷日打包 ${result.packed} 天：${result.before} → ${result.after} 字节`)
+        }
+      } catch (error) {
+        warn(`[dsh-lpipe-memory] 冷日打包失败：${error?.message ?? error}`)
+      }
+    }
+    const first = setTimeout(sweep, Number.isFinite(options.packDelay) ? options.packDelay : 5000)
+    const repeat = setInterval(sweep, 6 * 60 * 60 * 1000)
+    first.unref?.()
+    repeat.unref?.()
+    ctx.effect(
+      () => () => {
+        clearTimeout(first)
+        clearInterval(repeat)
+      },
+      'dsh-lpipe-memory: settle cold days',
+    )
+  }
+
+  ctx.inject(['sessions'], (sessionsCtx) => {
+    sessionsCtx.on('session/event', (session, event) => {
+      const sessionId = session?.id ?? ''
+      try {
+        // 轮次结束本身不入档，但它是这一轮完成的信号。
+        if (event?.type === TURN_END) {
+          commit(sessionId, event?.data?.turn ?? event?.turn ?? null)
+          return
+        }
+        const projected = project(event)
+        if (!projected) return
+        let payload = projected.payload
+        if (Array.isArray(payload?.parts) && payload.parts.some((part) => part?.attachment)) {
+          const captured = captureMedia({ attachments, parts: payload.parts, root, log: warn, sessionId, at: event?.time })
+          payload = { ...payload, parts: captured.parts, media: captured.media }
+        }
+        const rows = buffered.get(sessionId) ?? []
+        // 新一轮开始了：上一轮若没收到 turn/end，先把它收掉，别两轮混在一起。
+        if (
+          projected.kind === 'user/message' &&
+          rows.length > 0 &&
+          projected.turn !== null &&
+          rows[0].turn !== null &&
+          projected.turn !== rows[0].turn
+        ) {
+          commit(sessionId, projected.turn)
+        }
+        const target = buffered.get(sessionId) ?? []
+        target.push({
+          sessionId,
+          eventSeq: event?.seq,
+          at: event?.time,
+          kind: projected.kind,
+          turn: projected.turn,
+          step: projected.step,
+          payload,
+        })
+        buffered.set(sessionId, target)
+        if (projected.kind === 'user/message' && event?.seq === 0) {
+          const title = readSessionTitle?.(sessionsCtx.sessions, sessionId)
+          if (title?.title) writer.setTitle(sessionId, title.title)
+        }
+      } catch (error) {
+        warn(`[dsh-lpipe-memory] 写入失败：${error?.message ?? error}`)
+      }
+    })
+  })
+
+  return { writer, reader, stats }
+}

package/src/media.mjs

@@ -0,0 +1,97 @@
+// media 转存：把图片/附件的字节搬进「它所属的那一天」的媒体包，行里只留坐标。
+import { createHash } from 'node:crypto'
+import { readFileSync } from 'node:fs'
+
+import { dayOf } from './day.mjs'
+import { appendMedia } from './mediapack.mjs'
+
+const MAGIC = [
+  { ext: 'png', mediaType: 'image/png', test: (b) => b[0] === 0x89 && b[1] === 0x50 && b[2] === 0x4e && b[3] === 0x47 },
+  { ext: 'jpg', mediaType: 'image/jpeg', test: (b) => b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff },
+  { ext: 'gif', mediaType: 'image/gif', test: (b) => b.slice(0, 4).toString('latin1') === 'GIF8' },
+  {
+    ext: 'webp',
+    mediaType: 'image/webp',
+    test: (b) => b.slice(0, 4).toString('latin1') === 'RIFF' && b.slice(8, 12).toString('latin1') === 'WEBP',
+  },
+]
+
+/** 认格式只看魔数，不信声明。认不出就当二进制存。 */
+export function sniff(bytes) {
+  for (const entry of MAGIC) {
+    if (bytes.length >= 12 && entry.test(bytes)) return { ext: entry.ext, mediaType: entry.mediaType }
+  }
+  return { ext: 'bin', mediaType: 'application/octet-stream' }
+}
+
+function hostPathOf(attachments, ref) {
+  if (typeof ref !== 'object' || ref === null) return undefined
+  if (typeof ref.attachmentId === 'string' && ref.attachmentId.length === 0) return undefined
+  if (typeof attachments?.imageHostPath === 'function') {
+    try {
+      const path = attachments.imageHostPath(ref)
+      if (path) return path
+    } catch {
+      // 不是图片引用，接着试文件
+    }
+  }
+  if (typeof attachments?.fileHostPath === 'function') {
+    try {
+      return attachments.fileHostPath(ref) ?? undefined
+    } catch {
+      return undefined
+    }
+  }
+  return undefined
+}
+
+/**
+ * 把内容块里的附件字节搬进当天那一个媒体包。
+ * 返回新的 parts（含 sha256）与这一行要记的 media 清单；字节本身绝不进日志。
+ */
+export function captureMedia({ attachments, parts, root, log, sessionId, at }) {
+  if (!Array.isArray(parts) || parts.length === 0) return { parts, media: [] }
+  const media = []
+  const items = []
+  const next = []
+  for (const part of parts) {
+    const ref = part?.attachment
+    if ((part?.type !== 'image' && part?.type !== 'file') || !ref) {
+      next.push(part)
+      continue
+    }
+    try {
+      const path = hostPathOf(attachments, ref)
+      if (!path) throw new Error('宿主没给这个附件的落盘路径')
+      const bytes = readFileSync(path)
+      const sha256 = createHash('sha256').update(bytes).digest('hex')
+      const declared = typeof ref.mediaType === 'string' ? ref.mediaType : ''
+      const sniffed = sniff(bytes)
+      // 声明的类型只在它比魔数更具体时才算数：文件附件常带 application/octet-stream。
+      const trusted = declared !== '' && declared !== 'application/octet-stream'
+      const mediaType = trusted ? declared : sniffed.mediaType
+      const ext = trusted ? declared.split('/')[1]?.replace('jpeg', 'jpg') || sniffed.ext : sniffed.ext
+      items.push({ sha256, ext, mediaType, bytes, name: ref.name ?? undefined })
+      const record = { sha256, mediaType, bytes: bytes.length, name: ref.name ?? undefined }
+      media.push(record)
+      next.push({ ...part, sha256, mediaType, bytes: bytes.length })
+    } catch (error) {
+      const reason = String(error?.message ?? error).split('\n')[0]
+      log?.(`[dsh-lpipe-memory] 附件没转存：${reason}`)
+      next.push({ ...part, mediaError: reason })
+    }
+  }
+  if (items.length > 0) {
+    if (!sessionId) {
+      log?.('[dsh-lpipe-memory] 附件没转存：不知道属于哪个会话')
+    } else {
+      try {
+        appendMedia(root, sessionId, dayOf(Number.isFinite(at) ? at : Date.now()), items)
+      } catch (error) {
+        const reason = String(error?.message ?? error).split('\n')[0]
+        log?.(`[dsh-lpipe-memory] 媒体包写入失败：${reason}`)
+      }
+    }
+  }
+  return { parts: next, media }
+}