dsh-lpipe-memory 0.0.0-stage → 0.1.0

package/src/mediapack.mjs

@@ -0,0 +1,144 @@
+// 媒体包：图片不再散在全局 media/ 里，跟着它所属的那一天走。
+//
+// 一天一个包：`days/<day>.media.zst`，追加式帧。一帧 = 一行表头 + 若干张图的原始字节，
+// 表头记下每张图在自己这一帧明文里的偏移与长度，`days/<day>.media.idx` 再记下
+// 「这一帧在包里从哪开始、多长」，于是取一张图只解那一帧、只切一段。
+//
+// 图片本身已经是压缩格式，用等级 1 —— 再往上压不动，白白费时间。
+import { appendFileSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, readSync, statSync } from 'node:fs'
+import { dirname } from 'node:path'
+import { constants as C, zstdCompressSync, zstdDecompressSync } from 'node:zlib'
+
+import { mediaIdxFile, mediaPackFile } from './paths.mjs'
+
+export const MEDIA_LEVEL = 1
+
+function readLines(file) {
+  if (!existsSync(file)) return []
+  return readFileSync(file, 'utf8').split('\n').filter(Boolean)
+}
+
+function readRange(file, position, length) {
+  const fd = openSync(file, 'r')
+  try {
+    const buffer = Buffer.allocUnsafe(length)
+    readSync(fd, buffer, 0, length, position)
+    return buffer
+  } finally {
+    closeSync(fd)
+  }
+}
+
+/** 这一天的媒体索引：sha256 → (帧位置, 帧长, 帧内偏移, 字节数)。 */
+export function mediaIndex(root, sessionId, day) {
+  const index = new Map()
+  for (const line of readLines(mediaIdxFile(root, sessionId, day))) {
+    try {
+      const item = JSON.parse(line)
+      if (typeof item?.s === 'string') index.set(item.s, item)
+    } catch {
+      // 坏行跳过：最坏结果是那张图要重新转存一次
+    }
+  }
+  return index
+}
+
+/** 把一批图追加进这一天的媒体包；已经在这一天里的直接跳过。 */
+export function appendMedia(root, sessionId, day, items, { level = MEDIA_LEVEL } = {}) {
+  const pack = mediaPackFile(root, sessionId, day)
+  const known = mediaIndex(root, sessionId, day)
+  const fresh = []
+  const seen = new Set()
+  for (const item of items) {
+    const sha = item?.sha256
+    if (typeof sha !== 'string' || !Buffer.isBuffer(item?.bytes)) continue
+    if (known.has(sha) || seen.has(sha)) continue
+    seen.add(sha)
+    fresh.push(item)
+  }
+  if (fresh.length === 0) return { added: 0, skipped: items.length }
+
+  const header = []
+  let cursor = 0
+  for (const item of fresh) {
+    header.push({ s: item.sha256, o: cursor, l: item.bytes.length })
+    cursor += item.bytes.length
+  }
+  const head = Buffer.from(JSON.stringify(header) + '\n', 'utf8')
+  const payload = Buffer.concat([head, ...fresh.map((item) => item.bytes)])
+  const packed = zstdCompressSync(payload, { params: { [C.ZSTD_c_compressionLevel]: level } })
+
+  mkdirSync(dirname(pack), { recursive: true })
+  const at = existsSync(pack) ? statSync(pack).size : 0
+  appendFileSync(pack, packed)
+  const lines = fresh.map((item, i) => JSON.stringify({
+    s: item.sha256,
+    a: at,
+    z: packed.length,
+    o: head.length + header[i].o,
+    l: item.bytes.length,
+    e: item.ext ?? 'bin',
+    t: item.mediaType ?? 'application/octet-stream',
+    n: item.name ?? undefined,
+  }))
+  appendFileSync(mediaIdxFile(root, sessionId, day), lines.join('\n') + '\n')
+  return { added: fresh.length, skipped: items.length - fresh.length }
+}
+
+/** 取一张图的字节；这一天里没有就返回 null。 */
+export function readMedia(root, sessionId, day, sha256) {
+  const entry = mediaIndex(root, sessionId, day).get(sha256)
+  if (!entry) return null
+  const whole = readRange(mediaPackFile(root, sessionId, day), entry.a, entry.z)
+  const payload = zstdDecompressSync(whole)
+  return payload.subarray(entry.o, entry.o + entry.l)
+}
+
+/** 这一天的媒体包在磁盘上占多少（包 + 索引）。 */
+export function mediaPackBytes(root, sessionId, day) {
+  let total = 0
+  for (const file of [mediaPackFile(root, sessionId, day), mediaIdxFile(root, sessionId, day)]) {
+    try {
+      total += statSync(file).size
+    } catch {
+      // 没有媒体就是 0
+    }
+  }
+  return total
+}
+
+
+/** 这一天的每张图在磁盘上摊到多少字节：帧的压缩长度按原字节比例分给帧内的图。 */
+export function mediaShares(root, sessionId, day) {
+  const shares = new Map()
+  const index = mediaIndex(root, sessionId, day)
+  if (index.size === 0) return shares
+
+  let idxBytes = 0
+  try {
+    idxBytes = statSync(mediaIdxFile(root, sessionId, day)).size
+  } catch {
+    idxBytes = 0
+  }
+
+  const byFrame = new Map()
+  let totalWeight = 0
+  for (const entry of index.values()) {
+    const weight = Number(entry.l) || 1
+    totalWeight += weight
+    const key = `${entry.a}|${entry.z}`
+    const list = byFrame.get(key) ?? { frame: Number(entry.z) || 0, rows: [] }
+    list.rows.push({ sha: entry.s, weight })
+    byFrame.set(key, list)
+  }
+
+  for (const { frame, rows } of byFrame.values()) {
+    const sum = rows.reduce((acc, item) => acc + item.weight, 0) || 1
+    for (const item of rows) {
+      const packed = (frame * item.weight) / sum
+      const meta = (idxBytes * item.weight) / (totalWeight || 1)
+      shares.set(item.sha, packed + meta)
+    }
+  }
+  return shares
+}

package/src/pack.mjs

@@ -0,0 +1,94 @@
+// 冷日打包：过完的那一天整天重打成一条 zstd 流，等级高，能看见整天的上下文。
+// 正在写的那一天不动——追加只压新的一段，代价低；整条流一重写就要重来看不见的甜头。
+import { existsSync, readFileSync, readdirSync, renameSync, statSync, writeFileSync } from 'node:fs'
+import { constants as C, zstdCompressSync, zstdDecompressSync } from 'node:zlib'
+
+import { dayOf } from './day.mjs'
+import { decompressFrame } from './frames.mjs'
+import { dayFile, daysDir, framesFile, idxFile, sessionsDir } from './paths.mjs'
+
+/** 冷日的压缩等级：只有落盘那一刻付一次，读的时候照样只解一条流。 */
+export const COLD_LEVEL = 19
+
+function readLines(file) {
+  if (!existsSync(file)) return []
+  return readFileSync(file, 'utf8').split('\n').filter(Boolean)
+}
+
+function writeAtomic(file, bytes) {
+  const tmp = `${file}.pack-${process.pid}`
+  writeFileSync(tmp, bytes)
+  renameSync(tmp, file)
+}
+
+/** 这一天是不是已经打成一条流了。 */
+export function isPacked(root, sessionId, day, level = COLD_LEVEL) {
+  if (!existsSync(dayFile(root, sessionId, day))) return false
+  const frames = readLines(framesFile(root, sessionId, day)).map((line) => JSON.parse(line))
+  return frames.length === 1 && frames[0]?.lvl === level
+}
+
+/**
+ * 把一天的多帧重打成一条流：解开每一帧、拼回整天、整段压到一条。
+ * 偏移表里的 `o` 原本是「帧内偏移」，这里换算成「整天偏移」，`f` 一律归 0。
+ */
+export function packDay(root, sessionId, day, { level = COLD_LEVEL } = {}) {
+  const data = dayFile(root, sessionId, day)
+  if (!existsSync(data)) return { day, skipped: 'missing' }
+  const frames = readLines(framesFile(root, sessionId, day)).map((line) => JSON.parse(line))
+  if (frames.length === 0) return { day, skipped: 'no-frames' }
+  if (frames.length === 1 && frames[0].lvl === level) return { day, skipped: 'packed' }
+
+  const before = statSync(data).size
+  const bytes = readFileSync(data)
+  const plains = frames.map((frame) => decompressFrame(bytes.subarray(frame.at, frame.at + frame.len)))
+  const whole = plains.join('')
+  const packed = zstdCompressSync(Buffer.from(whole, 'utf8'), {
+    params: { [C.ZSTD_c_compressionLevel]: level },
+  })
+  // 写完先自己解一遍：解出来必须和原文一字不差，才允许替换。
+  if (zstdDecompressSync(packed).toString('utf8') !== whole) throw new Error(`打包校验失败：${sessionId}/${day}`)
+
+  const offsets = []
+  let cursor = 0
+  for (const plain of plains) {
+    offsets.push(cursor)
+    cursor += Buffer.byteLength(plain)
+  }
+  const index = readLines(idxFile(root, sessionId, day)).map((line) => {
+    const entry = JSON.parse(line)
+    return JSON.stringify({ ...entry, f: 0, o: (offsets[entry.f] ?? 0) + entry.o })
+  })
+
+  writeAtomic(data, packed)
+  writeAtomic(
+    framesFile(root, sessionId, day),
+    Buffer.from(JSON.stringify({ n: 0, at: 0, len: packed.length, lvl: level }) + '\n', 'utf8'),
+  )
+  writeAtomic(idxFile(root, sessionId, day), Buffer.from(index.join('\n') + '\n', 'utf8'))
+  return { day, session: sessionId, before, after: packed.length }
+}
+
+/** 把「今天以外」的日子全部打包一遍；已经打过的直接跳过。 */
+export function packColdDays(root, { level = COLD_LEVEL, today = dayOf(Date.now()), log } = {}) {
+  const sessions = existsSync(sessionsDir(root)) ? readdirSync(sessionsDir(root)) : []
+  const days = []
+  for (const sessionId of sessions) {
+    const dir = daysDir(root, sessionId)
+    if (!existsSync(dir)) continue
+    for (const name of readdirSync(dir)) {
+      if (!name.endsWith('.jsonl.zst')) continue
+      const day = name.replace('.jsonl.zst', '')
+      if (day === today) continue
+      try {
+        const result = packDay(root, sessionId, day, { level })
+        if (!result.skipped) days.push(result)
+      } catch (error) {
+        log?.(`[dsh-lpipe-memory] 冷日打包失败 ${sessionId}/${day}：${error?.message ?? error}`)
+      }
+    }
+  }
+  const before = days.reduce((sum, item) => sum + item.before, 0)
+  const after = days.reduce((sum, item) => sum + item.after, 0)
+  return { packed: days.length, before, after }
+}

package/src/paths.mjs

@@ -0,0 +1,54 @@
+// 路径规则：一条会话一个目录，目录里按天分卷。
+import { homedir } from 'node:os'
+import { join } from 'node:path'
+
+/** 记忆根目录。DSH_HOME 与宿主一致，没设就退回 ~/.dsh。 */
+export function memoryRoot(env = process.env) {
+  const base = typeof env.DSH_HOME === 'string' && env.DSH_HOME.trim() ? env.DSH_HOME.trim() : join(homedir(), '.dsh')
+  return join(base, 'life-memory')
+}
+
+export function sessionsDir(root) {
+  return join(root, 'sessions')
+}
+
+export function sessionDir(root, sessionId) {
+  return join(sessionsDir(root), sessionId)
+}
+
+export function daysDir(root, sessionId) {
+  return join(sessionDir(root, sessionId), 'days')
+}
+
+export function dayFile(root, sessionId, day) {
+  return join(daysDir(root, sessionId), `${day}.jsonl.zst`)
+}
+
+export function idxFile(root, sessionId, day) {
+  return join(daysDir(root, sessionId), `${day}.idx`)
+}
+
+export function framesFile(root, sessionId, day) {
+  return join(daysDir(root, sessionId), `${day}.frames`)
+}
+
+export function metaFile(root, sessionId) {
+  return join(sessionDir(root, sessionId), 'meta.json')
+}
+
+export function stateFile(root) {
+  return join(root, 'state.json')
+}
+
+export function mediaPackFile(root, sessionId, day) {
+  return join(daysDir(root, sessionId), `${day}.media.zst`)
+}
+
+export function mediaIdxFile(root, sessionId, day) {
+  return join(daysDir(root, sessionId), `${day}.media.idx`)
+}
+
+/** media 全局共享、内容寻址：同一份字节在两个会话里只存一次。 */
+export function mediaFile(root, sha256, ext = 'bin') {
+  return join(root, 'media', sha256.slice(0, 2), sha256.slice(2, 4), `${sha256}.${ext}`)
+}

package/src/project.mjs

@@ -0,0 +1,132 @@
+// 投影：把宿主的会话事件压成记忆行。白名单之外一律不写。
+const KIND = {
+  'user/message': 'user/message',
+  'assistant/message': 'assistant/message',
+  'tool/call': 'tool/call',
+  'tool/result': 'tool/result',
+}
+
+export const STORED_KINDS = new Set(Object.values(KIND))
+
+/**
+ * 注入进来的上下文：不是说过的话，不写入。
+ *
+ * 除了运行期快照，还包括管线自己注入的东西——回忆、联想、想象。它们是程序拼给模型看的，
+ * 写进记忆就等于把管线的加工结果当成原始对话，越滚越多。
+ */
+const INJECTED_SOURCES = new Set([
+  'runtime-context',
+  'dsh-lpipe-recall',
+  'dsh-lpipe-assoc',
+  'dsh-lpipe-imagine',
+])
+
+/** 管线工具的命名约定：recall_* / assoc_* / imagine_*。 */
+const PIPELINE_TOOL = /^(recall_|assoc_|imagine_)/
+
+/** 记下管线工具的 callId，好把它们的「结果」也一并丢掉（结果行本身不带工具名）。 */
+const pipelineCalls = new Set()
+
+function markPipelineCall(callId) {
+  const id = String(callId ?? '')
+  if (!id) return
+  pipelineCalls.add(id)
+  // 只留最近的一批，避免长会话里越攒越多。
+  if (pipelineCalls.size > 512) {
+    const oldest = pipelineCalls.values().next().value
+    pipelineCalls.delete(oldest)
+  }
+}
+
+export function resetPipelineCalls() {
+  pipelineCalls.clear()
+}
+
+/** 把消息里的内容块拆成「给人看的文本」与「结构化块」。 */
+function splitContent(content) {
+  if (typeof content === 'string') return { text: content, parts: [{ type: 'text', text: content }] }
+  if (!Array.isArray(content)) return { text: '', parts: [] }
+  const texts = []
+  for (const part of content) {
+    if (part?.type === 'text' && typeof part.text === 'string') texts.push(part.text)
+  }
+  return { text: texts.join('\n'), parts: content }
+}
+
+function reasoningOf(parts) {
+  const chunks = []
+  for (const part of parts ?? []) {
+    if (part?.type === 'reasoning' && typeof part.text === 'string') chunks.push(part.text)
+  }
+  return chunks.join('\n')
+}
+
+/**
+ * 只留搬不走的结构块：文本与思考已经单独存在 `text` / `reasoning` 里，
+ * 再存一份就是纯重复（历史上一度让 42% 的体积变成副本）。
+ * 图片、文件、工具结构化结果没有别的落点，必须留着。
+ */
+export function carriedParts(parts) {
+  if (!Array.isArray(parts)) return []
+  return parts.filter((part) => part && part.type !== 'text' && part.type !== 'reasoning')
+}
+
+function payloadWith(parts, extra) {
+  const carried = carriedParts(parts)
+  return carried.length > 0 ? { ...extra, parts: carried } : extra
+}
+
+/** 一条事件要么投影成行，要么返回 null（不入档）。 */
+export function project(event) {
+  const kind = KIND[event?.type]
+  if (!kind) return null
+  const data = event.data ?? {}
+  const turn = Number.isFinite(data.turn) ? data.turn : null
+  const step = Number.isFinite(data.step) ? data.step : null
+
+  if (kind === 'user/message') {
+    // 运行期注入的上下文不是「说过的话」，不进记忆层。
+    const source = data.source ?? null
+    if (INJECTED_SOURCES.has(source?.kind)) return null
+    const { text, parts } = splitContent(data.content)
+    return { kind, turn, step, payload: payloadWith(parts, { source, text }) }
+  }
+
+  if (kind === 'assistant/message') {
+    const { text, parts } = splitContent(data.message?.content)
+    return {
+      kind,
+      turn,
+      step,
+      payload: payloadWith(parts, {
+        text,
+        reasoning: reasoningOf(parts),
+        usage: data.usage ?? null,
+        interrupted: data.interrupted === true || undefined,
+      }),
+    }
+  }
+
+  if (kind === 'tool/call') {
+    // 管线工具的调用不是「对话里发生的事」，连调用带结果一起不要。
+    if (PIPELINE_TOOL.test(String(data.name ?? ''))) {
+      markPipelineCall(data.callId)
+      return null
+    }
+    return { kind, turn, step, payload: { callId: data.callId ?? null, name: data.name ?? '', arguments: data.arguments ?? '' } }
+  }
+
+  if (kind === 'tool/result' && pipelineCalls.has(String(data.callId ?? ''))) return null
+
+  const { text, parts } = splitContent(data.message?.content)
+  return {
+    kind,
+    turn,
+    step,
+    payload: payloadWith(parts, {
+      text,
+      isError: data.message?.isError === true || undefined,
+      error: data.error ? { name: data.error.name, code: data.error.code, reason: data.error.reason ?? '' } : undefined,
+    }),
+  }
+}