dsh-lpipe-memory 0.0.0-stage → 0.1.0

package/src/reader.mjs

@@ -0,0 +1,284 @@
+// 读取：按 seq 只解一帧；时间线跨会话合并；搜索走全扫（冷路径，调用方应放到 worker）。
+import { closeSync, existsSync, openSync, readFileSync, readSync, readdirSync } from 'node:fs'
+
+import { GENESIS, checkRow } from './chain.mjs'
+import { decompressFrame } from './frames.mjs'
+import { readMedia } from './mediapack.mjs'
+import { dayFile, daysDir, framesFile, idxFile, memoryRoot, stateFile } from './paths.mjs'
+
+function readJson(file, fallback) {
+  try {
+    return JSON.parse(readFileSync(file, 'utf8'))
+  } catch {
+    return fallback
+  }
+}
+
+function readLines(file) {
+  if (!existsSync(file)) return []
+  return readFileSync(file, 'utf8').split('\n').filter(Boolean)
+}
+
+function readRange(file, position, length) {
+  const fd = openSync(file, 'r')
+  try {
+    const buffer = Buffer.allocUnsafe(length)
+    readSync(fd, buffer, 0, length, position)
+    return buffer
+  } finally {
+    closeSync(fd)
+  }
+}
+
+export class MemoryReader {
+  constructor({ root = memoryRoot(), cacheFrames = 512 } = {}) {
+    this.root = root
+    this.cacheLimit = cacheFrames
+    this.frameCache = new Map()
+    // 帧缓存按字节封顶：一天的整包解压后有七八兆，按“帧数”记会把内存吃光。
+    this.frameBytes = 0
+    this.cacheBytes = 64 * 1024 * 1024
+    this.idxCache = new Map()
+    this.rowCache = new Map()
+    this.dayCache = new Map()
+    this.turnCache = new Map()
+    this.state = readJson(stateFile(root), { version: 1, seq: -1, hash: GENESIS, sessions: {} })
+  }
+
+  reload() {
+    this.state = readJson(stateFile(this.root), this.state)
+    this.frameCache.clear()
+    this.frameBytes = 0
+    this.idxCache.clear()
+    this.rowCache.clear()
+    this.dayCache.clear()
+    this.turnCache.clear()
+    return this
+  }
+
+  get size() {
+    return Object.keys(this.state.sessions ?? {}).length
+  }
+
+  /** 哪个会话拥有这个 seq。 */
+  sessionOf(seq) {
+    for (const [sessionId, info] of Object.entries(this.state.sessions ?? {})) {
+      if (seq >= info.firstSeq && seq <= info.lastSeq) return sessionId
+    }
+    return null
+  }
+
+  sessions() {
+    return Object.entries(this.state.sessions ?? {})
+      .map(([sessionId, info]) => ({ sessionId, firstSeq: info.firstSeq, lastSeq: info.lastSeq, rows: info.rows, lastAt: info.lastAt }))
+      .sort((a, b) => a.firstSeq - b.firstSeq)
+  }
+
+  daysOf(sessionId) {
+    if (this.dayCache.has(sessionId)) return this.dayCache.get(sessionId)
+    const dir = daysDir(this.root, sessionId)
+    const days = existsSync(dir)
+      ? readdirSync(dir)
+          .filter((name) => name.endsWith('.jsonl.zst'))
+          .map((name) => name.replace('.jsonl.zst', ''))
+          .sort()
+      : []
+    this.dayCache.set(sessionId, days)
+    return days
+  }
+
+  /** 取一行原文。命中帧缓存时零解压。 */
+  read(seq) {
+    const row = this.raw(seq)
+    if (!row || Number.isInteger(row.turn)) return row
+    // 早先落盘的 user/message 没有轮号（事件本身不带 turn）。
+    // 与其改写整条哈希链去补它，不如读的时候补：同一会话里之后最近一个有轮号的行。
+    const hint = this.#turnHints(this.sessionOf(seq)).get(seq)
+    return hint === undefined ? row : { ...row, turn: hint }
+  }
+
+  /** 原样读一行，不做任何补全。verify 要的是磁盘上的字节。 */
+  raw(seq) {
+    if (!Number.isInteger(seq)) throw new TypeError('read 需要整数 seq')
+    const sessionId = this.sessionOf(seq)
+    if (!sessionId) return null
+    const day = this.dayOf(seq)
+    if (!day) return null
+    const entry = this.#index(sessionId, day).get(seq)
+    const frame = this.#frameText(sessionId, day, entry.f)
+    return JSON.parse(frame.subarray(entry.o, entry.o + entry.l).toString('utf8'))
+  }
+
+  /** seq → 应当继承的轮号；一个会话只扫一遍。 */
+  #turnHints(sessionId) {
+    if (this.turnCache.has(sessionId)) return this.turnCache.get(sessionId)
+    const hints = new Map()
+    const info = this.state.sessions?.[sessionId]
+    if (info) {
+      let next = null
+      for (let seq = info.lastSeq; seq >= info.firstSeq; seq -= 1) {
+        const row = this.raw(seq)
+        if (!row) continue
+        if (Number.isInteger(row.turn)) next = row.turn
+        else if (next !== null) hints.set(seq, next)
+      }
+    }
+    this.turnCache.set(sessionId, hints)
+    return hints
+  }
+
+  /** 这一行实际躺在哪一天的文件里。行上的 `day` 字段按写入时刻算，换天时可能偏一格。 */
+  dayOf(seq) {
+    const sessionId = this.sessionOf(seq)
+    if (!sessionId) return null
+    return this.daysOf(sessionId).find((name) => this.#index(sessionId, name).has(seq)) ?? null
+  }
+
+  /** 取某一行带的那张图；字节在它所属那一天的媒体包里。 */
+  media(seq, sha256) {
+    const sessionId = this.sessionOf(seq)
+    if (!sessionId) return null
+    const day = this.dayOf(seq)
+    if (!day) return null
+    return readMedia(this.root, sessionId, day, sha256)
+  }
+
+  /**
+   * 一个会话的全部行（已补好轮号），按 seq 排序，解析一次就缓存。
+   * 统计与抽星都要整段原文，逐 seq read 会把同一帧反复解压。
+   */
+  rows(sessionId = null) {
+    const out = []
+    for (const info of this.sessions()) {
+      if (sessionId && info.sessionId !== sessionId) continue
+      const hints = this.#turnHints(info.sessionId)
+      for (const row of this.#rows(info.sessionId)) {
+        const hint = hints.get(row.seq)
+        out.push(hint === undefined || Number.isInteger(row.turn) ? row : { ...row, turn: hint })
+      }
+    }
+    out.sort((a, b) => a.seq - b.seq)
+    return out
+  }
+
+  /** 按时间回放；给了 session 就只回放那一个对话。 */
+  timeline({ from = 0, to = Number.MAX_SAFE_INTEGER, limit = 50, session = null } = {}) {
+    const out = []
+    for (const { sessionId } of this.sessions()) {
+      if (session && sessionId !== session) continue
+      for (const row of this.#rows(sessionId)) {
+        if (row.at < from || row.at > to) continue
+        out.push({
+          seq: row.seq,
+          at: row.at,
+          day: row.day,
+          session: sessionId,
+          kind: row.kind,
+          turn: row.turn,
+          step: row.step,
+          text: (row.text ?? row.arguments ?? '').slice(0, 160),
+        })
+      }
+    }
+    out.sort((a, b) => a.seq - b.seq)
+    return out.slice(0, limit)
+  }
+
+  /** 子串搜索。冷路径：调用方应放进 worker，别卡住宿主。 */
+  search({ text, session = null, limit = 50 } = {}) {
+    if (!text) return []
+    const hits = []
+    for (const { sessionId } of this.sessions()) {
+      if (session && sessionId !== session) continue
+      for (const row of this.#rows(sessionId)) {
+        const haystack = [row.text, row.arguments, row.reasoning, row.message].filter(Boolean).join('\n')
+        const at = haystack.indexOf(text)
+        if (at < 0) continue
+        hits.push({
+          seq: row.seq,
+          at: row.at,
+          day: row.day,
+          session: sessionId,
+          kind: row.kind,
+          snippet: haystack.slice(Math.max(0, at - 40), at + text.length + 40),
+        })
+        if (hits.length >= limit) return hits
+      }
+    }
+    return hits
+  }
+
+  /** 沿哈希链校验。返回问题清单，空数组即完好。 */
+  verify({ from = 0, to = Number.MAX_SAFE_INTEGER } = {}) {
+    const rows = []
+    for (const { sessionId } of this.sessions()) rows.push(...this.#rows(sessionId))
+    rows.sort((a, b) => a.seq - b.seq)
+    const problems = []
+    let prev = GENESIS
+    let expectSeq = rows.length ? rows[0].seq : 0
+    for (const row of rows) {
+      if (row.seq !== expectSeq) problems.push({ seq: row.seq, reason: `seq 不连续：期望 ${expectSeq}` })
+      expectSeq = row.seq + 1
+      const reason = checkRow(row, prev)
+      if (reason && row.seq >= from && row.seq <= to) problems.push({ seq: row.seq, reason })
+      prev = row.hash
+    }
+    return problems
+  }
+
+  // ---- 内部 ----
+
+  #index(sessionId, day) {
+    const key = `${sessionId}|${day}`
+    if (this.idxCache.has(key)) return this.idxCache.get(key)
+    const index = new Map()
+    for (const line of readLines(idxFile(this.root, sessionId, day))) {
+      const entry = JSON.parse(line)
+      index.set(entry.s, entry)
+    }
+    this.idxCache.set(key, index)
+    return index
+  }
+
+  #frameText(sessionId, day, frameNo) {
+    const key = `${sessionId}|${day}|${frameNo}`
+    const hit = this.frameCache.get(key)
+    if (hit) {
+      // 命中就挪到队尾，LRU 才有意义。
+      this.frameCache.delete(key)
+      this.frameCache.set(key, hit)
+      return hit
+    }
+    const meta = readLines(framesFile(this.root, sessionId, day))
+      .map((line) => JSON.parse(line))
+      .find((item) => item.n === frameNo)
+    if (!meta) throw new Error(`帧 ${frameNo} 不在索引里：${sessionId}/${day}`)
+    const buffer = Buffer.from(decompressFrame(readRange(dayFile(this.root, sessionId, day), meta.at, meta.len)), 'utf8')
+    this.frameCache.set(key, buffer)
+    this.frameBytes += buffer.length
+    while (this.frameCache.size > this.cacheLimit || this.frameBytes > this.cacheBytes) {
+      const oldest = this.frameCache.keys().next().value
+      if (oldest === key) break
+      const dropped = this.frameCache.get(oldest)
+      this.frameCache.delete(oldest)
+      this.frameBytes -= dropped?.length ?? 0
+    }
+    return buffer
+  }
+
+  #rows(sessionId) {
+    if (this.rowCache.has(sessionId)) return this.rowCache.get(sessionId)
+    const rows = []
+    for (const day of this.daysOf(sessionId)) {
+      const file = dayFile(this.root, sessionId, day)
+      for (const frame of readLines(framesFile(this.root, sessionId, day)).map((line) => JSON.parse(line))) {
+        for (const line of decompressFrame(readRange(file, frame.at, frame.len)).split('\n')) {
+          if (line) rows.push(JSON.parse(line))
+        }
+      }
+    }
+    rows.sort((a, b) => a.seq - b.seq)
+    this.rowCache.set(sessionId, rows)
+    return rows
+  }
+}

package/src/rpc.mjs

@@ -0,0 +1,77 @@
+// 记忆管线的 RPC：只读，给设置页看大小曲线用。
+export const MEMORY_RPC_ENDPOINT = 'dsh-lpipe-memory'
+
+const METHODS = new Set(['memory.stats'])
+
+function failure(code, message) {
+  return { ok: false, error: { code, message, details: {} } }
+}
+
+function isRecord(value) {
+  return value !== null && typeof value === 'object' && !Array.isArray(value)
+}
+
+function clampTurns(value) {
+  const turns = Number(value)
+  if (!Number.isFinite(turns)) return 30
+  return Math.min(120, Math.max(1, Math.round(turns)))
+}
+
+export function createMemoryRpcHandler({ stats }) {
+  return async function handle(method, payload) {
+    if (!METHODS.has(method)) return failure('memory/unknown-method', `Unknown method: ${method}`)
+    try {
+      if (method === 'memory.stats') {
+        return { ok: true, value: stats({ turns: clampTurns(payload?.turns) }) }
+      }
+      return failure('memory/unknown-method', `Unknown method: ${method}`)
+    } catch (error) {
+      return failure('memory/failed', String(error?.message ?? error))
+    }
+  }
+}
+
+/** 挂在宿主自己的 /api 载体上，鉴权与来源校验由宿主先做掉。 */
+export function registerMemoryRpc(ctx, handle) {
+  const registry = ctx?.connection?.fetch
+  if (typeof registry?.register !== 'function') {
+    throw new TypeError('dsh-lpipe-memory needs the Host Connection Fetch registry')
+  }
+  return registry.register({
+    path: `/api/${MEMORY_RPC_ENDPOINT}`,
+    methods: ['POST'],
+    requestBody: 'buffered',
+    async fetch(request) {
+      if (request.method !== 'POST') return new Response('method not allowed', { status: 405 })
+      if (request.headers.get('content-type')?.split(';', 1)[0].trim().toLowerCase() !== 'application/json') {
+        return new Response('content type must be application/json', { status: 415 })
+      }
+      let message
+      try {
+        message = await request.json()
+      } catch {
+        return new Response('body is not JSON', { status: 400 })
+      }
+      const rpcId = typeof message?.rpcId === 'string' ? message.rpcId : 'invalid-request'
+      const call = message?.payload
+      if (
+        !isRecord(message) ||
+        message.type !== 'client-request' ||
+        typeof message.rpcId !== 'string' ||
+        message.method !== MEMORY_RPC_ENDPOINT ||
+        !isRecord(call) ||
+        typeof call.method !== 'string' ||
+        !Object.hasOwn(call, 'payload')
+      ) {
+        return Response.json({ type: 'server-response', rpcId, result: failure('memory/bad-request', 'Invalid request.') })
+      }
+      let result
+      try {
+        result = await handle(call.method, call.payload, request.signal)
+      } catch (error) {
+        result = failure('memory/failed', String(error?.message ?? error))
+      }
+      return Response.json({ type: 'server-response', rpcId, result })
+    },
+  })
+}

package/src/stats.mjs

@@ -0,0 +1,286 @@
+// 记忆大小的统计：按轮累计「原文」与「落盘」两条曲线，供设置页画图用。
+//
+// 原文 = 每行 JSON 的字节 + 它第一次带来的 media 字节，未压缩。
+// 落盘 = 同样这些行在磁盘上真实占的字节：帧的压缩长度按行字节摊到每一行，
+//        再加上这一天的索引表、帧表，以及 media。
+import { readdirSync, readFileSync, statSync } from 'node:fs'
+import { join } from 'node:path'
+
+import { mediaShares } from './mediapack.mjs'
+import { daysDir, framesFile, idxFile, memoryRoot, sessionsDir } from './paths.mjs'
+
+/** media 目录树加起来有多大（内容寻址，天然去重）。 */
+function directoryBytes(dir) {
+  let total = 0
+  let entries
+  try {
+    entries = readdirSync(dir, { withFileTypes: true })
+  } catch {
+    return 0
+  }
+  for (const entry of entries) {
+    const full = join(dir, entry.name)
+    if (entry.isDirectory()) total += directoryBytes(full)
+    else if (entry.isFile()) {
+      try {
+        total += statSync(full).size
+      } catch {
+        // 文件刚被挪走，跳过
+      }
+    }
+  }
+  return total
+}
+
+function fileBytes(file) {
+  try {
+    return statSync(file).size
+  } catch {
+    return 0
+  }
+}
+
+function readLines(file) {
+  try {
+    return readFileSync(file, 'utf8').split('\n').filter(Boolean)
+  } catch {
+    return []
+  }
+}
+
+/** 磁盘上到底占了多少：每天的分卷 + 帧表 + 偏移表 + media。 */
+export function diskUsage(root = memoryRoot()) {
+  let bytes = 0
+  let sessions = []
+  try {
+    sessions = readdirSync(sessionsDir(root), { withFileTypes: true })
+      .filter((entry) => entry.isDirectory())
+      .map((entry) => entry.name)
+  } catch {
+    return 0
+  }
+  for (const sessionId of sessions) {
+    bytes += directoryBytes(join(daysDir(root, sessionId)))
+  }
+  bytes += directoryBytes(join(root, 'media'))
+  return bytes
+}
+
+/**
+ * 一天的分卷里，每一行摊到多少磁盘字节。
+ *
+ * 帧是压缩单位，一行自己压出来多少没法量；所以按行 JSON 长度把帧的压缩长度分下去——
+ * 同一帧里长的行分得多。这天的 `.idx` 与 `.frames` 也按同样比例摊进每一行，
+ * 否则曲线末尾会比真实磁盘少掉几万个字节。
+ */
+function dayShares(root, sessionId, day) {
+  const shares = new Map()
+  const idx = readLines(idxFile(root, sessionId, day))
+    .map((line) => JSON.parse(line))
+    .filter((item) => Number.isFinite(item?.s))
+  if (idx.length === 0) return shares
+
+  const frames = new Map()
+  let metaBytes = fileBytes(idxFile(root, sessionId, day)) + fileBytes(framesFile(root, sessionId, day))
+  for (const line of readLines(framesFile(root, sessionId, day))) {
+    const item = JSON.parse(line)
+    if (Number.isFinite(item?.n) && Number.isFinite(item?.len)) frames.set(item.n, item.len)
+  }
+
+  const byFrame = new Map()
+  for (const item of idx) {
+    const list = byFrame.get(item.f) ?? []
+    list.push(item)
+    byFrame.set(item.f, list)
+  }
+  const totalWeight = idx.reduce((sum, item) => sum + (Number(item.l) || 1), 0) || 1
+
+  for (const [frameNo, rows] of byFrame) {
+    const weight = rows.reduce((sum, item) => sum + (Number(item.l) || 1), 0) || 1
+    const packed = frames.get(frameNo) ?? 0
+    for (const item of rows) {
+      const l = Number(item.l) || 1
+      const compressed = (packed * l) / weight
+      const meta = (metaBytes * l) / totalWeight
+      shares.set(item.s, compressed + meta)
+    }
+  }
+  return shares
+}
+
+/**
+ * 全库的「一行摊到多少磁盘字节」。
+ *
+ * 不按行上的 `day` 去找文件——那个字段是写入时按 `at − 4h` 算的，跨零点时会和
+ * 它实际躺在哪个天文件里对不上，查错了文件就摊到 0，缺口最后全堆到最后一轮上。
+ * 所以直接扫盘：把每个会话每一天的 `.idx` / `.frames` 读一遍，按 seq 建表。
+ */
+/** 全库的「一张图摊到多少磁盘字节」。 */
+function diskMediaShares(root) {
+  const shares = new Map()
+  let sessions = []
+  try {
+    sessions = readdirSync(sessionsDir(root), { withFileTypes: true })
+      .filter((entry) => entry.isDirectory())
+      .map((entry) => entry.name)
+  } catch {
+    return shares
+  }
+  for (const sessionId of sessions) {
+    let days = []
+    try {
+      days = readdirSync(daysDir(root, sessionId))
+        .filter((name) => name.endsWith('.media.zst'))
+        .map((name) => name.slice(0, -'.media.zst'.length))
+    } catch {
+      continue
+    }
+    for (const day of days) {
+      for (const [sha, bytes] of mediaShares(root, sessionId, day)) shares.set(sha, bytes)
+    }
+  }
+  return shares
+}
+
+function diskShares(root) {
+  const shares = new Map()
+  let sessions = []
+  try {
+    sessions = readdirSync(sessionsDir(root), { withFileTypes: true })
+      .filter((entry) => entry.isDirectory())
+      .map((entry) => entry.name)
+  } catch {
+    return shares
+  }
+  for (const sessionId of sessions) {
+    let days = []
+    try {
+      days = readdirSync(daysDir(root, sessionId))
+        .filter((name) => name.endsWith('.jsonl.zst'))
+        .map((name) => name.slice(0, -'.jsonl.zst'.length))
+    } catch {
+      continue
+    }
+    for (const day of days) {
+      for (const [seq, bytes] of dayShares(root, sessionId, day)) shares.set(seq, bytes)
+    }
+  }
+  return shares
+}
+
+/**
+ * 逐轮汇总：每一轮新增多少字节、累计到多少。原文与落盘两条线同起点、同刻度。
+ * 一轮的原文按这一轮写下的行（JSON 字节）加它新带来的 media 字节算；
+ * 落盘按同样的行在磁盘上的摊分字节算。
+ */
+export function computeStats(reader, { turns = 30, root = memoryRoot() } = {}) {
+  const state = reader?.state ?? {}
+  const lastSeq = Number.isFinite(state.seq) ? state.seq : -1
+  const groups = new Map()
+  const seenMedia = new Set()
+  const shares = diskShares(root)
+  const mediaShares_ = diskMediaShares(root)
+  let total = 0
+  let diskTotal = 0
+  let rows = 0
+
+  let all = []
+  try {
+    all = typeof reader.rows === 'function' ? reader.rows() : []
+  } catch {
+    all = []
+  }
+  if (all.length === 0) {
+    // 老版本 reader 没有 rows()：退回逐 seq，慢一点但不至于没有数。
+    for (let seq = 0; seq <= lastSeq; seq += 1) {
+      try {
+        const row = reader.read(seq)
+        if (row) all.push(row)
+      } catch {
+        // 坏行跳过
+      }
+    }
+  }
+
+  for (const row of all) {
+    const bytes = Buffer.byteLength(JSON.stringify(row)) + 1
+    rows += 1
+    total += bytes
+
+    const diskRow = shares.get(row.seq) ?? 0
+    diskTotal += diskRow
+
+    let mediaBytes = 0
+    let mediaDisk = 0
+    for (const item of row.media ?? []) {
+      const sha = item?.sha256
+      if (!sha || seenMedia.has(sha)) continue
+      seenMedia.add(sha)
+      mediaBytes += Number(item.bytes) || 0
+      mediaDisk += mediaShares_.get(sha) ?? 0
+    }
+    total += mediaBytes
+
+    const key = `${row.session}|${row.turn}`
+    const entry = groups.get(key) ?? {
+      session: row.session, turn: row.turn, at: row.at, added: 0, total: 0, diskAdded: 0, diskTotal: 0,
+    }
+    entry.added += bytes + mediaBytes
+    entry.diskAdded += diskRow + mediaDisk
+    entry.at = Math.max(entry.at ?? 0, row.at ?? 0)
+    groups.set(key, entry)
+  }
+
+  const series = [...groups.values()]
+  let runningTotal = 0
+  let runningDisk = 0
+  for (const entry of series) {
+    runningTotal += entry.added
+    runningDisk += entry.diskAdded
+    entry.total = runningTotal
+    entry.diskTotal = runningDisk
+  }
+
+  // state.json、meta.json、没有行引用到的 media 这些零头没落到任何一行上。
+  // 按各轮已有的磁盘增量等比摊开，而不是一股脑塞进最后一轮——否则界面上会看到
+  // 「最近一轮 +434 KB」这种假暴涨，而那一轮其实只写了几 KB。
+  const actual = diskUsage(root)
+  const gap = actual - runningDisk
+  if (gap > 0 && runningDisk > 0) {
+    const scale = 1 + gap / runningDisk
+    for (const entry of series) entry.diskAdded *= scale
+    runningDisk = 0
+    for (const entry of series) {
+      runningDisk += entry.diskAdded
+      entry.diskTotal = runningDisk
+    }
+  }
+
+  return {
+    total,
+    disk: actual,
+    rows,
+    sessions: Object.keys(state.sessions ?? {}).length,
+    generatedAt: Date.now(),
+    turns: series.slice(-Math.max(1, turns)),
+  }
+}
+
+/** 按 state.seq 缓存：库没变就不重算，设置页每几秒问一次也不心疼。 */
+export function createStatsCache({ reader, root = memoryRoot() } = {}) {
+  let cachedSeq = null
+  let cached = null
+  return {
+    get({ turns = 30 } = {}) {
+      const seq = reader?.state?.seq ?? -1
+      if (cached && cachedSeq === seq) return { ...cached, turns: cached.turns.slice(-Math.max(1, turns)) }
+      cached = computeStats(reader, { turns: 60, root })
+      cachedSeq = seq
+      return { ...cached, turns: cached.turns.slice(-Math.max(1, turns)) }
+    },
+    invalidate() {
+      cachedSeq = null
+      cached = null
+    },
+  }
+}