@mzzsfy/dsh-usage-dash 0.2.0 → 0.4.0

package/src/collector.js

@@ -5,12 +5,36 @@
 // 游标(liveFirstSeq 分区 + backfilledSessions)保证。采集是观测性的:一切
 // store 失败计入 recordFailures,绝不成为逃逸拒绝。
 
+import { createArchiveReader } from './archive-reader.js'
+
 const UNKNOWN_SESSION_ID = '(unknown-session)'
 const SEQ_UNKNOWN = -1
 const DEFAULT_BACKFILL_CONCURRENCY = 4
 const MARK_BATCH = 32
 const SCAN_LOG_MAX_ENTRIES = 200
 
+// 不可读会话分类:宿主可见性/可读性边界的三代实测形态(见 README 存档兼容)
+const SKIP_KINDS = { descriptor: 'descriptor', corrupt: 'corrupt', legacy: 'legacy', other: 'other' }
+const SKIP_PATTERNS = [
+  // 宿主新校验拒读历史档(0.1.5 descriptor v2),宿主修复后自动补扫
+  [SKIP_KINDS.descriptor, /unsupported descriptor version/i],
+  // 存档损坏(seq gap 等 fail-closed 拒绝),宿主 repair 后自动补扫
+  [SKIP_KINDS.corrupt, /corrupt|seq gap|SessionFormatError/i],
+  // 超旧格式词汇(v0 时代字段),宿主迁移链覆盖后自动补扫
+  [SKIP_KINDS.legacy, /format v0|unexpected member|does not provide|not a function/i],
+]
+
+function classifySkip(detail) {
+  for (const [kind, pattern] of SKIP_PATTERNS) {
+    if (pattern.test(detail)) return kind
+  }
+  return SKIP_KINDS.other
+}
+
+function emptySkipCounts() {
+  return { [SKIP_KINDS.descriptor]: 0, [SKIP_KINDS.corrupt]: 0, [SKIP_KINDS.legacy]: 0, [SKIP_KINDS.other]: 0 }
+}
+
 // 会话内单 pass 折叠:跟踪每个 (turn,step) 槽的最新报告,只把首次发射交 store
 export class UsageFold {
   constructor() {
@@ -95,8 +119,9 @@ function sessionIdOf(target) {
   return typeof id === 'string' && id !== '' ? id : UNKNOWN_SESSION_ID
 }
 
-// 规范 provider/model 引用:双全拼引用,仅 model 用裸名
-function refOf(route) {
+// 规范 provider/model 引用:双全拼引用,仅 model 用裸名;
+// 与 client 侧注入点B turnModelOf 双实现同源(routes[].model 是裸模型名,展示侧才拼 provider),改一侧必须同步 parity
+export function refOf(route) {
   if (!route) return undefined
   if (route.provider && route.model) return `${route.provider}/${route.model}`
   return route.model || undefined
@@ -126,6 +151,10 @@ export class UsageCollector {
       lastSessionId: undefined,
       error: undefined,
       log: [],
+      // 计数独立累加,不受日志截断影响;skipped 细分供面板归因
+      skippedTotal: 0,
+      recordTotal: 0,
+      skipCounts: emptySkipCounts(),
     }
     // error 保留给采集器自身故障;单会话读取失败走日志 skipped 条目,计数由日志派生
   }
@@ -141,17 +170,24 @@ export class UsageCollector {
       scannedSessions: this.#state.scannedSessions,
       lastSessionId: this.#state.lastSessionId,
       error: this.#state.error,
-      skippedSessions: log.filter((entry) => entry.kind === 'skipped').length,
-      recordFailures: log.filter((entry) => entry.kind === 'record').length,
+      skippedSessions: this.#state.skippedTotal,
+      recordFailures: this.#state.recordTotal,
+      skippedBreakdown: { ...this.#state.skipCounts },
       log,
     }
   }
 
-  // 扫描异常日志:供面板明细展示,超上限丢最旧
+  // 扫描异常日志:供面板明细展示,超上限丢最旧;计数独立累加不随截断漂移
   pushLog(kind, detail) {
     const log = this.#state.log
     log.push({ time: Date.now(), kind, detail })
     if (log.length > SCAN_LOG_MAX_ENTRIES) log.splice(0, log.length - SCAN_LOG_MAX_ENTRIES)
+    if (kind === 'skipped') {
+      this.#state.skippedTotal += 1
+      this.#state.skipCounts[classifySkip(detail)] += 1
+    } else if (kind === 'record') {
+      this.#state.recordTotal += 1
+    }
   }
 
   get running() {
@@ -239,7 +275,9 @@ export class UsageCollector {
     const controller = new AbortController()
     this.scanController = controller
     await this.store.readyPromise()
-    await this.backfill(this.ctx.sessionPersistence, this.ctx.sessions, controller.signal)
+    // 适配发生在宿主服务边界:回扫主体只面向 ArchiveReader 内部契约
+    const reader = createArchiveReader(this.ctx.sessionPersistence)
+    await this.backfill(reader, this.ctx.sessions, controller.signal)
   }
 
   abort() {
@@ -271,22 +309,30 @@ export class UsageCollector {
     return this.resetInFlight
   }
 
-  // 回扫:persistence.list 驱动,逐会话全新 fold 重放;seen 独自决定是否重扫,
-  // liveFirstSeq 边界划走实时已拥区间,liveness 复查防陈旧快照放大重放范围,
-  // inheritedCut 跳过 fork 继承前缀。只有干净重放完的会话进游标,失败下轮重试
-  async backfill(persistence, sessions, signal) {
+  // 回扫:ArchiveReader 契约驱动(宿主 persistence 多版本适配见 archive-reader),
+  // 逐会话全新 fold 重放;seen 独自决定是否重扫,liveFirstSeq 边界划走实时
+  // 已拥区间,liveness 复查防陈旧快照放大重放范围,inheritedCut 跳过 fork
+  // 继承前缀。只有干净重放完的会话进游标,失败下轮重试
+  async backfill(reader, sessions, signal) {
     if (this.#state.running) return
     if (signal?.aborted) return
     this.#state.running = true
     this.#state.error = undefined
     try {
-      const headers = await persistence.list(signal)
+      const headers = await reader.list(signal).catch((error) => {
+        // 枚举失败(含宿主 API 未识别)按采集器自身故障呈现,面板可见
+        this.#state.error = error instanceof Error ? error.message : String(error)
+        throw error
+      })
       const seen = await this.store.seenSessions()
       const liveSeq = await this.store.liveSequences()
       const targets = headers.filter((header) => !seen.has(header.id))
       this.#state.total = targets.length
       this.#state.done = 0
       this.#state.log = []
+      this.#state.skippedTotal = 0
+      this.#state.recordTotal = 0
+      this.#state.skipCounts = emptySkipCounts()
       const workerCount = Math.min(DEFAULT_BACKFILL_CONCURRENCY, Math.max(1, targets.length))
       let next = 0
       const completed = []
@@ -321,7 +367,7 @@ export class UsageCollector {
           const fold = new UsageFold()
           let route = ''
           try {
-            const inspection = await persistence.inspect(header.id, signal)
+            const inspection = await reader.readLog(header.id, signal)
             const inheritedCut = inspection.inheritedEventCount
             for (const event of inspection.events) {
               if (signal?.aborted) return

package/src/direct-log-reader.js

@@ -0,0 +1,126 @@
+// 会话档案文件直读器:宿主 persistence 服务 fail-closed 拒读(descriptor 校验、
+// 格式校验、seq gap 等)时,对磁盘档案做降级直读。统计只消费 usage 词汇
+// (assistant/message 的 usage 与 request/context 路由),无需完整会话语义,
+// 宽松解析即可恢复;corrupt(seq gap)与 legacy(未知成员)对统计无影响。
+//
+// 物理格式(与宿主 jsonl 后端同构):首行 header JSON + 事件行 JSONL 文本,
+// 按 zstd 帧独立压缩顺序拼接;帧以 magic 28 b5 2f fd 起始。压缩由 Node 内置
+// node:zlib 提供(Node >= 24),不依赖宿主模块,不受宿主升级影响。
+
+import { existsSync, readdirSync, readFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { homedir } from 'node:os'
+import { zstdDecompressSync } from 'node:zlib'
+
+const ZSTD_MAGIC = Buffer.from([0x28, 0xb5, 0x2f, 0xfd])
+
+// 档案根目录:DSH_HOME 可重定向(多版本共存测试的隔离约定),默认 ~/.dsh
+function sessionsRoot() {
+  const home = process.env.DSH_HOME || join(homedir(), '.dsh')
+  return join(home, 'sessions')
+}
+
+// 按帧边界切分 zstd 流:帧首 magic 定位,各帧解压范围到下一 magic 或流尾
+function frameRanges(bytes) {
+  const ranges = []
+  let at = bytes.indexOf(ZSTD_MAGIC)
+  while (at >= 0) {
+    if (ranges.length > 0) ranges[ranges.length - 1].end = at
+    ranges.push({ start: at, end: bytes.length })
+    at = bytes.indexOf(ZSTD_MAGIC, at + ZSTD_MAGIC.length)
+  }
+  return ranges
+}
+
+// 多帧解压;任何一帧失败即整体失败(宁缺勿错,调用方按原拒读路径上报)
+export function decodeZstdFile(bytes) {
+  const ranges = frameRanges(bytes)
+  if (ranges.length === 0) throw new Error('no zstd frame found')
+  let text = ''
+  for (const range of ranges) {
+    text += zstdDecompressSync(bytes.subarray(range.start, range.end)).toString('utf8')
+  }
+  return text
+}
+
+// JSONL 文本 → 宽松事件流:首行为档案 header,其余行逐行解析,坏行跳过;
+// 只要求行能解出 JSON 对象,词汇/代际差异由上层折叠消化
+export function parseJsonlEvents(text) {
+  const events = []
+  for (const line of text.split('\n')) {
+    const trimmed = line.trim()
+    if (trimmed === '') continue
+    try {
+      const row = JSON.parse(trimmed)
+      if (row && typeof row === 'object' && row.type !== undefined && row.type !== 'session/header') {
+        events.push(row)
+      }
+    } catch {}
+  }
+  return events
+}
+
+// 会话目录定位:sessions/<cwd 编码桶>/<id>/;桶名是宿主对 cwd 的编码,不解析,
+// 逐桶探测 id 子目录
+export function findSessionDir(id, root = sessionsRoot()) {
+  const sessions = root
+  if (!existsSync(sessions)) return undefined
+  for (const bucket of readdirSync(sessions)) {
+    const candidate = join(sessions, bucket, id)
+    if (existsSync(candidate) && statDirSync(candidate)) return candidate
+  }
+  return undefined
+}
+
+function statDirSync(path) {
+  try {
+    return readdirSync(path).length >= 0
+  } catch {
+    return false
+  }
+}
+
+// 会话目录内选最新代档案:V3 专用名优先,退化为任意 jsonl(压缩/明文)
+function pickLogFile(dir) {
+  const names = readdirSync(dir)
+  const v3 = names.find((name) => name.startsWith('session.v3.') && name.endsWith('.zstd'))
+  if (v3) return join(dir, v3)
+  const compressed = names.find((name) => name.endsWith('.jsonl.zstd'))
+  if (compressed) return join(dir, compressed)
+  const plain = names.find((name) => name.endsWith('.jsonl'))
+  if (plain) return join(dir, plain)
+  return undefined
+}
+
+// 直读单会话事件流;文件缺失/无法解压抛错,由调用方决定降级链终止
+export function readSessionLogDirect(id, root) {
+  const dir = findSessionDir(id, root)
+  if (dir === undefined) throw new Error(`session artifact directory not found: ${id}`)
+  const file = pickLogFile(dir)
+  if (file === undefined) throw new Error(`session artifact file not found: ${id}`)
+  const bytes = readFileSync(file)
+  const text = file.endsWith('.zstd') ? decodeZstdFile(bytes) : bytes.toString('utf8')
+  return parseJsonlEvents(text)
+}
+
+// 磁盘全量会话 id:sessions/<cwd 编码桶>/<id>/;旧代宿主的 list 不枚举新代
+// 文件名的档案,直读侧补齐可见集
+export function listSessionIdsDirect(root = sessionsRoot()) {
+  if (!existsSync(root)) return []
+  const ids = []
+  for (const bucket of readdirSync(root)) {
+    const bucketPath = join(root, bucket)
+    let entries
+    try {
+      entries = readdirSync(bucketPath)
+    } catch {
+      continue
+    }
+    for (const id of entries) {
+      if (existsSync(join(bucketPath, id, 'session.jsonl.zstd')) || existsSync(join(bucketPath, id, 'session.v3.jsonl.zstd'))) {
+        ids.push(id)
+      }
+    }
+  }
+  return ids
+}

package/src/pricing.js

@@ -30,15 +30,14 @@ const toMinutesOfDay = (hhmm) => {
   return Number.isFinite(h) && Number.isFinite(m) ? h * MINUTES_PER_HOUR + m : Number.NaN
 }
 
-// from<to 含头不含尾;from>to 跨午夜;from===to 全天生效
+// 所有范围条件统一双侧包含;from>to 跨午夜/跨月环绕;from===to 单点/单日
 const dailyWindowMatches = (condition, date) => {
   const from = toMinutesOfDay(condition.from)
   const to = toMinutesOfDay(condition.to)
   if (Number.isNaN(from) || Number.isNaN(to)) return false
   const m = minutesOfDay(date)
-  if (from < to) return m >= from && m < to
-  if (from > to) return m >= from || m < to
-  return true
+  if (from <= to) return m >= from && m <= to
+  return m >= from || m <= to
 }
 
 // days 空数组不成立;0=周日,取 getDay()
@@ -47,7 +46,7 @@ const weekdaysMatches = (condition, date) => {
   return Array.isArray(days) && days.length > 0 && days.includes(date.getDay())
 }
 
-// 号段双闭;from>to 跨月环绕(如 26~25 账单周期);日号必须整数,2 月无 31 号自然不触发
+// 号段双侧包含;from>to 跨月环绕(账单周期);from===to 单日,2 月无 31 号自然不触发
 const monthDaysMatches = (condition, date) => {
   const { from, to } = condition
   if (!Number.isInteger(from) || !Number.isInteger(to)) return false
@@ -55,7 +54,7 @@ const monthDaysMatches = (condition, date) => {
   return from <= to ? d >= from && d <= to : d >= from || d <= to
 }
 
-// 要求零填充 YYYY-MM-DD 字典序双闭;from>to 属配置错误不成立,非规范串同样不成立
+// 零填充 YYYY-MM-DD 字典序双侧包含;from>to 配置错误不成立(编辑器校验拦截)
 const dateRangeMatches = (condition, date) => {
   const { from, to } = condition
   if (typeof from !== 'string' || typeof to !== 'string') return false

package/src/query.js

@@ -105,6 +105,8 @@ export function aggregateRange(rows, g, from, to) {
   const modelTotals = new Map()
   const providerTotals = new Map()
   const activeBuckets = new Set()
+  // 槽级速度配对:桶串 → {outputTokens, durationMs},与模型级同口径(仅带时长行)
+  const slotSpeeds = new Map()
   for (const row of rows) {
     const slot = slotByKey.get(row.bucket)
     // 桶串未落在枚举序列(如改粒度前的历史残行)不可归属,跳过防崩
@@ -115,6 +117,12 @@ export function aggregateRange(rows, g, from, to) {
     activeBuckets.add(row.bucket)
     slot.byModel[row.model] = (slot.byModel[row.model] ?? 0) + tokens
     slot.byProvider[row.provider] = (slot.byProvider[row.provider] ?? 0) + tokens
+    if (row.durationMs) {
+      const pair = slotSpeeds.get(row.bucket) ?? { outputTokens: 0, durationMs: 0 }
+      pair.outputTokens += row.outputTokens
+      pair.durationMs += row.durationMs
+      slotSpeeds.set(row.bucket, pair)
+    }
     const modelTotal = modelTotals.get(row.model)
     if (modelTotal) {
       modelTotal.tokens += tokens
@@ -130,6 +138,11 @@ export function aggregateRange(rows, g, from, to) {
     }
     providerTotals.set(row.provider, (providerTotals.get(row.provider) ?? 0) + tokens)
   }
+  // 槽级 speed 条件挂:无时长数据的槽不挂字段(存量槽形契约不变)
+  for (const slot of slots) {
+    const pair = slotSpeeds.get(slot.day)
+    if (pair && pair.durationMs > 0) slot.speed = pair.outputTokens / (pair.durationMs / MS_PER_SECOND)
+  }
   const totals = { tokens: 0, requests: 0, turns: 0, cacheHit: 0, cacheMiss: 0 }
   for (const slot of slots) {
     totals.tokens += slot.total

package/src/routes.js

@@ -274,7 +274,10 @@ const pricingHandler = (deps) => async (req, res) => {
 }
 
 const statusHandler = (deps) => async (req, res) => {
-  writeJson(res, { ok: true, value: deps.collector.status() })
+  writeJson(res, {
+    ok: true,
+    value: { ...deps.collector.status(), backup: deps.store.backupInfo() },
+  })
 }
 
 // 409 只拒 boot 期扫描;reset 自身引发的重扫同样 running 为真,经 rebuilding 放行
@@ -286,6 +289,15 @@ const resetHandler = (deps) => async (req, res) => {
   writeJson(res, { ok: true, value: deps.collector.status() })
 }
 
+// 回退到最近一次重建(或恢复)前的快照;扫描进行中同样拒绝,避免写交错
+const restoreHandler = (deps) => async (req, res) => {
+  if (deps.collector.running) {
+    throw usageError(HTTP_STATUS_CONFLICT, MESSAGE_BACKFILL_RUNNING)
+  }
+  const result = await deps.store.restoreFromBackup()
+  writeJson(res, { ok: true, value: result })
+}
+
 const STANDARD_GUARDS = [rejectWrongMethod, rejectCrossOrigin, rejectNonJson]
 const PRICING_GUARDS = [rejectPricingMethod, rejectCrossOrigin]
 
@@ -295,6 +307,7 @@ const ENDPOINTS = [
   { path: `${ROUTE_PREFIX}/minutes`, mount: minuteHandler },
   { path: `${ROUTE_PREFIX}/status`, mount: statusHandler },
   { path: `${ROUTE_PREFIX}/reset`, mount: resetHandler },
+  { path: `${ROUTE_PREFIX}/restore`, mount: restoreHandler },
   { path: `${ROUTE_PREFIX}/pricing`, mount: pricingHandler, guards: PRICING_GUARDS },
 ]
 

package/src/store.js

@@ -104,6 +104,24 @@ export const usageStatsDomain = defineDomain({
   },
 })
 
+// 重建回退备份域:与主域同构表,global 多记 takenAt(快照时刻)。
+// 独立域而非主域内备份表——reset 清主域数据,同域备份会陪葬
+export const usageBackupDomain = defineDomain({
+  name: `${DOMAIN_NAME}_backup`,
+  version: 1,
+  tables: {
+    [TABLE_BUCKETS]: domainTable(usageRowSchema),
+  },
+  global: {
+    schema: z.object({
+      backfilledSessions: z.array(z.string()),
+      liveFirstSeq: z.record(z.string(), z.number()).optional(),
+      takenAt: z.number().optional(),
+    }),
+    initial: { backfilledSessions: [], liveFirstSeq: {}, takenAt: undefined },
+  },
+})
+
 export function rowKey(g, bucket, provider, model) {
   return `${g}|${bucket}|${provider}|${model}`
 }
@@ -171,6 +189,8 @@ export class UsageStore {
     this.onFlushError = options.onFlushError
     this.table = null
     this.domain = null
+    this.backupDomain = undefined
+    this.backupTable = undefined
     this.openError = undefined
     this.lastPruneDay = ''
     this.markChain = Promise.resolve()
@@ -191,6 +211,9 @@ export class UsageStore {
     if ((cursor?.backfilledSessions?.length ?? 0) === 0 && !this.table.keys().next().done) {
       for (const key of [...this.table.keys()]) await this.table.delete(key)
     }
+    // 备份域打开失败仅禁用回退能力,不拖垮主采集(openError 只覆盖主域)
+    this.backupDomain = await facility.open(usageBackupDomain).catch(() => undefined)
+    this.backupTable = this.backupDomain?.table(TABLE_BUCKETS)
   }
 
   get degradation() {
@@ -375,6 +398,9 @@ export class UsageStore {
   reset(boundaries) {
     return this.enqueueGlobalWrite(async () => {
       await this.flushBatch()
+      // 清空前快照到备份域:重建丢数据(规则缺陷/中断/口径变化)可整体回退;
+      // 备份不可用视为 reset 失败——回退安全带是重建的前置条件
+      await this.snapshotToBackup()
       const table = this.requireTable()
       for (const key of [...table.keys()]) await table.delete(key)
       const liveFirstSeq = {}
@@ -383,11 +409,53 @@ export class UsageStore {
     })
   }
 
-  // global 只有整值覆写:全部游标写挂同一链,读改写不再交错
+  // 当前全量(表行 + 游标 + 时刻)写入备份域,覆盖旧备份:恢复点恒为最近一次清空/恢复前;
+  // 备份域不可用时跳过(重建不被卡死,仅失去回退点,backupInfo 透明展示)
+  async snapshotToBackup() {
+    if (this.backupTable === undefined) return
+    const cursor = this.readCursor() ?? {}
+    for (const key of [...this.backupTable.keys()]) await this.backupTable.delete(key)
+    for (const [key, row] of this.requireTable().entries()) await this.backupTable.put(key, row)
+    await this.backupDomain.global.set({
+      backfilledSessions: cursor.backfilledSessions ?? [],
+      liveFirstSeq: cursor.liveFirstSeq ?? {},
+      takenAt: this.now(),
+    })
+  }
+
+  // 备份写回主域;当前态先入备份(恢复动作自身可回退),故必须先取备份内容
+  // 再覆盖备份域;返回快照时刻与回写行数
+  async restoreFromBackup() {
+    if (this.backupTable === undefined) throw new Error('usage backup domain unavailable')
+    return this.enqueueGlobalWrite(async () => {
+      await this.flushBatch()
+      const savedRows = [...this.backupTable.entries()]
+      const saved = this.backupDomain.global.get()
+      await this.snapshotToBackup()
+      for (const key of [...this.requireTable().keys()]) await this.table.delete(key)
+      for (const [key, row] of savedRows) await this.table.put(key, row)
+      await this.writeGlobal(saved?.backfilledSessions ?? [], saved?.liveFirstSeq ?? {})
+      return { takenAt: saved?.takenAt, rows: savedRows.length }
+    })
+  }
+
+  // 备份元信息:available=false 表示无回退点(从未重建/备份域不可用)
+  backupInfo() {
+    if (this.backupTable === undefined) return { available: false }
+    const saved = this.backupDomain.global.get()
+    return {
+      available: typeof saved?.takenAt === 'number',
+      takenAt: saved?.takenAt,
+      rows: [...this.backupTable.keys()].length,
+      sessions: saved?.backfilledSessions?.length ?? 0,
+    }
+  }
+
+  // global 只有整值覆写:全部游标写挂同一链,读改写不再交错;write 返回值透传
   enqueueGlobalWrite(write) {
     const pending = this.markChain.then(async () => {
       await this.ready
-      await write()
+      return write()
     })
     this.markChain = pending.then(() => {}, () => {})
     return pending