dsh-cost-meter 1.9.49 → 1.9.50

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,32 @@
1
+ # v1.9.50 — billion-context 输入误计与历史账本纠正
2
+
3
+ ## 中文
4
+
5
+ 本次修复 [#458](https://github.com/Han-1413141/dsh-cost-meter/issues/458) 报告的费用虚高问题。感谢 [@191xhq](https://github.com/191xhq) 提供账本与会话回放对照、503 条异常记录的统计、缓存判断和替换事件的定位,以及过滤建议。
6
+
7
+ billion-context 可以在本地返回压缩结果,同时生成仅含输入 Token 的上下文估计值。旧版在没有观察到模型调用时,也会把这个值作为一次调用收费;已经保存的 `host` 判断会在重启和历史读取时继续沿用。
8
+
9
+ - 没有模型调用记录、只有输入估计值的样本不再入账。缓存判断、实时导入、归档读取和历史明细使用同一检查。
10
+ - 不设置 100 万 Token 上限。有真实调用记录的大上下文请求和有效的宿主用量继续计费。
11
+ - 无法用于计费的估计值只排除同一次调用中数值完全相同的宿主样本,不再把该步骤的其他真实用量一起隐藏。
12
+ - 启动时自动纠正有入账凭据的历史误计记录,更新会话、日期、模型、套餐小时统计及冻结的币种金额,沿用原价格与汇率。纠正与去重标记在同一次账本写入中保存。
13
+
14
+ ### 更新与使用
15
+
16
+ 更新到 **1.9.50**,完全退出并重新启动 DSH。启动后的历史导入会自动执行纠正,无需点击“按当前价格重算历史”。如果有多个 DSH 宿主实例,均更新并重启。
17
+
18
+ 纠正前会在账本旁保存 `ledger.json.before-issue-458-<ID>.json`,插件日志会记录纠正条数、移除的输入 Token 和备份位置。通常账本位于 `$DSH_HOME/storages/cost-meter/ledger.json`;未配置 `DSH_HOME` 时位于 `~/.dsh/storages/cost-meter/ledger.json`。
19
+
20
+ 如果记录缺少入账凭据,或账本已被手动修改导致数量不符,日志会列出未处理条数,原记录保留。需要进一步处理时,提供这条日志及脱敏后的相关记录;不要公开整个账本或凭据。备份失败时不会修改账本,下次启动重试。
21
+
22
+ 本次回归覆盖旧缓存、归档、实时流、真实大上下文、正常会话、历史价格与汇率、套餐统计、重复启动、并发宿主写入、备份失败和清空历史。
23
+
24
+ ## English
25
+
26
+ Fixes [#458](https://github.com/Han-1413141/dsh-cost-meter/issues/458). Thanks to [@191xhq](https://github.com/191xhq) for the ledger/replay comparison, affected-record statistics, root-cause analysis and filtering proposal.
27
+
28
+ The bridge no longer bills input-only context estimates when no model call was observed. Cached decisions, live imports, archived metadata and history replay use the same guard. Real model calls are not subject to a token ceiling, and unrelated host usage remains visible.
29
+
30
+ After updating to **1.9.50**, fully restart every DSH host. Startup repairs previously admitted extra host charges using their durable billing receipts and historical prices/FX. It updates session, daily, model and plan-hour totals, writes a backup beside the ledger, and records the correction atomically so it cannot run twice. A current-price recalculation is unnecessary.
31
+
32
+ The log reports the correction count and `ledger.json.before-issue-458-<ID>.json` backup path. Records without matching receipts or consistent ledger totals remain unchanged and are reported for inspection. A failed backup leaves the ledger untouched and the correction retryable.
@@ -10,7 +10,7 @@ import { createBillionInventory } from './billion-context-inventory.js'
10
10
  import { createBillionPeerLookup } from './billion-context-peer.js'
11
11
  import { maintainBillionStorage, readBillionStreamArchive } from './billion-context-storage.js'
12
12
  import { billionRoot, billionSessionKey, billionStreamPath, writeBillionJson, readBillionJson, listBillionFiles, iterateBillionCalls, readBillionStreamAsync,
13
- readBillionStreamFileAsync, billionProvider, usageOnly, usageBuckets, usageSignature, validUsage, validSessionId, billionCoverage, billionCoverageIndex, billionHostMeter } from './billion-context-history.js'
13
+ readBillionStreamFileAsync, billionProvider, usageOnly, usageBuckets, usageSignature, validUsage, validSessionId, billionCoverage, billionCoverageIndex, billionHostMeter, checkedBillionCoverage } from './billion-context-history.js'
14
14
 
15
15
  const bridgeKey = Symbol.for('dsh-cost-meter.billion-context.v1'), bridges = new WeakMap()
16
16
  const plainId = value => typeof value === 'string' ? value.slice(0, 200) : ''
@@ -36,6 +36,7 @@ export function installBillionContextBilling(ctx, ledger) {
36
36
  detected: false, version: '', compatibility: 'unknown', noticeId: '', unresolved: 0 }
37
37
  const warned = new Set()
38
38
  const applyCoverage = (stream, coverage) => {
39
+ coverage = checkedBillionCoverage(stream, coverage)
39
40
  stream.coverage = coverage
40
41
  if (coverage.mode !== 'proxy') { stream.covered = false; stream.hostSuppressed = true }
41
42
  if (coverage.mode === 'unresolved' && !warned.has(stream.streamId)) {
@@ -43,7 +44,7 @@ export function installBillionContextBilling(ctx, ledger) {
43
44
  } else if (coverage.mode !== 'unresolved' && warned.delete(stream.streamId)) status.unresolved--
44
45
  }
45
46
  const importHost = stream => {
46
- if (stream.coverage?.mode !== 'host' || !validUsage(stream.reported)) return
47
+ if (checkedBillionCoverage(stream, stream.coverage)?.mode !== 'host' || !validUsage(stream.reported)) return
47
48
  const meter = billionHostMeter(stream)
48
49
  if (stream.startedAtMs <= (ledger.historyReset?.at ?? 0)) ledger.advanceUsageCursor(meter)
49
50
  else ledger.account(usageBuckets(stream.reported), stream.model, stream.sessionId, stream.startedAtMs, stream.provider, meter)
@@ -136,6 +136,19 @@ function observeCoverage(counts, stream, row) {
136
136
  if (row.usage === null) counts.unknown++
137
137
  else { counts.known++; if (row.model === stream.model && usageSignature(row.usage) === usageSignature(stream.reported)) counts.matched = true }
138
138
  }
139
+ // The proxy can answer compaction locally and forge input-only usage from the
140
+ // uncompressed context size. A receipt with zero model calls is not a bill.
141
+ // Do not impose a token ceiling: real model calls can exceed one million tokens.
142
+ export function checkedBillionCoverage(stream, coverage) {
143
+ const usage = stream.reported
144
+ if (coverage?.mode === 'host' && validUsage(usage) && usage.inputTokens > 0
145
+ && tokenKeys.slice(1).every(key => (usage[key] ?? 0) === 0)
146
+ && coverage.known === 0 && coverage.unknown === 0) {
147
+ return { ...coverage, mode: 'unresolved', reason: 'input-only-without-model-call' }
148
+ }
149
+ return coverage
150
+ }
151
+ export const unverifiedBillionHost = coverage => coverage?.reason === 'input-only-without-model-call'
139
152
  function coverageResult(stream, { known, unknown, matched }) {
140
153
  // One host sample belongs to one stream, never to each unknown attempt.
141
154
  // Mixed aggregate/attempt usage cannot be separated without more evidence.
@@ -143,11 +156,11 @@ function coverageResult(stream, { known, unknown, matched }) {
143
156
  const eligible = stream.hostEligible === true || stream.covered === true
144
157
  const mode = !eligible ? 'proxy' : !known && validUsage(stream.reported) ? 'host'
145
158
  : unknown && !matched ? 'unresolved' : 'proxy'
146
- return { version: 1, mode, known, unknown }
159
+ return checkedBillionCoverage(stream, { version: 1, mode, known, unknown })
147
160
  }
148
161
  /** A persistence receipt says nothing about whether model usage was observed. */
149
162
  export async function billionCoverage(ledgerPath, stream, offsets) {
150
- if (stream.coverage?.version === 1) return stream.coverage
163
+ if (stream.coverage?.version === 1) return checkedBillionCoverage(stream, stream.coverage)
151
164
  const counts = coverageCounts()
152
165
  for (const file of listBillionFiles(ledgerPath, stream.sessionId)) for await (const { row } of iterateBillionCalls(file, offsets?.get(file.path)?.offset ?? 0)) {
153
166
  if (row.streamId === stream.streamId) observeCoverage(counts, stream, row)
@@ -166,10 +179,11 @@ export async function billionCoverageIndex(ledgerPath, sessionId) {
166
179
  if (stream?.finished && !stream.coverage) observeCoverage(counts, stream, row)
167
180
  }
168
181
  return new Map([...entries].filter(([, value]) => value.stream?.finished)
169
- .map(([id, { stream, counts }]) => [id, stream.coverage ?? coverageResult(stream, counts)]))
182
+ .map(([id, { stream, counts }]) => [id, checkedBillionCoverage(stream, stream.coverage ?? coverageResult(stream, counts))]))
170
183
  }
171
184
  export const billionHostMeter = stream => ({ source: stream.streamId + ':' + billionSessionKey('host:' + stream.sessionId), seq: 1 })
172
- export function billionHostRecord(stream) {
185
+ export function billionHostRecord(stream, coverage = stream.coverage ?? { mode: 'host', known: 0, unknown: 0 }) {
186
+ if (!validUsage(stream.reported) || unverifiedBillionHost(checkedBillionCoverage(stream, coverage))) return null
173
187
  return { type: BILLION_USAGE_EVENT, time: stream.startedAtMs, data: { provider: stream.provider, model: stream.model, usage: stream.reported,
174
188
  startedAtMs: stream.startedAtMs, requestId: stream.streamId, kind: 'model', turn: stream.turn ?? null, step: stream.step ?? null,
175
189
  metering: { source: 'billion-context', requestId: stream.streamId, attempt: 1, complete: false, status: 'complete', upstream: 'host-usage' } } }
@@ -201,10 +215,12 @@ export async function billionRecordsFor(ledgerPath, sessionId) {
201
215
  if (records.length > 200000) throw Error('Proxy billing detail limit exceeded')
202
216
  }
203
217
  for (const stream of streams.values()) {
204
- const coverage = stream.coverage ?? coverageResult(stream, counts.get(stream.streamId) ?? coverageCounts())
205
- if (coverage.mode === 'host') records.push(billionHostRecord(stream))
218
+ const coverage = checkedBillionCoverage(stream, stream.coverage ?? coverageResult(stream, counts.get(stream.streamId) ?? coverageCounts()))
219
+ const host = coverage.mode === 'host' ? billionHostRecord(stream, coverage) : null
220
+ if (host) records.push(host)
206
221
  if ((stream.covered || stream.hostSuppressed || coverage.mode === 'host' || coverage.mode === 'unresolved') && validUsage(stream.reported)) records.push({ type: BILLION_REPLACEMENT_EVENT, data: {
207
222
  streamId: stream.streamId, sourceSeqs: stream.sourceSeqs ?? [], turn: stream.turn ?? null, step: stream.step ?? null, usage: stream.reported,
223
+ ...(unverifiedBillionHost(coverage) ? { exactUsageOnly: true } : {}),
208
224
  provider: stream.provider, model: stream.model } })
209
225
  }
210
226
  return records
@@ -216,8 +232,9 @@ export function excludeBillionReportedUsage(records) {
216
232
  const usage = row.type === 'assistant/message' || row.type === 'compaction/summary' ? row.data?.usage
217
233
  : row.type === 'assistant/chunk' && row.data?.chunk?.type === 'usage' ? row.data.chunk.usage : null
218
234
  if (!usage) return true
219
- return !replacements.some(replacement => replacement.sourceSeqs.includes(row.seq) || (replacement.step != null && replacement.turn != null
235
+ return !replacements.some(replacement => (!replacement.exactUsageOnly || usageSignature(usage) === usageSignature(replacement.usage))
236
+ && (replacement.sourceSeqs.includes(row.seq) || (replacement.step != null && replacement.turn != null
220
237
  && row.data?.turn === replacement.turn && row.data?.step === replacement.step && (row.type === 'assistant/chunk' || usageSignature(usage) === usageSignature(replacement.usage))
221
- && (!row.data?.message?.source?.model || row.data.message.source.model === replacement.model)))
238
+ && (!row.data?.message?.source?.model || row.data.message.source.model === replacement.model))))
222
239
  })
223
240
  }
@@ -0,0 +1,121 @@
1
+ /** Remove only receipted input-only host charges with no observed model call (#458). */
2
+ import { randomUUID } from 'node:crypto'
3
+ import { lstat } from 'node:fs/promises'
4
+ import { createBillionInventory } from './billion-context-inventory.js'
5
+ import { readBillionStreamArchive } from './billion-context-storage.js'
6
+ import { readBillionStreamFileAsync, billionCoverage, checkedBillionCoverage, unverifiedBillionHost,
7
+ billionHostMeter, BILLION_USAGE_EVENT } from './billion-context-history.js'
8
+ import { replaySessionRecords } from './backfill.js'
9
+ import { bindPricingHistory, encodePricingHistory } from './billing-history.js'
10
+ import { withLedgerLock, writeLedger, writeDurableJson, cloneLedgerSnapshot } from './ledger-persistence.js'
11
+ import { FX_FIELDS } from './exchange-rates.js'
12
+ import { planProviderIdOf, isRoutedThirdPartyCall } from './plan-billing.js'
13
+
14
+ const fields = ['input', 'output', 'cacheRead', 'cacheWrite', 'reasoning', 'calls', 'cost', 'apiCost', ...FX_FIELDS]
15
+ const finite = value => typeof value === 'number' && Number.isFinite(value) && value >= 0
16
+ const amount = (row, key) => key === 'apiCost' ? row?.apiCost ?? row?.cost ?? 0 : row?.[key] ?? 0
17
+ const contains = (row, delta) => row && fields.every(key => finite(amount(row, key)) && finite(delta[key])
18
+ && amount(row, key) + (['input', 'output', 'cacheRead', 'cacheWrite', 'reasoning', 'calls'].includes(key) ? 0 : 1e-8) >= delta[key])
19
+ const subtract = (row, delta) => { for (const key of fields) row[key] = Math.max(0, amount(row, key) - delta[key]) }
20
+
21
+ async function affectedStreams(path) {
22
+ const inventory = await createBillionInventory(path).poll(), streams = new Map()
23
+ for (const file of inventory.streams) {
24
+ const stream = await readBillionStreamFileAsync(file)
25
+ if (stream?.finished) streams.set(stream.streamId, stream)
26
+ }
27
+ // An archive can coexist with an active original during compaction. The active
28
+ // original wins; for duplicate archives the newest published record wins.
29
+ const active = new Set(streams.keys()), archives = []
30
+ for (const file of inventory.archives) {
31
+ try { archives.push({ ...file, at: (await lstat(file.path)).mtimeMs }) }
32
+ catch (error) { if (error.code !== 'ENOENT') throw error }
33
+ }
34
+ archives.sort((a, b) => a.at - b.at || a.path.localeCompare(b.path))
35
+ for (const file of archives) for (const stream of await readBillionStreamArchive(file)) {
36
+ if (!active.has(stream.streamId)) streams.set(stream.streamId, stream)
37
+ if (streams.size > 200000) throw Error('Proxy billing repair stream limit exceeded')
38
+ }
39
+ const affected = []
40
+ for (const stream of streams.values()) {
41
+ const coverage = checkedBillionCoverage(stream, stream.coverage ?? await billionCoverage(path, stream))
42
+ if (unverifiedBillionHost(coverage)) affected.push(stream)
43
+ }
44
+ return affected
45
+ }
46
+
47
+ /**
48
+ * The existing host meter is a durable proof that this exact extra call was
49
+ * admitted. Sequence 2 retires that sequence-1 charge in the same transaction as
50
+ * its subtraction. No global residual deletion or current-price recalculation.
51
+ */
52
+ export async function repairBillionHostLedger(ledger) {
53
+ const stats = { repaired: 0, skipped: 0, input: 0, cost: 0, backup: '' }
54
+ if (!ledger.path || ledger.closed) return stats
55
+ const streams = await affectedStreams(ledger.path)
56
+ if (!streams.length || ledger.closed) return stats
57
+ // Drain our worker before acquiring its lock; keep other hosts' new calls by
58
+ // reading the latest disk snapshot inside that lock, then adopt the result.
59
+ ledger.flush(5000)
60
+ if (ledger.pendingWrite || ledger.backgroundWrite) throw Error('Ledger is still being saved; billing repair will retry at next startup')
61
+ return withLedgerLock(ledger.path, () => {
62
+ const before = cloneLedgerSnapshot(ledger.readDiskSnapshot() ?? ledger.snapshot())
63
+ const state = cloneLedgerSnapshot(before)
64
+ const config = bindPricingHistory(state.config, state.pricingHistory)
65
+ const groups = new Map()
66
+ for (const stream of streams) {
67
+ const meter = billionHostMeter(stream)
68
+ if ((state.usageCursors?.[meter.source] ?? 0) >= 2 || stream.startedAtMs <= (state.historyReset?.at ?? 0)) continue
69
+ if (state.usageCursors?.[meter.source] !== 1) {
70
+ // New unresolved streams were never charged by the guarded importer.
71
+ if (stream.coverage?.mode === 'host') stats.skipped++
72
+ continue
73
+ }
74
+ const record = { type: BILLION_USAGE_EVENT, time: stream.startedAtMs, data: { provider: stream.provider, model: stream.model,
75
+ usage: stream.reported, requestId: stream.streamId, startedAtMs: stream.startedAtMs, kind: 'model' } }
76
+ const replay = replaySessionRecords([{ type: 'session', id: stream.sessionId }, record], config, null, true)
77
+ const sample = replay.samples?.[0]
78
+ if (!sample) { stats.skipped++; continue }
79
+ const delta = replay.days[sample.date]?.[sample.providerKey]
80
+ const key = JSON.stringify([sample.date, stream.sessionId, sample.providerKey])
81
+ let group = groups.get(key)
82
+ if (!group) { group = { date: sample.date, sessionId: stream.sessionId, providerKey: sample.providerKey,
83
+ delta: Object.fromEntries(fields.map(field => [field, 0])), streams: [], hours: new Map() }; groups.set(key, group) }
84
+ for (const field of fields) group.delta[field] += delta[field] ?? 0
85
+ group.streams.push(meter.source)
86
+ const planId = sample.plan ? planProviderIdOf(stream.provider, config.planBilling, stream.model)
87
+ ?? (isRoutedThirdPartyCall(stream.provider, stream.model, config.prices) ? 'go' : null) : null
88
+ if (planId) {
89
+ const hour = Math.floor(stream.startedAtMs / 3600000) * 3600000, hourKey = JSON.stringify([planId, hour])
90
+ const slot = group.hours.get(hourKey) ?? { planId, hour, tokens: 0, cost: 0 }
91
+ slot.tokens += delta.input; slot.cost += delta.cost; group.hours.set(hourKey, slot)
92
+ }
93
+ }
94
+ for (const group of groups.values()) {
95
+ const day = state.days[group.date], session = day?.sessions?.find(row => row.id === group.sessionId)
96
+ const rows = [day, session, day?.byProviderModel?.[group.providerKey], session?.byProviderModel?.[group.providerKey]]
97
+ // Pruned, manually changed or already rebuilt data must not lose unrelated
98
+ // usage. Leave those receipts untouched and report them for inspection.
99
+ if (!rows.every(row => contains(row, group.delta))) { stats.skipped += group.streams.length; continue }
100
+ for (const row of rows) subtract(row, group.delta)
101
+ for (const { planId, hour, tokens, cost } of group.hours.values()) {
102
+ const slot = state.planHourBuckets?.[planId]?.[hour]
103
+ if (slot) { slot.tokens = Math.max(0, slot.tokens - tokens); slot.cost = Math.max(0, slot.cost - cost) }
104
+ }
105
+ for (const source of group.streams) state.usageCursors[source] = 2
106
+ stats.repaired += group.streams.length; stats.input += group.delta.input; stats.cost += group.delta.cost
107
+ if (state.balanceRef?.date === group.date) state.balanceRef = null
108
+ }
109
+ if (stats.repaired) {
110
+ stats.backup = `${ledger.path}.before-issue-458-${randomUUID()}.json`
111
+ // A failed backup aborts the correction. The backup and ledger contain the
112
+ // existing secret-stripped config, never runtime credential fallbacks.
113
+ writeDurableJson(stats.backup, { ...before, pricingHistory: encodePricingHistory(before.pricingHistory) })
114
+ writeLedger(ledger.path, state)
115
+ ledger.adoptSnapshot(state)
116
+ ledger.writeBase = ledger.snapshot()
117
+ ledger.diskRevision = null
118
+ }
119
+ return stats
120
+ }, 5000)
121
+ }
package/lib/index.js CHANGED
@@ -75,6 +75,7 @@ import { getSessionTrajectory } from './trajectory.js'
75
75
  import { createStatisticsWarmup, warmSessionStatistics } from './statistics-warmup.js'
76
76
  import { billingStatistics } from './billing-statistics.js'
77
77
  import { pricingConfigAt } from './billing-history.js'
78
+ import { repairBillionHostLedger } from './billion-context-repair.js'
78
79
 
79
80
  export const name = 'cost-meter'
80
81
 
@@ -3393,6 +3394,11 @@ export async function runStartupImports(ledger, sessionsRoot, sessionSource = nu
3393
3394
  const { cacheHit, cacheMiss, output } = oldPrices.default
3394
3395
  oldPrices.default = { cacheHit, cacheMiss, output }
3395
3396
  }
3397
+ try {
3398
+ const repaired = await repairBillionHostLedger(ledger)
3399
+ if (repaired.repaired) console.warn(`[dsh-cost-meter] 已纠正 billion-context 的 ${repaired.repaired} 条无模型调用输入记录,移除 ${repaired.input} 个误计 Token;原账本备份:${repaired.backup}`)
3400
+ if (repaired.skipped) console.warn(`[dsh-cost-meter] ${repaired.skipped} 条 billion-context 记录缺少入账凭据或与现有账本不符,保留原账本;见 issue #458`)
3401
+ } catch (error) { console.warn(`[dsh-cost-meter] billion-context 历史纠正未完成,下次启动重试:${String(error?.message ?? error)}`) }
3396
3402
  const filled = await backfillLegacyLedger(ledger, sessionsRoot, sessionSource)
3397
3403
  if (filled.days > 0 || filled.sessions > 0 || filled.titles > 0) {
3398
3404
  const extras = [