dsh-all-usage 1.1.3 → 1.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +58 -2
- package/README.md +31 -44
- package/assets/model-icons/LICENSE.upstream-lobe-icons.txt +51 -0
- package/assets/model-icons/claude-color.svg +1 -0
- package/assets/model-icons/deepseek-color.svg +1 -0
- package/assets/model-icons/doubao-color.svg +1 -0
- package/assets/model-icons/gemini-color.svg +1 -0
- package/assets/model-icons/grok.svg +1 -0
- package/assets/model-icons/kimi-color.svg +1 -0
- package/assets/model-icons/manifest.json +213 -0
- package/assets/model-icons/meta-color.svg +1 -0
- package/assets/model-icons/minimax-color.svg +1 -0
- package/assets/model-icons/openai-color.svg +1 -0
- package/assets/model-icons/qwen-color.svg +1 -0
- package/assets/model-icons/zhipu-color.svg +1 -0
- package/assets/screenshot-1.png +0 -0
- package/assets/screenshot-2.png +0 -0
- package/assets/screenshot-3.png +0 -0
- package/lib/aggregation.js +58 -10
- package/lib/client.js +1 -1
- package/lib/http.js +1 -1
- package/lib/ledger.js +34 -5
- package/lib/plugin.js +4 -0
- package/lib/pricing-runtime.js +135 -11
- package/lib/pricing.js +335 -6
- package/lib/session-sync.js +329 -121
- package/lib/usage-core.js +51 -0
- package/package.json +1 -1
package/lib/session-sync.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import { extractUsageEvent } from './usage-core.js'
|
|
1
|
+
import { contextTimeKey, extractUsageEvent, pickPricingTime, touchContextTimes, workspacePathKey } from './usage-core.js'
|
|
2
2
|
|
|
3
3
|
const RECONCILE_INTERVAL_MS = 120000
|
|
4
4
|
const RECONCILE_HINT_DELAY_MS = 3000
|
|
5
|
+
const WORKSPACE_SYNC_DEBOUNCE_MS = 300
|
|
5
6
|
|
|
6
7
|
export function createSessionSync(host) {
|
|
7
8
|
const { ctx, state } = host
|
|
@@ -25,6 +26,7 @@ export function createSessionSync(host) {
|
|
|
25
26
|
drainLedgerWrites,
|
|
26
27
|
} = host.ledger
|
|
27
28
|
const backfillUnpricedCosts = (...args) => host.pricing.backfillUnpricedCosts(...args)
|
|
29
|
+
const reconcileTemporalPricing = (...args) => host.pricing.reconcileTemporalPricing(...args)
|
|
28
30
|
const safeContextTimeout = async (ms) => {
|
|
29
31
|
if (state.disposed) return false
|
|
30
32
|
try {
|
|
@@ -38,7 +40,14 @@ export function createSessionSync(host) {
|
|
|
38
40
|
|
|
39
41
|
function foldEvent(wsId, time, type, data, sid, seq, materialization = 'live') {
|
|
40
42
|
if ((type === 'turn/end' || type === 'assistant/message' || type === 'assistant/chunk') && !validEventTime(time)) return
|
|
41
|
-
if (type === 'request/context'
|
|
43
|
+
if (type === 'request/context') {
|
|
44
|
+
state.sessionModel.set(sid, identityFromRoute(data, state.sessionModel.get(sid)))
|
|
45
|
+
if (validEventTime(time)) {
|
|
46
|
+
let times = state.sessionContextTimes.get(sid)
|
|
47
|
+
if (times === undefined) { times = new Map(); state.sessionContextTimes.set(sid, times) }
|
|
48
|
+
touchContextTimes(times, contextTimeKey(data && data.turn, data && data.step), time)
|
|
49
|
+
}
|
|
50
|
+
} else if (type === 'request/header') {
|
|
42
51
|
state.sessionModel.set(sid, identityFromRoute(data, state.sessionModel.get(sid)))
|
|
43
52
|
} else if (type === 'turn/end') {
|
|
44
53
|
addTurn(wsId, time, sid, data && typeof data.turn === 'number' ? data.turn : null, state.sessionModel.get(sid), materialization, seq)
|
|
@@ -47,10 +56,15 @@ export function createSessionSync(host) {
|
|
|
47
56
|
if (usageEvent === null) return
|
|
48
57
|
const identity = usageEvent.kind === 'message' ? identityFromMessage(data, state.sessionModel.get(sid)) : coerceIdentity(state.sessionModel.get(sid))
|
|
49
58
|
state.sessionModel.set(sid, identity)
|
|
50
|
-
|
|
59
|
+
const pricing = pickPricingTime(state.sessionContextTimes.get(sid), time, usageEvent.turn, usageEvent.step)
|
|
60
|
+
addUsage(wsId, time, usageEvent.usage, identity, sid, data, seq, materialization, pricing.time, pricing.source)
|
|
51
61
|
}
|
|
52
62
|
}
|
|
53
63
|
function foldEvents(wsId, events, fromSeq, sid, materialization = 'scan') {
|
|
64
|
+
// Full folds keep their callers responsible for atomicity: runBaseline and
|
|
65
|
+
// syncLiveSession remove the session's old usage/turn/context/identity
|
|
66
|
+
// first when the snapshot is authoritative (see rebuildSession below), so
|
|
67
|
+
// usage deleted by a history rewrite disappears from the aggregate.
|
|
54
68
|
for (const ev of events) {
|
|
55
69
|
if (fromSeq !== undefined) {
|
|
56
70
|
const s = typeof ev.seq === 'number' ? ev.seq : -1
|
|
@@ -94,13 +108,56 @@ export function createSessionSync(host) {
|
|
|
94
108
|
state.ledgerDirtySessions.add(sid)
|
|
95
109
|
state.ledgerDirtyEpochs.set(sid, (state.ledgerDirtyEpochs.get(sid) || 0) + 1)
|
|
96
110
|
}
|
|
111
|
+
function buildWorkspaceSnapshot(workspaces) {
|
|
112
|
+
const wsMeta = new Map()
|
|
113
|
+
const pathIndex = new Map()
|
|
114
|
+
const memberOf = new Map()
|
|
115
|
+
for (const w of workspaces) {
|
|
116
|
+
const id = w && w.id
|
|
117
|
+
const path = w && typeof w.path === 'string' ? w.path : ''
|
|
118
|
+
const title = w && typeof w.title === 'string' ? w.title : ''
|
|
119
|
+
if (id === undefined) continue
|
|
120
|
+
wsMeta.set(id, { id, title, path })
|
|
121
|
+
if (path !== '') pathIndex.set(workspacePathKey(path), id)
|
|
122
|
+
if (w && Array.isArray(w.sessionIds)) {
|
|
123
|
+
for (const sid of w.sessionIds) memberOf.set(sid, id)
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
return { wsMeta, pathIndex, memberOf }
|
|
127
|
+
}
|
|
128
|
+
function installWorkspaceSnapshot(next) {
|
|
129
|
+
let metadataChanged = state.wsMeta.size !== next.wsMeta.size || state.pathIndex.size !== next.pathIndex.size || state.memberOf.size !== next.memberOf.size
|
|
130
|
+
if (!metadataChanged) {
|
|
131
|
+
for (const [id, value] of next.wsMeta) {
|
|
132
|
+
const previous = state.wsMeta.get(id)
|
|
133
|
+
if (previous === undefined || previous.title !== value.title || previous.path !== value.path) { metadataChanged = true; break }
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
if (!metadataChanged) {
|
|
137
|
+
for (const [path, id] of next.pathIndex) if (state.pathIndex.get(path) !== id) { metadataChanged = true; break }
|
|
138
|
+
}
|
|
139
|
+
if (!metadataChanged) {
|
|
140
|
+
for (const [sid, id] of next.memberOf) if (state.memberOf.get(sid) !== id) { metadataChanged = true; break }
|
|
141
|
+
}
|
|
142
|
+
state.wsMeta.clear()
|
|
143
|
+
state.pathIndex.clear()
|
|
144
|
+
state.memberOf.clear()
|
|
145
|
+
for (const [id, value] of next.wsMeta) state.wsMeta.set(id, value)
|
|
146
|
+
for (const [path, id] of next.pathIndex) state.pathIndex.set(path, id)
|
|
147
|
+
for (const [sid, id] of next.memberOf) state.memberOf.set(sid, id)
|
|
148
|
+
if (metadataChanged) markStatsChanged('metadata')
|
|
149
|
+
return metadataChanged
|
|
150
|
+
}
|
|
151
|
+
function workspaceForCwd(cwd) {
|
|
152
|
+
if (typeof cwd !== 'string' || cwd === '') return undefined
|
|
153
|
+
return state.pathIndex.get(workspacePathKey(cwd))
|
|
154
|
+
}
|
|
97
155
|
function wsForLiveSession(session, sid) {
|
|
98
156
|
let wsId = state.memberOf.get(sid)
|
|
99
157
|
if (wsId !== undefined) return wsId
|
|
100
158
|
const header = session && session.header
|
|
101
159
|
const cwd = header && typeof header.cwd === 'string' ? header.cwd : ''
|
|
102
|
-
|
|
103
|
-
wsId = state.pathIndex.get(cwd)
|
|
160
|
+
wsId = workspaceForCwd(cwd)
|
|
104
161
|
if (wsId !== undefined) state.memberOf.set(sid, wsId)
|
|
105
162
|
return wsId
|
|
106
163
|
}
|
|
@@ -140,13 +197,41 @@ export function createSessionSync(host) {
|
|
|
140
197
|
const snap = await ctx.sessionQuery.readSession(sid)
|
|
141
198
|
if (state.disposed || generation !== state.aggregationGeneration) return true
|
|
142
199
|
if (snap && Array.isArray(snap.events)) {
|
|
143
|
-
const previousLast = state.sessionSeq.get(sid)
|
|
144
|
-
const previousLastSafe = Number.isSafeInteger(previousLast) && previousLast >= 0 ? previousLast : -1
|
|
145
200
|
const snapshotLast = lastSeqOf(snap.events)
|
|
146
|
-
|
|
147
|
-
let nextLast = Math.max(previousLastSafe, snapshotLast)
|
|
201
|
+
const snapshotProfile = sequenceProfile(snap.events)
|
|
148
202
|
const eventSeq = event === null || event === undefined ? -1 : (Number.isSafeInteger(event.seq) && event.seq >= 0 ? event.seq : -1)
|
|
149
|
-
|
|
203
|
+
// The snapshot may replace the session only when it is provably
|
|
204
|
+
// complete AND it already contains the triggering event: a timer / full
|
|
205
|
+
// resync (no event), or an explicitly contained, monotonic event
|
|
206
|
+
// stream. A snapshot that merely has a higher tail than the event is
|
|
207
|
+
// NOT proof — the event could be a live append the snapshot missed (or
|
|
208
|
+
// the snapshot could have been rewritten without it), so the event is
|
|
209
|
+
// upserted and the session stays pending for a follow-up alignment.
|
|
210
|
+
// Events without a usable seq never trigger a destructive replacement.
|
|
211
|
+
const containsEvent = event !== null && event !== undefined && eventSeq >= 0 && snap.events.some((candidate) => candidate && Number.isSafeInteger(candidate.seq) && candidate.seq === eventSeq)
|
|
212
|
+
const authoritative = event === null || event === undefined ? true : eventSeq < 0 ? false : containsEvent && !snapshotProfile.nonMonotonic && !snapshotProfile.hasInvalid
|
|
213
|
+
if (authoritative) {
|
|
214
|
+
host.aggregation.removeSession(sid)
|
|
215
|
+
// The authoritative snapshot is the new truth for both the in-memory
|
|
216
|
+
// aggregate and the durable ledger: rebuild the record from scratch
|
|
217
|
+
// and persist it, so a restart after a live history rewrite cannot
|
|
218
|
+
// resurrect usage that the snapshot deleted.
|
|
219
|
+
const rebuiltRecord = buildLedgerRecord({ id: sid, events: snap.events }, wsId, 'scan', undefined, null, true)
|
|
220
|
+
if (rebuiltRecord !== null) {
|
|
221
|
+
const canonical = replaceLedgerRecord(rebuiltRecord)
|
|
222
|
+
if (canonical === rebuiltRecord) {
|
|
223
|
+
void persistLedgerRecord(rebuiltRecord).then(() => {
|
|
224
|
+
if (state.ledgerWriteFailedSessions && state.ledgerWriteFailedSessions.has(sid)) {
|
|
225
|
+
console.error('[all-usage] authoritative resync ledger persist failed; session stays dirty:', sid)
|
|
226
|
+
noteSyncError('resync-ledger-persist-failed')
|
|
227
|
+
}
|
|
228
|
+
})
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
foldEvents(wsId, snap.events, undefined, sid, 'live')
|
|
233
|
+
let nextLast = snapshotLast
|
|
234
|
+
const needsFollowup = !authoritative && event !== null && event !== undefined
|
|
150
235
|
if (needsFollowup) {
|
|
151
236
|
foldLiveFallback(sid, wsId, event)
|
|
152
237
|
const current = state.sessionSeq.get(sid)
|
|
@@ -154,8 +239,14 @@ export function createSessionSync(host) {
|
|
|
154
239
|
}
|
|
155
240
|
state.sessionSeq.set(sid, nextLast)
|
|
156
241
|
state.sessionCount.add(sid)
|
|
157
|
-
if (needsFollowup)
|
|
158
|
-
|
|
242
|
+
if (needsFollowup) {
|
|
243
|
+
// The snapshot is behind: keep the pending flag so the follow-up
|
|
244
|
+
// resync realigns the cursor once the missing history arrives.
|
|
245
|
+
scheduleLiveResync(sid, wsId, generation)
|
|
246
|
+
} else {
|
|
247
|
+
state.liveResyncPending.delete(sid)
|
|
248
|
+
cancelLiveResync(sid)
|
|
249
|
+
}
|
|
159
250
|
return true
|
|
160
251
|
}
|
|
161
252
|
} catch (err) {
|
|
@@ -212,6 +303,81 @@ export function createSessionSync(host) {
|
|
|
212
303
|
return undefined
|
|
213
304
|
})
|
|
214
305
|
}
|
|
306
|
+
function scanOneSession(sid, record, wsId, generation, snapshots) {
|
|
307
|
+
return enqueue(sid, async () => {
|
|
308
|
+
if (state.disposed || generation !== state.aggregationGeneration) return
|
|
309
|
+
try {
|
|
310
|
+
if (state.sessionSeq.has(sid) && !state.liveResyncPending.has(sid)) return
|
|
311
|
+
// v1.0.8: when the persisted log revision is unchanged since the last ledger
|
|
312
|
+
// write, the whole readSession (full event transfer) is skipped — the ledger
|
|
313
|
+
// record is applied directly and the live feed keeps catching new events.
|
|
314
|
+
const previousRecord = state.ledgerRecords.get(sid)
|
|
315
|
+
const revision = snapshots === null ? undefined : snapshots.get(sid)
|
|
316
|
+
if (!state.liveResyncPending.has(sid) && previousRecord !== undefined && previousRecord.needsUpgrade !== true && previousRecord.rebuildRequired === undefined && previousRecord.workspaceId === wsId && typeof previousRecord.lastRevision === 'string' && typeof revision === 'string' && revision === previousRecord.lastRevision) {
|
|
317
|
+
state.sync.sessionsSkippedByRevision += 1
|
|
318
|
+
applyLedgerRecord(previousRecord, 'ledger-reuse')
|
|
319
|
+
state.sessionSeq.set(sid, previousRecord.lastSeq)
|
|
320
|
+
state.sessionCount.add(sid)
|
|
321
|
+
markStatsChanged('scan')
|
|
322
|
+
return
|
|
323
|
+
}
|
|
324
|
+
state.sync.sessionsRead += 1
|
|
325
|
+
markStatsChanged('scan')
|
|
326
|
+
const snap = await ctx.sessionQuery.readSession(sid)
|
|
327
|
+
if (state.disposed || generation !== state.aggregationGeneration) return
|
|
328
|
+
if (snap && Array.isArray(snap.events)) {
|
|
329
|
+
// v1.0.7: incremental seed — the durable ledger doubles as a per-session
|
|
330
|
+
// cursor (cc-switch session_log_sync mtime+offset parity). An unchanged
|
|
331
|
+
// session applies its canonical record directly and never re-folds;
|
|
332
|
+
// a changed session seeds the previous record once, then folds only the
|
|
333
|
+
// new tail (previously every listed session was re-read and fully rebuilt).
|
|
334
|
+
const sequence = sequenceProfile(snap.events)
|
|
335
|
+
const currentLastSeq = sequence.lastSeq
|
|
336
|
+
const previous = state.ledgerRecords.get(sid)
|
|
337
|
+
const canFoldTail = previous !== undefined && previous.needsUpgrade !== true && previous.rebuildRequired === undefined && previous.workspaceId === wsId && !sequence.nonMonotonic && !sequence.hasInvalid && previous.lastSeq >= 0 && currentLastSeq > previous.lastSeq
|
|
338
|
+
if (canFoldTail) {
|
|
339
|
+
applyLedgerRecord(previous, 'ledger-reuse')
|
|
340
|
+
foldEvents(wsId, snap.events, previous.lastSeq, sid, 'scan')
|
|
341
|
+
} else {
|
|
342
|
+
// A changed revision with no new tail may still contain a replacement;
|
|
343
|
+
// rebuild from the complete read instead of trusting lastSeq alone.
|
|
344
|
+
foldEvents(wsId, snap.events, undefined, sid, 'scan')
|
|
345
|
+
}
|
|
346
|
+
const ledger = buildLedgerRecord({ id: sid, header: record.header, events: snap.events }, wsId, 'scan', revision, previous, !sequence.hasInvalid)
|
|
347
|
+
const canonical = ledger === null ? state.ledgerRecords.get(sid) : (canFoldTail ? storeLedgerRecord(ledger) : replaceLedgerRecord(ledger))
|
|
348
|
+
if (canonical === ledger) {
|
|
349
|
+
void persistLedgerRecord(ledger)
|
|
350
|
+
}
|
|
351
|
+
const observedLastSeq = state.sessionSeq.get(sid)
|
|
352
|
+
const nextLastSeq = Math.max(currentLastSeq, observedLastSeq === undefined ? -1 : observedLastSeq)
|
|
353
|
+
state.sessionSeq.set(sid, nextLastSeq)
|
|
354
|
+
state.sessionCount.add(sid)
|
|
355
|
+
if (observedLastSeq === undefined || observedLastSeq <= currentLastSeq) cancelLiveResync(sid)
|
|
356
|
+
else scheduleLiveResync(sid, wsId, generation)
|
|
357
|
+
}
|
|
358
|
+
} catch (err) {
|
|
359
|
+
if (generation !== state.aggregationGeneration) return
|
|
360
|
+
state.sync.sessionsFailed += 1
|
|
361
|
+
state.scan.failed += 1
|
|
362
|
+
noteSyncError('session-read-failed')
|
|
363
|
+
const saved = state.ledgerRecords.get(sid)
|
|
364
|
+
if (saved !== undefined) {
|
|
365
|
+
applyLedgerRecord(saved, 'ledger-recovery')
|
|
366
|
+
if (saved.turns.length > 0 || saved.usage.length > 0) state.sync.sessionsRestoredFromLedger += 1
|
|
367
|
+
state.sessionSeq.set(sid, -1)
|
|
368
|
+
state.sessionCount.add(sid)
|
|
369
|
+
} else {
|
|
370
|
+
state.sessionSeq.set(sid, -1)
|
|
371
|
+
}
|
|
372
|
+
} finally {
|
|
373
|
+
if (generation === state.aggregationGeneration) {
|
|
374
|
+
state.scan.scanned += 1
|
|
375
|
+
markStatsChanged('scan')
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
})
|
|
379
|
+
}
|
|
380
|
+
|
|
215
381
|
async function runBaseline(generation = state.aggregationGeneration) {
|
|
216
382
|
if (state.scan.started || state.disposed || generation !== state.aggregationGeneration) return
|
|
217
383
|
state.scan.started = true
|
|
@@ -221,40 +387,7 @@ export function createSessionSync(host) {
|
|
|
221
387
|
let setupFailed = false
|
|
222
388
|
try {
|
|
223
389
|
const workspaces = ctx.workspaceRegistry.list()
|
|
224
|
-
|
|
225
|
-
const nextPathIndex = new Map()
|
|
226
|
-
const nextMemberOf = new Map()
|
|
227
|
-
for (const w of workspaces) {
|
|
228
|
-
const id = w && w.id
|
|
229
|
-
const path = w && typeof w.path === 'string' ? w.path : ''
|
|
230
|
-
const title = w && typeof w.title === 'string' ? w.title : ''
|
|
231
|
-
if (id === undefined) continue
|
|
232
|
-
nextWsMeta.set(id, { id, title, path })
|
|
233
|
-
if (path !== '') nextPathIndex.set(path, id)
|
|
234
|
-
if (w && Array.isArray(w.sessionIds)) {
|
|
235
|
-
for (const sid of w.sessionIds) nextMemberOf.set(sid, id)
|
|
236
|
-
}
|
|
237
|
-
}
|
|
238
|
-
let metadataChanged = state.wsMeta.size !== nextWsMeta.size || state.pathIndex.size !== nextPathIndex.size || state.memberOf.size !== nextMemberOf.size
|
|
239
|
-
if (!metadataChanged) {
|
|
240
|
-
for (const [id, value] of nextWsMeta) {
|
|
241
|
-
const previous = state.wsMeta.get(id)
|
|
242
|
-
if (previous === undefined || previous.title !== value.title || previous.path !== value.path) { metadataChanged = true; break }
|
|
243
|
-
}
|
|
244
|
-
}
|
|
245
|
-
if (!metadataChanged) {
|
|
246
|
-
for (const [path, id] of nextPathIndex) if (state.pathIndex.get(path) !== id) { metadataChanged = true; break }
|
|
247
|
-
}
|
|
248
|
-
if (!metadataChanged) {
|
|
249
|
-
for (const [sid, id] of nextMemberOf) if (state.memberOf.get(sid) !== id) { metadataChanged = true; break }
|
|
250
|
-
}
|
|
251
|
-
state.wsMeta.clear()
|
|
252
|
-
state.pathIndex.clear()
|
|
253
|
-
state.memberOf.clear()
|
|
254
|
-
for (const [id, value] of nextWsMeta) state.wsMeta.set(id, value)
|
|
255
|
-
for (const [path, id] of nextPathIndex) state.pathIndex.set(path, id)
|
|
256
|
-
for (const [sid, id] of nextMemberOf) state.memberOf.set(sid, id)
|
|
257
|
-
if (metadataChanged) markStatsChanged('metadata')
|
|
390
|
+
installWorkspaceSnapshot(buildWorkspaceSnapshot(workspaces))
|
|
258
391
|
} catch (err) {
|
|
259
392
|
console.error('[all-usage] workspace list failed:', err)
|
|
260
393
|
setupFailed = true
|
|
@@ -306,11 +439,18 @@ export function createSessionSync(host) {
|
|
|
306
439
|
if (record === undefined || record === null || record.header === undefined) continue
|
|
307
440
|
const sid = record.header.id
|
|
308
441
|
const cwd = typeof record.header.cwd === 'string' ? record.header.cwd : ''
|
|
309
|
-
const wsId =
|
|
442
|
+
const wsId = workspaceForCwd(cwd)
|
|
310
443
|
if (sid !== undefined && wsId !== undefined) listedSessionIds.add(sid)
|
|
311
444
|
}
|
|
312
445
|
for (const [sid, record] of state.ledgerRecords) {
|
|
313
446
|
if (!listedSessionIds.has(sid)) {
|
|
447
|
+
// A ledger row from a deleted cwd is stale historical data, not an
|
|
448
|
+
// active workspace. Registered records remain recoverable; unknown
|
|
449
|
+
// records are recovered only while their source cwd still exists. Old
|
|
450
|
+
// synthetic rows predate sourceCwd persistence and cannot be validated,
|
|
451
|
+
// so they are fail-closed rather than resurrected after a restart.
|
|
452
|
+
if (record.sourceCwd !== undefined && workspaceForCwd(record.sourceCwd) === undefined) continue
|
|
453
|
+
if (typeof record.workspaceId === 'string' && record.workspaceId.startsWith('unregistered:')) continue
|
|
314
454
|
applyLedgerRecord(record, 'ledger-recovery')
|
|
315
455
|
if (record.turns.length > 0 || record.usage.length > 0) state.sync.sessionsRestoredFromLedger += 1
|
|
316
456
|
}
|
|
@@ -325,85 +465,14 @@ export function createSessionSync(host) {
|
|
|
325
465
|
}
|
|
326
466
|
const sid = record.header.id
|
|
327
467
|
const cwd = typeof record.header.cwd === 'string' ? record.header.cwd : ''
|
|
328
|
-
const wsId =
|
|
468
|
+
const wsId = workspaceForCwd(cwd)
|
|
329
469
|
if (sid === undefined || wsId === undefined) {
|
|
330
470
|
state.scan.scanned += 1
|
|
331
471
|
markStatsChanged('scan')
|
|
332
472
|
continue
|
|
333
473
|
}
|
|
334
474
|
listedSessionIds.add(sid)
|
|
335
|
-
await
|
|
336
|
-
if (state.disposed || generation !== state.aggregationGeneration) return
|
|
337
|
-
try {
|
|
338
|
-
if (state.sessionSeq.has(sid) && !state.liveResyncPending.has(sid)) return
|
|
339
|
-
// v1.0.8: when the persisted log revision is unchanged since the last ledger
|
|
340
|
-
// write, the whole readSession (full event transfer) is skipped — the ledger
|
|
341
|
-
// record is applied directly and the live feed keeps catching new events.
|
|
342
|
-
const previousRecord = state.ledgerRecords.get(sid)
|
|
343
|
-
const revision = snapshots === null ? undefined : snapshots.get(sid)
|
|
344
|
-
if (!state.liveResyncPending.has(sid) && previousRecord !== undefined && previousRecord.needsUpgrade !== true && previousRecord.rebuildRequired === undefined && previousRecord.workspaceId === wsId && typeof previousRecord.lastRevision === 'string' && typeof revision === 'string' && revision === previousRecord.lastRevision) {
|
|
345
|
-
state.sync.sessionsSkippedByRevision += 1
|
|
346
|
-
applyLedgerRecord(previousRecord, 'ledger-reuse')
|
|
347
|
-
state.sessionSeq.set(sid, previousRecord.lastSeq)
|
|
348
|
-
state.sessionCount.add(sid)
|
|
349
|
-
markStatsChanged('scan')
|
|
350
|
-
return
|
|
351
|
-
}
|
|
352
|
-
state.sync.sessionsRead += 1
|
|
353
|
-
markStatsChanged('scan')
|
|
354
|
-
const snap = await ctx.sessionQuery.readSession(sid)
|
|
355
|
-
if (state.disposed || generation !== state.aggregationGeneration) return
|
|
356
|
-
if (snap && Array.isArray(snap.events)) {
|
|
357
|
-
// v1.0.7: incremental seed — the durable ledger doubles as a per-session
|
|
358
|
-
// cursor (cc-switch session_log_sync mtime+offset parity). An unchanged
|
|
359
|
-
// session applies its canonical record directly and never re-folds;
|
|
360
|
-
// a changed session seeds the previous record once, then folds only the
|
|
361
|
-
// new tail (previously every listed session was re-read and fully rebuilt).
|
|
362
|
-
const sequence = sequenceProfile(snap.events)
|
|
363
|
-
const currentLastSeq = sequence.lastSeq
|
|
364
|
-
const previous = state.ledgerRecords.get(sid)
|
|
365
|
-
const canFoldTail = previous !== undefined && previous.needsUpgrade !== true && previous.rebuildRequired === undefined && previous.workspaceId === wsId && !sequence.nonMonotonic && !sequence.hasInvalid && previous.lastSeq >= 0 && currentLastSeq > previous.lastSeq
|
|
366
|
-
if (canFoldTail) {
|
|
367
|
-
applyLedgerRecord(previous, 'ledger-reuse')
|
|
368
|
-
foldEvents(wsId, snap.events, previous.lastSeq, sid, 'scan')
|
|
369
|
-
} else {
|
|
370
|
-
// A changed revision with no new tail may still contain a replacement;
|
|
371
|
-
// rebuild from the complete read instead of trusting lastSeq alone.
|
|
372
|
-
foldEvents(wsId, snap.events, undefined, sid, 'scan')
|
|
373
|
-
}
|
|
374
|
-
const ledger = buildLedgerRecord({ id: sid, header: record.header, events: snap.events }, wsId, 'scan', revision, previous, !sequence.hasInvalid)
|
|
375
|
-
const canonical = ledger === null ? state.ledgerRecords.get(sid) : (canFoldTail ? storeLedgerRecord(ledger) : replaceLedgerRecord(ledger))
|
|
376
|
-
if (canonical === ledger) {
|
|
377
|
-
void persistLedgerRecord(ledger)
|
|
378
|
-
}
|
|
379
|
-
const observedLastSeq = state.sessionSeq.get(sid)
|
|
380
|
-
const nextLastSeq = Math.max(currentLastSeq, observedLastSeq === undefined ? -1 : observedLastSeq)
|
|
381
|
-
state.sessionSeq.set(sid, nextLastSeq)
|
|
382
|
-
state.sessionCount.add(sid)
|
|
383
|
-
if (observedLastSeq === undefined || observedLastSeq <= currentLastSeq) cancelLiveResync(sid)
|
|
384
|
-
else scheduleLiveResync(sid, wsId, generation)
|
|
385
|
-
}
|
|
386
|
-
} catch (err) {
|
|
387
|
-
if (generation !== state.aggregationGeneration) return
|
|
388
|
-
state.sync.sessionsFailed += 1
|
|
389
|
-
state.scan.failed += 1
|
|
390
|
-
noteSyncError('session-read-failed')
|
|
391
|
-
const saved = state.ledgerRecords.get(sid)
|
|
392
|
-
if (saved !== undefined) {
|
|
393
|
-
applyLedgerRecord(saved, 'ledger-recovery')
|
|
394
|
-
if (saved.turns.length > 0 || saved.usage.length > 0) state.sync.sessionsRestoredFromLedger += 1
|
|
395
|
-
state.sessionSeq.set(sid, -1)
|
|
396
|
-
state.sessionCount.add(sid)
|
|
397
|
-
} else {
|
|
398
|
-
state.sessionSeq.set(sid, -1)
|
|
399
|
-
}
|
|
400
|
-
} finally {
|
|
401
|
-
if (generation === state.aggregationGeneration) {
|
|
402
|
-
state.scan.scanned += 1
|
|
403
|
-
markStatsChanged('scan')
|
|
404
|
-
}
|
|
405
|
-
}
|
|
406
|
-
})
|
|
475
|
+
await scanOneSession(sid, record, wsId, generation, snapshots)
|
|
407
476
|
if (!(await safeContextTimeout(0))) {
|
|
408
477
|
state.scan.started = false
|
|
409
478
|
noteSyncError('baseline-yield-unavailable')
|
|
@@ -420,6 +489,12 @@ export function createSessionSync(host) {
|
|
|
420
489
|
markStatsChanged('pricing')
|
|
421
490
|
}
|
|
422
491
|
if (state.disposed || generation !== state.aggregationGeneration) return
|
|
492
|
+
const temporalReconcile = reconcileTemporalPricing()
|
|
493
|
+
if (temporalReconcile.reconciled > 0) {
|
|
494
|
+
await drainLedgerWrites()
|
|
495
|
+
markStatsChanged('pricing')
|
|
496
|
+
}
|
|
497
|
+
if (state.disposed || generation !== state.aggregationGeneration) return
|
|
423
498
|
state.knownSessionIds.clear()
|
|
424
499
|
for (const sid of listedSessionIds) state.knownSessionIds.add(sid)
|
|
425
500
|
state.scan.done = true
|
|
@@ -432,13 +507,132 @@ export function createSessionSync(host) {
|
|
|
432
507
|
if (state.reconcilePending) scheduleReconcileHint()
|
|
433
508
|
}
|
|
434
509
|
|
|
510
|
+
function scheduleWorkspaceSync() {
|
|
511
|
+
if (state.disposed) return
|
|
512
|
+
if (state.workspaceSyncTimer !== null) return
|
|
513
|
+
state.workspaceSyncTimer = setTimeout(() => {
|
|
514
|
+
state.workspaceSyncTimer = null
|
|
515
|
+
if (!state.disposed) void runWorkspaceSync()
|
|
516
|
+
}, WORKSPACE_SYNC_DEBOUNCE_MS)
|
|
517
|
+
if (state.workspaceSyncTimer && typeof state.workspaceSyncTimer.unref === 'function') state.workspaceSyncTimer.unref()
|
|
518
|
+
}
|
|
519
|
+
async function runWorkspaceSync() {
|
|
520
|
+
if (state.disposed) return
|
|
521
|
+
if (state.workspaceSyncInFlight) {
|
|
522
|
+
state.workspaceSyncPending = true
|
|
523
|
+
return
|
|
524
|
+
}
|
|
525
|
+
state.workspaceSyncInFlight = true
|
|
526
|
+
try {
|
|
527
|
+
await synchronizeWorkspaceRegistry()
|
|
528
|
+
} catch (err) {
|
|
529
|
+
console.error('[all-usage] workspace synchronize failed:', err)
|
|
530
|
+
noteSyncError('workspace-sync-failed')
|
|
531
|
+
} finally {
|
|
532
|
+
state.workspaceSyncInFlight = false
|
|
533
|
+
if (state.workspaceSyncPending) {
|
|
534
|
+
state.workspaceSyncPending = false
|
|
535
|
+
scheduleWorkspaceSync()
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
async function synchronizeWorkspaceRegistry() {
|
|
540
|
+
// The first baseline reads the registry with the latest list, so a probe
|
|
541
|
+
// arriving before it completes has nothing to add.
|
|
542
|
+
if (state.disposed || !state.scan.done) return
|
|
543
|
+
let workspaces
|
|
544
|
+
try {
|
|
545
|
+
workspaces = ctx.workspaceRegistry.list()
|
|
546
|
+
} catch (err) {
|
|
547
|
+
console.error('[all-usage] workspace list failed:', err)
|
|
548
|
+
noteSyncError('workspace-list-failed')
|
|
549
|
+
return
|
|
550
|
+
}
|
|
551
|
+
const next = buildWorkspaceSnapshot(workspaces)
|
|
552
|
+
const removedWs = new Set()
|
|
553
|
+
for (const id of state.wsMeta.keys()) if (!next.wsMeta.has(id)) removedWs.add(id)
|
|
554
|
+
const addedWs = new Set()
|
|
555
|
+
for (const id of next.wsMeta.keys()) if (!state.wsMeta.has(id)) addedWs.add(id)
|
|
556
|
+
// Reuse path: no membership change — metadata only, zero rescan.
|
|
557
|
+
if (removedWs.size === 0 && addedWs.size === 0) {
|
|
558
|
+
installWorkspaceSnapshot(next)
|
|
559
|
+
return
|
|
560
|
+
}
|
|
561
|
+
// Remove path: subtract every session that belonged to a deregistered workspace.
|
|
562
|
+
if (removedWs.size > 0) {
|
|
563
|
+
const removedSids = new Set()
|
|
564
|
+
for (const [sid, record] of state.ledgerRecords) {
|
|
565
|
+
if (record !== null && record !== undefined && typeof record.workspaceId === 'string' && removedWs.has(record.workspaceId)) removedSids.add(sid)
|
|
566
|
+
}
|
|
567
|
+
for (const [sid, id] of state.memberOf) if (removedWs.has(id)) removedSids.add(sid)
|
|
568
|
+
for (const sid of removedSids) {
|
|
569
|
+
if (typeof sid !== 'string' || sid === '') continue
|
|
570
|
+
host.aggregation.removeSession(sid)
|
|
571
|
+
state.ledgerRecords.delete(sid)
|
|
572
|
+
state.knownSessionIds.delete(sid)
|
|
573
|
+
state.sessionCount.delete(sid)
|
|
574
|
+
state.sessionSeq.delete(sid)
|
|
575
|
+
state.sessionModel.delete(sid)
|
|
576
|
+
state.sessionContextTimes.delete(sid)
|
|
577
|
+
state.memberOf.delete(sid)
|
|
578
|
+
state.liveResyncPending.delete(sid)
|
|
579
|
+
cancelLiveResync(sid)
|
|
580
|
+
}
|
|
581
|
+
for (const id of removedWs) {
|
|
582
|
+
state.perWorkspace.delete(id)
|
|
583
|
+
for (const day of state.byDay.values()) day.byWs.delete(id)
|
|
584
|
+
for (const day of state.byDayUtc.values()) day.byWs.delete(id)
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
installWorkspaceSnapshot(next)
|
|
588
|
+
// Add path: scan only the sessions that belong to newly registered workspaces.
|
|
589
|
+
if (addedWs.size === 0) return
|
|
590
|
+
let records = null
|
|
591
|
+
try {
|
|
592
|
+
records = await ctx.sessionQuery.listSessions()
|
|
593
|
+
} catch (err) {
|
|
594
|
+
console.error('[all-usage] session list failed:', err)
|
|
595
|
+
noteSyncError('session-list-failed')
|
|
596
|
+
}
|
|
597
|
+
if (!Array.isArray(records)) return
|
|
598
|
+
state.sync.sessionsTotal = records.length
|
|
599
|
+
markStatsChanged('scan')
|
|
600
|
+
let snapshots = null
|
|
601
|
+
if (sessionPersistence !== undefined && typeof sessionPersistence.listSnapshots === 'function') {
|
|
602
|
+
try {
|
|
603
|
+
const rows = await sessionPersistence.listSnapshots()
|
|
604
|
+
if (Array.isArray(rows)) {
|
|
605
|
+
snapshots = new Map()
|
|
606
|
+
for (const row of rows) {
|
|
607
|
+
const rid = row && row.header && typeof row.header.id === 'string' ? row.header.id : undefined
|
|
608
|
+
if (rid !== undefined && row && typeof row.revision === 'string') snapshots.set(rid, row.revision)
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
} catch (err) {
|
|
612
|
+
console.error('[all-usage] session persistence snapshots unavailable:', err)
|
|
613
|
+
markStatsChanged('scan')
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
for (const record of records) {
|
|
617
|
+
if (state.disposed) return
|
|
618
|
+
if (record === undefined || record === null || record.header === undefined) continue
|
|
619
|
+
const sid = record.header.id
|
|
620
|
+
const cwd = typeof record.header.cwd === 'string' ? record.header.cwd : ''
|
|
621
|
+
const wsId = workspaceForCwd(cwd)
|
|
622
|
+
if (sid === undefined || wsId === undefined) continue
|
|
623
|
+
if (!addedWs.has(wsId)) continue
|
|
624
|
+
state.knownSessionIds.add(sid)
|
|
625
|
+
scanOneSession(sid, record, wsId, state.aggregationGeneration, snapshots)
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
|
|
435
629
|
function sessionIdsFromRecords(records) {
|
|
436
630
|
const ids = new Set()
|
|
437
631
|
for (const record of records) {
|
|
438
632
|
if (record === undefined || record === null || record.header === undefined) continue
|
|
439
633
|
const sid = record.header.id
|
|
440
634
|
const cwd = typeof record.header.cwd === 'string' ? record.header.cwd : ''
|
|
441
|
-
const wsId =
|
|
635
|
+
const wsId = workspaceForCwd(cwd)
|
|
442
636
|
if (sid !== undefined && wsId !== undefined) ids.add(sid)
|
|
443
637
|
}
|
|
444
638
|
return ids
|
|
@@ -563,6 +757,18 @@ export function createSessionSync(host) {
|
|
|
563
757
|
ctx.on('session/disposed', () => {
|
|
564
758
|
if (!state.disposed) scheduleReconcileHint()
|
|
565
759
|
})
|
|
760
|
+
// Workspace registry probe: DSH emits domain/changed after every durable
|
|
761
|
+
// workspace-domain write (create/delete/rename/reorder/archive/attach/detach).
|
|
762
|
+
// On any such change we re-read workspaceRegistry.list() and, when membership
|
|
763
|
+
// changed, rescan only the affected workspaces — unchanged ones keep their
|
|
764
|
+
// computed aggregates and ledger untouched.
|
|
765
|
+
ctx.on('domain/changed', (change) => {
|
|
766
|
+
if (state.disposed) return
|
|
767
|
+
if (change === undefined || change === null) return
|
|
768
|
+
if (change.domain !== 'workspace') return
|
|
769
|
+
if (change.table !== 'workspaces' && change.table !== '') return
|
|
770
|
+
scheduleWorkspaceSync()
|
|
771
|
+
})
|
|
566
772
|
|
|
567
773
|
return {
|
|
568
774
|
foldEvent,
|
|
@@ -580,6 +786,8 @@ export function createSessionSync(host) {
|
|
|
580
786
|
scheduleNativeBaselineRetry,
|
|
581
787
|
scheduleBaselineRetry,
|
|
582
788
|
runBaseline,
|
|
789
|
+
synchronizeWorkspaceRegistry,
|
|
790
|
+
scheduleWorkspaceSync,
|
|
583
791
|
sessionIdsFromRecords,
|
|
584
792
|
reconcileSessions,
|
|
585
793
|
scheduleReconcileHint,
|
package/lib/usage-core.js
CHANGED
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
/** Canonical comparison key for Windows workspace paths. */
|
|
2
|
+
export function workspacePathKey(value) {
|
|
3
|
+
if (typeof value !== 'string') return ''
|
|
4
|
+
const normalized = value.trim().replaceAll('\\', '/').toLowerCase()
|
|
5
|
+
return normalized.length > 3 ? normalized.replace(/\/+$/, '') : normalized
|
|
6
|
+
}
|
|
7
|
+
|
|
1
8
|
const TOKEN_FIELDS = Object.freeze({
|
|
2
9
|
input: 'inputTokens',
|
|
3
10
|
output: 'outputTokens',
|
|
@@ -107,6 +114,50 @@ export function usageStepKey(sid, data, seq, fallback) {
|
|
|
107
114
|
return safeSid + ':event:' + (serialized === undefined ? String(fallback === undefined ? '' : fallback) : serialized)
|
|
108
115
|
}
|
|
109
116
|
|
|
117
|
+
/** Recalculate the billing instant for a re-priced sample, preferring the recorded request time. */
|
|
118
|
+
export function billingInstantOf(item, oldCost) {
|
|
119
|
+
const oldAt = oldCost != null && Number.isFinite(oldCost.pricingAt) ? oldCost.pricingAt : null
|
|
120
|
+
const at = Number.isFinite(item.pricingAt) ? item.pricingAt : oldAt !== null ? oldAt : validEventTime(item.time) ? item.time : null
|
|
121
|
+
const oldSource = oldCost != null && (oldCost.pricingTimeSource === 'request-context' || oldCost.pricingTimeSource === 'usage-event') ? oldCost.pricingTimeSource : null
|
|
122
|
+
const source = typeof item.pricingTimeSource === 'string' && (item.pricingTimeSource === 'request-context' || item.pricingTimeSource === 'usage-event') ? item.pricingTimeSource : (oldSource || 'usage-event')
|
|
123
|
+
return { at, source }
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** Bounded context archive shared by the live fold and the persisted ledger. */
|
|
127
|
+
export const MAX_CONTEXT_TIMES = 512
|
|
128
|
+
|
|
129
|
+
/** Insert-or-refresh a context entry with real LRU eviction (delete + set). */
|
|
130
|
+
export function touchContextTimes(times, key, time) {
|
|
131
|
+
if (typeof key !== 'string' || key === '') return
|
|
132
|
+
if (times.has(key)) times.delete(key)
|
|
133
|
+
times.set(key, time)
|
|
134
|
+
while (times.size > MAX_CONTEXT_TIMES) times.delete(times.keys().next().value)
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** Stable per-(turn, step) key for the pricing-time context of a request. */
|
|
138
|
+
export function contextTimeKey(turn, step) {
|
|
139
|
+
const safeTurn = Number.isSafeInteger(turn) && turn >= 0 ? turn : null
|
|
140
|
+
const safeStep = Number.isSafeInteger(step) && step >= 0 ? step : null
|
|
141
|
+
if (safeTurn !== null && safeStep !== null) return 'context:' + safeTurn + ':' + safeStep
|
|
142
|
+
return 'context:__latest__'
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Pick the billing instant for a usage event: the request/context time that
|
|
147
|
+
* matches the same turn/step wins (parallel requests stay per-step), otherwise
|
|
148
|
+
* the usage event time itself is the auditable fallback.
|
|
149
|
+
*/
|
|
150
|
+
export function pickPricingTime(contextTimes, eventTime, turn, step) {
|
|
151
|
+
if (contextTimes instanceof Map && validEventTime(eventTime)) {
|
|
152
|
+
const exact = contextTimes.get(contextTimeKey(turn, step))
|
|
153
|
+
const candidate = exact !== undefined ? exact : contextTimes.get(contextTimeKey(null, null))
|
|
154
|
+
if (Number.isFinite(candidate) && candidate <= eventTime) {
|
|
155
|
+
return { time: candidate, source: 'request-context' }
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
return { time: Number.isFinite(eventTime) && eventTime >= 0 ? eventTime : null, source: 'usage-event' }
|
|
159
|
+
}
|
|
160
|
+
|
|
110
161
|
/** Replace one logical sample; stale lower-seq replays are ignored. */
|
|
111
162
|
/** Event sequence contract: -1 (missing) or a non-negative safe integer. */
|
|
112
163
|
export function normalizeEventSeq(value) {
|