@ljwei-stak/dsh-model-router 0.16.0 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dsh-plugin/client.js +182 -50
- package/.dsh-plugin/index.mjs +23 -17
- package/.dsh-plugin/shared/adaptive-feedback.mjs +120 -40
- package/.dsh-plugin/shared/dynamic-data-view.mjs +59 -4
- package/.dsh-plugin/shared/dynamic-data.mjs +161 -83
- package/.dsh-plugin/shared/router-state.mjs +139 -38
- package/.dsh-plugin/shared/run-ledger.mjs +32 -11
- package/CHANGELOG.md +11 -0
- package/INSTALLATION_GUIDE.zh.md +12 -12
- package/README.md +7 -5
- package/README.zh.md +7 -5
- package/docs/ROUTING_ADAPTIVE_LEARNING.zh.md +2 -0
- package/docs/ROUTING_MATURITY.zh.md +76 -0
- package/package.json +2 -1
|
@@ -9,15 +9,16 @@
|
|
|
9
9
|
* their own outbound network policy if untrusted users can configure endpoints.
|
|
10
10
|
*/
|
|
11
11
|
import { createHash, randomUUID } from 'node:crypto'
|
|
12
|
-
import { isIP } from 'node:net'
|
|
13
12
|
import { fetchLiveBenchSnapshot } from './livebench.mjs'
|
|
14
|
-
|
|
13
|
+
import { projectDynamicData, publicDynamicEndpoint } from './dynamic-data-view.mjs'
|
|
14
|
+
export { applyPricingSnapshot, dynamicDataSummary, routingData, projectDynamicData, publicDynamicEndpoint } from './dynamic-data-view.mjs'
|
|
15
15
|
|
|
16
16
|
const DEFAULT_TTL_MS = 86_400_000
|
|
17
17
|
const TIMEOUT_MS = 8000
|
|
18
18
|
const MAX_PAYLOAD_BYTES = 4 * 1024 * 1024
|
|
19
19
|
const MAX_ROWS = 2000
|
|
20
20
|
const MAX_CLOCK_SKEW_MS = 60_000
|
|
21
|
+
const RESERVATION_GRACE_MS = 5000
|
|
21
22
|
const ROW_FIELDS = new Set(['provider', 'model', 'input', 'output', 'cacheRead', 'cacheWrite', 'currency', 'unit', 'asOf', 'sourceUrl'])
|
|
22
23
|
const PRICE_FIELDS = new Set(['schemaVersion', 'kind', 'version', 'publishedAt', 'rows'])
|
|
23
24
|
const SCORE_FIELDS = new Set(['reasoning', 'code', 'math', 'research', 'writing', 'vision', 'summarization', 'classification'])
|
|
@@ -43,16 +44,7 @@ function revisionOf(data) {
|
|
|
43
44
|
return hash({ liveBench: data.liveBench?.hash ?? null, pricing: data.pricing?.hash ?? null, status: data.status })
|
|
44
45
|
}
|
|
45
46
|
|
|
46
|
-
|
|
47
|
-
if (typeof value !== 'string' || value.length > 2048) throw new Error('invalid-public-url')
|
|
48
|
-
const url = new URL(value)
|
|
49
|
-
const host = url.hostname.toLowerCase()
|
|
50
|
-
if (url.protocol !== 'https:' || url.username || url.password || url.search || url.hash
|
|
51
|
-
|| (url.port && url.port !== '443') || isIP(host) || host.startsWith('[')
|
|
52
|
-
|| !/^(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z]{2,}$/u.test(host)
|
|
53
|
-
|| /(?:^|\.)(?:localhost|local|internal|lan|home|test|invalid)$/u.test(host)) throw new Error('invalid-public-url')
|
|
54
|
-
return url.toString()
|
|
55
|
-
}
|
|
47
|
+
const publicUrl = publicDynamicEndpoint
|
|
56
48
|
|
|
57
49
|
function identifier(value, max = 240) {
|
|
58
50
|
if (typeof value !== 'string' || value.trim() !== value || !value || value.length > max
|
|
@@ -148,59 +140,94 @@ function mayPromote(previous, candidate) {
|
|
|
148
140
|
return true
|
|
149
141
|
}
|
|
150
142
|
|
|
151
|
-
|
|
143
|
+
function cancelBody(body) {
|
|
144
|
+
try { Promise.resolve(body?.cancel?.()).catch(() => {}) } catch { /* best effort; never block rejection */ }
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
async function limitedText(response, signal) {
|
|
152
148
|
const length = Number(response.headers?.get?.('content-length'))
|
|
153
|
-
if (Number.isFinite(length) && length > MAX_PAYLOAD_BYTES)
|
|
149
|
+
if (Number.isFinite(length) && length > MAX_PAYLOAD_BYTES) {
|
|
150
|
+
cancelBody(response.body)
|
|
151
|
+
throw new Error('payload-too-large')
|
|
152
|
+
}
|
|
154
153
|
if (!response.body?.getReader) {
|
|
155
154
|
const text = await response.text()
|
|
156
155
|
if (Buffer.byteLength(text, 'utf8') > MAX_PAYLOAD_BYTES) throw new Error('payload-too-large')
|
|
157
156
|
return text
|
|
158
157
|
}
|
|
159
158
|
const reader = response.body.getReader()
|
|
160
|
-
const decoder = new TextDecoder()
|
|
159
|
+
const decoder = new TextDecoder('utf-8', { fatal: true })
|
|
161
160
|
let size = 0
|
|
162
161
|
let text = ''
|
|
162
|
+
const cancelReader = () => { try { Promise.resolve(reader.cancel()).catch(() => {}) } catch {} }
|
|
163
|
+
signal?.addEventListener('abort', cancelReader, { once: true })
|
|
163
164
|
try {
|
|
164
165
|
for (;;) {
|
|
166
|
+
if (signal?.aborted) throw new Error('refresh-aborted')
|
|
165
167
|
const chunk = await reader.read()
|
|
168
|
+
if (signal?.aborted) throw new Error('refresh-aborted')
|
|
166
169
|
if (chunk.done) return text + decoder.decode()
|
|
167
170
|
size += chunk.value.byteLength
|
|
168
171
|
if (size > MAX_PAYLOAD_BYTES) throw new Error('payload-too-large')
|
|
169
172
|
text += decoder.decode(chunk.value, { stream: true })
|
|
170
173
|
}
|
|
171
174
|
} catch (error) {
|
|
172
|
-
|
|
175
|
+
cancelReader()
|
|
173
176
|
throw error
|
|
174
|
-
} finally { reader.releaseLock() }
|
|
177
|
+
} finally { signal?.removeEventListener('abort', cancelReader); reader.releaseLock() }
|
|
175
178
|
}
|
|
176
179
|
|
|
177
|
-
function publicFetch(fetchImpl) {
|
|
180
|
+
function publicFetch(fetchImpl, outerSignal) {
|
|
178
181
|
return async (input, options = {}) => {
|
|
179
182
|
const endpoint = publicUrl(String(input))
|
|
180
|
-
const
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
}
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
183
|
+
const controller = new AbortController()
|
|
184
|
+
const signals = [...new Set([outerSignal, options.signal].filter(Boolean))]
|
|
185
|
+
const abort = () => controller.abort(new Error('refresh-aborted'))
|
|
186
|
+
const cleanup = () => { for (const signal of signals) signal.removeEventListener('abort', abort) }
|
|
187
|
+
for (const signal of signals) {
|
|
188
|
+
if (signal.aborted) abort()
|
|
189
|
+
else signal.addEventListener('abort', abort, { once: true })
|
|
190
|
+
}
|
|
191
|
+
controller.signal.addEventListener('abort', cleanup, { once: true })
|
|
192
|
+
let response
|
|
193
|
+
try {
|
|
194
|
+
if (controller.signal.aborted) throw new Error('refresh-aborted')
|
|
195
|
+
response = await fetchImpl(endpoint, {
|
|
196
|
+
method: 'GET', headers: { accept: 'application/json,text/csv,text/html' },
|
|
197
|
+
credentials: 'omit', redirect: 'error', signal: controller.signal,
|
|
198
|
+
})
|
|
199
|
+
if (controller.signal.aborted || !response?.ok || response.redirected) {
|
|
200
|
+
throw new Error('public-fetch-failed')
|
|
201
|
+
}
|
|
202
|
+
if (response.url && publicUrl(response.url) !== endpoint) {
|
|
203
|
+
throw new Error('public-fetch-redirected')
|
|
204
|
+
}
|
|
205
|
+
return { ok: true, status: response.status, headers: response.headers,
|
|
206
|
+
text: async () => { try { return await limitedText(response, controller.signal) } finally { cleanup() } } }
|
|
207
|
+
} catch (error) { cancelBody(response?.body); cleanup(); throw error }
|
|
188
208
|
}
|
|
189
209
|
}
|
|
190
210
|
|
|
191
|
-
async function withTimeout(run) {
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
timer = setTimeout(() =>
|
|
197
|
-
try {
|
|
198
|
-
|
|
211
|
+
async function withTimeout(run, controller, timeoutMs) {
|
|
212
|
+
let rejectAborted
|
|
213
|
+
const aborted = new Promise((_, reject) => { rejectAborted = reject })
|
|
214
|
+
const onAbort = () => rejectAborted(new Error('refresh-aborted'))
|
|
215
|
+
controller.signal.addEventListener('abort', onAbort, { once: true })
|
|
216
|
+
const timer = setTimeout(() => controller.abort(new Error('refresh-timeout')), timeoutMs)
|
|
217
|
+
try {
|
|
218
|
+
if (controller.signal.aborted) throw new Error('refresh-aborted')
|
|
219
|
+
return await Promise.race([run(controller.signal), aborted])
|
|
220
|
+
} finally { clearTimeout(timer); controller.signal.removeEventListener('abort', onAbort) }
|
|
199
221
|
}
|
|
200
222
|
|
|
201
223
|
function due(data, kind, endpoint, ttlMs, at, force) {
|
|
202
224
|
const status = data.status?.[kind]
|
|
203
225
|
if (force || status?.endpoint !== endpoint || !finite(status?.lastAttemptAt)) return true
|
|
226
|
+
if (at < status.lastAttemptAt) return true // a corrected clock must not freeze refresh forever
|
|
227
|
+
if (status.requestId) {
|
|
228
|
+
const deadline = finite(status.requestDeadlineAt) ? status.requestDeadlineAt : status.lastAttemptAt + TIMEOUT_MS + RESERVATION_GRACE_MS
|
|
229
|
+
return at >= deadline
|
|
230
|
+
}
|
|
204
231
|
const delay = status.error ? Math.min(ttlMs, 300_000) : ttlMs
|
|
205
232
|
return at - status.lastAttemptAt >= delay
|
|
206
233
|
}
|
|
@@ -210,11 +237,32 @@ function due(data, kind, endpoint, ttlMs, at, force) {
|
|
|
210
237
|
* mutates the fresh state under the caller's interprocess/atomic write lock.
|
|
211
238
|
*/
|
|
212
239
|
export function createDynamicDataRefresher({ store, fetchImpl = globalThis.fetch, now = Date.now,
|
|
213
|
-
fetchBenchmark = fetchLiveBenchSnapshot } = {}) {
|
|
240
|
+
fetchBenchmark = fetchLiveBenchSnapshot, timeoutMs = TIMEOUT_MS } = {}) {
|
|
214
241
|
if (typeof store?.read !== 'function' || typeof store?.update !== 'function') throw new Error('dynamic-data-store-required')
|
|
242
|
+
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > TIMEOUT_MS) throw new Error('invalid-refresh-timeout')
|
|
215
243
|
const flights = new Map()
|
|
244
|
+
const active = new Map()
|
|
245
|
+
const sourceStates = new Map()
|
|
246
|
+
const generationOf = kind => sourceStates.get(kind)?.generation ?? 0
|
|
216
247
|
const refresh = async (options = {}) => {
|
|
217
|
-
if (options.enabled !== true)
|
|
248
|
+
if (options.enabled !== true) {
|
|
249
|
+
for (const kind of ['liveBench', 'pricing']) sourceStates.set(kind, { fingerprint: '', generation: generationOf(kind) + 1 })
|
|
250
|
+
const pending = [...active.values()]
|
|
251
|
+
for (const request of pending) request.controller.abort()
|
|
252
|
+
if (pending.length) {
|
|
253
|
+
try { await store.update(state => {
|
|
254
|
+
const data = dataOf(state)
|
|
255
|
+
for (const request of pending) if (data.status[request.kind]?.requestId === request.requestId) {
|
|
256
|
+
data.status[request.kind] = { ...data.status[request.kind], disabled: true, endpoint: '', error: '' }
|
|
257
|
+
delete data.status[request.kind].requestId
|
|
258
|
+
delete data.status[request.kind].requestDeadlineAt
|
|
259
|
+
}
|
|
260
|
+
data.revision = revisionOf(data)
|
|
261
|
+
state.dynamicData = data
|
|
262
|
+
}) } catch { /* disabled projection is safe even if cancellation cannot be persisted */ }
|
|
263
|
+
}
|
|
264
|
+
return emptyData()
|
|
265
|
+
}
|
|
218
266
|
const ttlMs = finite(options.ttlMs) && options.ttlMs >= 0 ? options.ttlMs : DEFAULT_TTL_MS
|
|
219
267
|
const liveBenchEndpoint = options.liveBenchEndpoint ?? 'https://livebench.ai'
|
|
220
268
|
const pricingSnapshotEndpoint = options.pricingSnapshotEndpoint ?? ''
|
|
@@ -225,66 +273,96 @@ export function createDynamicDataRefresher({ store, fetchImpl = globalThis.fetch
|
|
|
225
273
|
try { return { kind, endpoint: publicUrl(raw), disabled: false, invalid: false } }
|
|
226
274
|
catch { return { kind, endpoint: '', disabled: false, invalid: true } }
|
|
227
275
|
})
|
|
228
|
-
const
|
|
276
|
+
for (const source of sources) {
|
|
277
|
+
const fingerprint = stable(source)
|
|
278
|
+
if (fingerprint !== sourceStates.get(source.kind)?.fingerprint) {
|
|
279
|
+
sourceStates.set(source.kind, { fingerprint, generation: generationOf(source.kind) + 1 })
|
|
280
|
+
for (const request of active.values()) if (request.kind === source.kind) request.controller.abort()
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
const generations = Object.fromEntries(sources.map(source => [source.kind, generationOf(source.kind)]))
|
|
284
|
+
const key = stable([sources, generations, ttlMs, options.force === true])
|
|
229
285
|
if (flights.has(key)) return flights.get(key)
|
|
230
286
|
const operation = (async () => {
|
|
231
287
|
const initial = dataOf(await store.read())
|
|
288
|
+
const storageFailures = new Set()
|
|
289
|
+
await Promise.all(sources.filter(source => source.disabled && (!initial.status[source.kind]?.disabled || initial.status[source.kind]?.requestId))
|
|
290
|
+
.map(async ({ kind }) => {
|
|
291
|
+
try { await store.update(state => {
|
|
292
|
+
if (generationOf(kind) !== generations[kind]) return
|
|
293
|
+
const data = dataOf(state)
|
|
294
|
+
data.status[kind] = { ...(data.status[kind] ?? {}), disabled: true, endpoint: '', error: '' }
|
|
295
|
+
delete data.status[kind].requestId
|
|
296
|
+
delete data.status[kind].requestDeadlineAt
|
|
297
|
+
data.revision = revisionOf(data)
|
|
298
|
+
state.dynamicData = data
|
|
299
|
+
}) } catch { storageFailures.add(kind) }
|
|
300
|
+
}))
|
|
232
301
|
const requests = sources.filter(source => !source.disabled)
|
|
233
302
|
.filter(({ kind, endpoint }) => due(initial, kind, endpoint, ttlMs, now(), options.force === true))
|
|
234
|
-
await Promise.
|
|
235
|
-
const at = now()
|
|
303
|
+
const settled = await Promise.allSettled(requests.map(async ({ kind, endpoint, invalid }) => {
|
|
236
304
|
const requestId = randomUUID()
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
const data = dataOf(state)
|
|
241
|
-
if (!due(data, kind, endpoint, ttlMs, at, options.force === true)) return false
|
|
242
|
-
data.status[kind] = { ...(data.status[kind] ?? {}), endpoint,
|
|
243
|
-
lastAttemptAt: at, requestId, error: '' }
|
|
244
|
-
data.revision = revisionOf(data)
|
|
245
|
-
state.dynamicData = data
|
|
246
|
-
return true
|
|
247
|
-
})
|
|
248
|
-
if (!reserved) return
|
|
249
|
-
let candidate
|
|
250
|
-
let error = ''
|
|
305
|
+
const controller = new AbortController()
|
|
306
|
+
const request = { kind, endpoint, requestId, controller }
|
|
307
|
+
active.set(requestId, request)
|
|
251
308
|
try {
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
const
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
}
|
|
260
|
-
|
|
261
|
-
|
|
309
|
+
// Reserve before network I/O. Another process or source change wins by
|
|
310
|
+
// replacing this token; a late completion then cannot mutate its data.
|
|
311
|
+
const reserved = await store.update(state => {
|
|
312
|
+
if (generationOf(kind) !== generations[kind] || controller.signal.aborted) return false
|
|
313
|
+
const data = dataOf(state)
|
|
314
|
+
const at = now()
|
|
315
|
+
if (!due(data, kind, endpoint, ttlMs, at, options.force === true)) return false
|
|
316
|
+
data.status[kind] = { ...(data.status[kind] ?? {}), disabled: false, endpoint,
|
|
317
|
+
lastAttemptAt: at, requestId, requestDeadlineAt: at + timeoutMs + RESERVATION_GRACE_MS }
|
|
318
|
+
data.revision = revisionOf(data)
|
|
319
|
+
state.dynamicData = data
|
|
320
|
+
return true
|
|
262
321
|
})
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
322
|
+
if (!reserved) return
|
|
323
|
+
let candidate
|
|
324
|
+
let error = ''
|
|
325
|
+
try {
|
|
326
|
+
if (invalid) throw new Error('invalid-public-url')
|
|
327
|
+
if (typeof fetchImpl !== 'function') throw new Error('fetch-unavailable')
|
|
328
|
+
candidate = await withTimeout(async signal => {
|
|
329
|
+
const fetchPublic = publicFetch(fetchImpl, signal)
|
|
330
|
+
if (kind === 'liveBench') {
|
|
331
|
+
const payload = await fetchBenchmark({ endpoint, fetchImpl: fetchPublic, timeoutMs, signal, strict: true })
|
|
332
|
+
return benchmarkSnapshot(payload, endpoint, now())
|
|
333
|
+
}
|
|
334
|
+
const response = await fetchPublic(endpoint, { signal })
|
|
335
|
+
return pricingSnapshot(JSON.parse(await response.text()), endpoint, now())
|
|
336
|
+
}, controller, timeoutMs)
|
|
337
|
+
} catch { error = 'refresh-failed' }
|
|
338
|
+
await store.update(state => {
|
|
339
|
+
const data = dataOf(state)
|
|
340
|
+
if (data.status[kind]?.requestId !== requestId) return false
|
|
341
|
+
if (candidate && !mayPromote(data[kind], candidate)) error = 'snapshot-rollback-rejected'
|
|
342
|
+
if (candidate && !error) data[kind] = candidate
|
|
343
|
+
data.status[kind] = { ...data.status[kind], error,
|
|
344
|
+
...(candidate && !error ? { lastSuccessAt: now() } : {}) }
|
|
345
|
+
delete data.status[kind].requestId
|
|
346
|
+
delete data.status[kind].requestDeadlineAt
|
|
347
|
+
data.revision = revisionOf(data)
|
|
348
|
+
state.dynamicData = data
|
|
349
|
+
return true
|
|
350
|
+
})
|
|
351
|
+
} finally { active.delete(requestId) }
|
|
276
352
|
}))
|
|
353
|
+
for (let index = 0; index < settled.length; index += 1) if (settled[index].status === 'rejected') storageFailures.add(requests[index].kind)
|
|
277
354
|
const result = dataOf(await store.read())
|
|
355
|
+
for (const kind of storageFailures) {
|
|
356
|
+
const endpoint = sources.find(source => source.kind === kind)?.endpoint ?? ''
|
|
357
|
+
result.status[kind] = { ...(result.status[kind]?.endpoint === endpoint ? result.status[kind] : {}),
|
|
358
|
+
endpoint, error: 'storage-failed' }
|
|
359
|
+
delete result.status[kind].requestId
|
|
360
|
+
delete result.status[kind].requestDeadlineAt
|
|
361
|
+
}
|
|
278
362
|
// A user clearing a source disables its use immediately, without erasing
|
|
279
363
|
// recoverable last-good snapshots from disk.
|
|
280
|
-
for (const { kind, disabled } of sources) {
|
|
281
|
-
if (disabled) {
|
|
282
|
-
result[kind] = null
|
|
283
|
-
result.status[kind] = { disabled: true, endpoint: '', error: '' }
|
|
284
|
-
}
|
|
285
|
-
}
|
|
286
364
|
result.revision = revisionOf(result)
|
|
287
|
-
return result
|
|
365
|
+
return projectDynamicData(result, options)
|
|
288
366
|
})()
|
|
289
367
|
flights.set(key, operation)
|
|
290
368
|
try { return await operation }
|
|
@@ -4,13 +4,15 @@
|
|
|
4
4
|
* errors, cost, review and user ratings). One JSON file under the DSH home. Never stores keys.
|
|
5
5
|
*
|
|
6
6
|
* Several Host processes may share one DSH home, so every update takes a
|
|
7
|
-
* lock file (`state.json.lock`, exclusive create; stale locks
|
|
7
|
+
* lock file (`state.json.lock`, exclusive create; only stale dead-owner locks
|
|
8
|
+
* are recovered). A busy live owner causes a bounded, explicit timeout,
|
|
8
9
|
* re-reads the file, applies the change to the fresh copy and writes it back
|
|
9
10
|
* atomically (temp file + rename). Reads re-read the file when its mtime or
|
|
10
11
|
* size changed. A file that cannot be parsed is kept as
|
|
11
12
|
* `state.json.corrupt-<timestamp>` and a notice is recorded instead of
|
|
12
13
|
* silently discarding the history.
|
|
13
14
|
*/
|
|
15
|
+
import { randomUUID } from 'node:crypto'
|
|
14
16
|
import { mkdir, open, readFile, rename, stat, unlink, writeFile } from 'node:fs/promises'
|
|
15
17
|
import { homedir } from 'node:os'
|
|
16
18
|
import { dirname, isAbsolute, join, resolve } from 'node:path'
|
|
@@ -39,7 +41,7 @@ const LOCK_STALE_MS = 15_000
|
|
|
39
41
|
const LOCK_TIMEOUT_MS = 10_000
|
|
40
42
|
|
|
41
43
|
function emptyState() {
|
|
42
|
-
return { version: STATE_VERSION, onboarding: { completedAt: null }, health: null, quota: {}, authFailures: {}, runs: [], dynamicData: { liveBench: null, pricing: null, status: {}, revision: '' }, archivedSpending: sanitizeArchivedSpending(), notices: [], terminalSessions: [] }
|
|
44
|
+
return { version: STATE_VERSION, historyIncomplete: false, onboarding: { completedAt: null }, health: null, quota: {}, authFailures: {}, runs: [], dynamicData: { liveBench: null, pricing: null, status: {}, revision: '' }, archivedSpending: sanitizeArchivedSpending(), notices: [], terminalSessions: [] }
|
|
43
45
|
}
|
|
44
46
|
|
|
45
47
|
function sanitizeNotices(value) {
|
|
@@ -55,16 +57,62 @@ function retainRuns(state, maxRuns) {
|
|
|
55
57
|
return state
|
|
56
58
|
}
|
|
57
59
|
|
|
60
|
+
const record = value => value !== null && typeof value === 'object' && !Array.isArray(value)
|
|
61
|
+
const nonnegative = value => typeof value === 'number' && Number.isFinite(value) && value >= 0
|
|
62
|
+
function invalidArchivedSpending(value) {
|
|
63
|
+
if (value === undefined || value === null) return false // old v1 files omit it
|
|
64
|
+
if (!record(value)) return true
|
|
65
|
+
for (const [field, pattern] of [['days', /^\d{4}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12]\d|3[01])$/u],
|
|
66
|
+
['months', /^\d{4}-(?:0[1-9]|1[0-2])$/u]]) {
|
|
67
|
+
if (value[field] === undefined) continue
|
|
68
|
+
if (!record(value[field])) return true
|
|
69
|
+
for (const [key, totals] of Object.entries(value[field])) {
|
|
70
|
+
if (!pattern.test(key) || !record(totals)) return true
|
|
71
|
+
if (['costUsd', 'unknown', 'subscriptionUsd', 'subscriptionRuns'].some(name => totals[name] !== undefined
|
|
72
|
+
&& (!nonnegative(totals[name]) || (['unknown', 'subscriptionRuns'].includes(name) && !Number.isInteger(totals[name]))))) return true
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return false
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function invalidRunSpending(run) {
|
|
79
|
+
if (!record(run) || invalidArchivedSpending(run.priorAttemptSpending)) return true
|
|
80
|
+
for (const field of ['packages', 'reviews']) {
|
|
81
|
+
if (run[field] === undefined) continue // missing legacy fields are not fabricated
|
|
82
|
+
if (!Array.isArray(run[field])) return true
|
|
83
|
+
for (const item of run[field]) {
|
|
84
|
+
if (!record(item) || (item.ran !== undefined && typeof item.ran !== 'boolean')) return true
|
|
85
|
+
if (['costUsd', 'referenceCostUsd'].some(name => item[name] !== undefined && item[name] !== null && !nonnegative(item[name]))) return true
|
|
86
|
+
if (item.ran === true || nonnegative(item.costUsd)) {
|
|
87
|
+
const at = item.finishedAt ?? run.createdAt
|
|
88
|
+
if (!nonnegative(at) || !Number.isFinite(new Date(at).getTime())) return true
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return false
|
|
93
|
+
}
|
|
94
|
+
|
|
58
95
|
function sanitize(value, maxRuns) {
|
|
59
|
-
if (!value || typeof value !== 'object' || value.version !== STATE_VERSION
|
|
96
|
+
if (!value || typeof value !== 'object' || Array.isArray(value) || value.version !== STATE_VERSION
|
|
97
|
+
|| !Array.isArray(value.runs)) {
|
|
98
|
+
const error = new Error('模型路由状态版本或结构不受支持;原文件未修改,请先恢复或升级兼容版本。')
|
|
99
|
+
error.code = 'ROUTER_STATE_UNSUPPORTED'
|
|
100
|
+
throw error
|
|
101
|
+
}
|
|
102
|
+
const runs = value.runs.filter(run => run && typeof run.id === 'string')
|
|
60
103
|
return retainRuns({
|
|
61
104
|
version: STATE_VERSION,
|
|
105
|
+
// Unlike bounded notices, an incomplete-history marker cannot silently
|
|
106
|
+
// expire. Restoring a verified state file is required to recover budgets.
|
|
107
|
+
historyIncomplete: value.historyIncomplete === true || runs.length !== value.runs.length
|
|
108
|
+
|| invalidArchivedSpending(value.archivedSpending) || runs.some(invalidRunSpending)
|
|
109
|
+
|| sanitizeNotices(value.notices).some(notice => notice.kind === 'state-corrupt'),
|
|
62
110
|
onboarding: { completedAt: Number.isFinite(value.onboarding?.completedAt) ? value.onboarding.completedAt : null },
|
|
63
111
|
health: value.health && Array.isArray(value.health.tools) ? value.health : null,
|
|
64
112
|
quota: value.quota && typeof value.quota === 'object' && !Array.isArray(value.quota) ? value.quota : {},
|
|
65
113
|
authFailures: Object.fromEntries(Object.entries(value.authFailures && typeof value.authFailures === 'object' && !Array.isArray(value.authFailures) ? value.authFailures : {})
|
|
66
114
|
.filter(([, entry]) => Number.isFinite(entry?.at))),
|
|
67
|
-
runs
|
|
115
|
+
runs,
|
|
68
116
|
dynamicData: value.dynamicData && typeof value.dynamicData === 'object' && !Array.isArray(value.dynamicData) ? value.dynamicData : { liveBench: null, pricing: null, status: {}, revision: '' },
|
|
69
117
|
archivedSpending: sanitizeArchivedSpending(value.archivedSpending),
|
|
70
118
|
notices: sanitizeNotices(value.notices),
|
|
@@ -93,36 +141,66 @@ export function terminalSessionRecord(record) {
|
|
|
93
141
|
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
|
|
94
142
|
const stamp = at => new Date(at).toISOString().replace(/[:.]/g, '-')
|
|
95
143
|
|
|
96
|
-
|
|
144
|
+
function ownerPid(raw) {
|
|
145
|
+
try {
|
|
146
|
+
const pid = JSON.parse(raw).pid
|
|
147
|
+
if (Number.isSafeInteger(pid) && pid > 0) return pid
|
|
148
|
+
} catch { /* support the previous pid/timestamp format */ }
|
|
149
|
+
const pid = Number(/^(\d+)\s+\d+\s*$/u.exec(raw)?.[1])
|
|
150
|
+
return Number.isSafeInteger(pid) && pid > 0 ? pid : null
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
function ownerIsDead(pid) {
|
|
154
|
+
if (pid === null) return false
|
|
155
|
+
try { process.kill(pid, 0); return false }
|
|
156
|
+
catch (error) { return error?.code === 'ESRCH' }
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/** Never steal a live or unknown owner's lock, even after the wait timeout. */
|
|
97
160
|
async function acquireLock(lockFile, { staleMs = LOCK_STALE_MS, timeoutMs = LOCK_TIMEOUT_MS, now = Date.now } = {}) {
|
|
98
|
-
|
|
161
|
+
// Elapsed waits must not freeze or jump when wall-clock time changes.
|
|
162
|
+
const started = performance.now()
|
|
99
163
|
for (let attempt = 0; ; attempt += 1) {
|
|
100
164
|
try {
|
|
101
165
|
const handle = await open(lockFile, 'wx')
|
|
102
|
-
|
|
166
|
+
const owner = `${JSON.stringify({ pid: process.pid, token: randomUUID(), at: now() })}\n`
|
|
167
|
+
try { await handle.writeFile(owner, 'utf8') }
|
|
168
|
+
catch (error) { await handle.close(); await unlink(lockFile).catch(() => {}); throw error }
|
|
103
169
|
await handle.close()
|
|
104
|
-
return async () => {
|
|
170
|
+
return async () => {
|
|
171
|
+
if (await readFile(lockFile, 'utf8').catch(() => null) === owner) await unlink(lockFile).catch(() => {})
|
|
172
|
+
}
|
|
105
173
|
} catch (error) {
|
|
106
174
|
if (error?.code !== 'EEXIST') throw error
|
|
107
175
|
const info = await stat(lockFile).catch(() => null)
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
176
|
+
const owner = await readFile(lockFile, 'utf8').catch(() => null)
|
|
177
|
+
if (info && owner !== null && now() - info.mtimeMs > staleMs && ownerIsDead(ownerPid(owner))) {
|
|
178
|
+
const currentInfo = await stat(lockFile).catch(() => null)
|
|
179
|
+
if (currentInfo?.ino === info.ino && currentInfo?.mtimeMs === info.mtimeMs
|
|
180
|
+
&& await readFile(lockFile, 'utf8').catch(() => null) === owner) {
|
|
181
|
+
await unlink(lockFile).catch(() => {})
|
|
182
|
+
continue
|
|
183
|
+
}
|
|
111
184
|
}
|
|
112
|
-
if (now() - started
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
185
|
+
if (performance.now() - started >= timeoutMs) {
|
|
186
|
+
const failure = new Error('模型路由状态正在被其他操作使用,保存未完成;请稍后重试。')
|
|
187
|
+
failure.code = 'ROUTER_STATE_LOCK_TIMEOUT'
|
|
188
|
+
throw failure
|
|
116
189
|
}
|
|
117
190
|
await sleep(Math.min(5 + attempt * 5, 50) + Math.floor(Math.random() * 10))
|
|
118
191
|
}
|
|
119
192
|
}
|
|
120
193
|
}
|
|
121
194
|
|
|
122
|
-
export function createRouterState({ file = defaultStatePath(), maxRuns = MAX_RUNS, now = Date.now
|
|
195
|
+
export function createRouterState({ file = defaultStatePath(), maxRuns = MAX_RUNS, now = Date.now,
|
|
196
|
+
lockTimeoutMs = LOCK_TIMEOUT_MS, lockStaleMs = LOCK_STALE_MS } = {}) {
|
|
197
|
+
if (![lockTimeoutMs, lockStaleMs].every(value => Number.isFinite(value) && value >= 1 && value <= 3_600_000)) {
|
|
198
|
+
throw new RangeError('Lock timing must be between 1 and 3600000 milliseconds')
|
|
199
|
+
}
|
|
123
200
|
const historyLimit = Number.isInteger(maxRuns) && maxRuns > 0 ? Math.min(maxRuns, MAX_RUNS) : MAX_RUNS
|
|
124
201
|
let cache = null
|
|
125
202
|
let cacheKey = ''
|
|
203
|
+
let cacheCorrupt = false
|
|
126
204
|
let chain = Promise.resolve()
|
|
127
205
|
let pendingNotices = []
|
|
128
206
|
const lockFile = `${file}.lock`
|
|
@@ -131,43 +209,53 @@ export function createRouterState({ file = defaultStatePath(), maxRuns = MAX_RUN
|
|
|
131
209
|
return info ? `${info.mtimeMs}:${info.size}:${info.ino}` : 'missing'
|
|
132
210
|
}
|
|
133
211
|
/** Parse the file; keep an unparseable file as a backup and record a notice. */
|
|
134
|
-
const readFresh = async () => {
|
|
212
|
+
const readFresh = async ({ recoverCorrupt = false } = {}) => {
|
|
135
213
|
let raw
|
|
136
214
|
try { raw = await readFile(file, 'utf8') }
|
|
137
215
|
catch (error) {
|
|
138
216
|
if (error?.code === 'ENOENT') return { state: emptyState(), corrupt: false }
|
|
139
217
|
throw error
|
|
140
218
|
}
|
|
141
|
-
|
|
219
|
+
let parsed
|
|
220
|
+
try { parsed = JSON.parse(raw) }
|
|
142
221
|
catch {
|
|
222
|
+
// Readers do not rename files outside the writer lock. Recheck under the
|
|
223
|
+
// lock so a concurrent valid replacement cannot be discarded as corrupt.
|
|
224
|
+
if (!recoverCorrupt) return { state: emptyState(), corrupt: true }
|
|
143
225
|
const at = now()
|
|
144
|
-
const backup = `${file}.corrupt-${stamp(at)}`
|
|
145
|
-
|
|
146
|
-
try { await rename(file, backup) } catch { saved = false }
|
|
226
|
+
const backup = `${file}.corrupt-${stamp(at)}-${randomUUID()}`
|
|
227
|
+
await rename(file, backup)
|
|
147
228
|
pendingNotices.push({
|
|
148
|
-
kind: 'state-corrupt', at, backup
|
|
149
|
-
message:
|
|
150
|
-
? `模型路由状态文件已损坏,无法读取,已备份为 ${backup} 并重新开始记录。执行历史、评价和订阅冷却信息需要从备份中人工恢复。`
|
|
151
|
-
: '模型路由状态文件已损坏,无法读取,备份失败;已重新开始记录。',
|
|
229
|
+
kind: 'state-corrupt', at, backup,
|
|
230
|
+
message: `模型路由状态文件已损坏,无法读取,已备份为 ${backup} 并重新开始记录。执行历史、评价和订阅冷却信息需要从备份中人工恢复。`,
|
|
152
231
|
})
|
|
153
|
-
return { state: emptyState(), corrupt: true }
|
|
232
|
+
return { state: { ...emptyState(), historyIncomplete: true }, corrupt: true }
|
|
154
233
|
}
|
|
234
|
+
// Valid JSON with an unknown schema is not corrupt: preserve it exactly,
|
|
235
|
+
// and refuse both reads-as-empty and writes that would downgrade it.
|
|
236
|
+
return { state: sanitize(parsed, historyLimit), corrupt: false }
|
|
155
237
|
}
|
|
156
238
|
const load = async () => {
|
|
157
239
|
const key = await fileKey()
|
|
158
240
|
if (cache && key === cacheKey) return cache
|
|
159
241
|
const fresh = await readFresh()
|
|
160
242
|
cache = fresh.state
|
|
161
|
-
cacheKey =
|
|
243
|
+
cacheKey = key
|
|
244
|
+
cacheCorrupt = fresh.corrupt
|
|
162
245
|
return cache
|
|
163
246
|
}
|
|
164
247
|
const persist = async state => {
|
|
165
248
|
await mkdir(dirname(file), { recursive: true })
|
|
249
|
+
const committed = structuredClone(state)
|
|
250
|
+
const serialized = `${JSON.stringify(committed)}\n`
|
|
166
251
|
const temp = `${file}.${process.pid}.${now()}.${Math.random().toString(36).slice(2, 8)}.tmp`
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
252
|
+
try {
|
|
253
|
+
await writeFile(temp, serialized, 'utf8')
|
|
254
|
+
await rename(temp, file)
|
|
255
|
+
} finally { await unlink(temp).catch(() => {}) }
|
|
256
|
+
cache = committed
|
|
170
257
|
cacheKey = await fileKey()
|
|
258
|
+
cacheCorrupt = false
|
|
171
259
|
}
|
|
172
260
|
/**
|
|
173
261
|
* Serialize read-modify-write updates within this process and, through the
|
|
@@ -176,17 +264,26 @@ export function createRouterState({ file = defaultStatePath(), maxRuns = MAX_RUN
|
|
|
176
264
|
const update = change => {
|
|
177
265
|
const next = chain.then(async () => {
|
|
178
266
|
await mkdir(dirname(file), { recursive: true })
|
|
179
|
-
const release = await acquireLock(lockFile, { now })
|
|
267
|
+
const release = await acquireLock(lockFile, { now, timeoutMs: lockTimeoutMs, staleMs: lockStaleMs })
|
|
180
268
|
try {
|
|
181
269
|
cache = null
|
|
182
|
-
const state = await
|
|
183
|
-
if (pendingNotices.length)
|
|
270
|
+
const { state } = await readFresh({ recoverCorrupt: true })
|
|
271
|
+
if (pendingNotices.length) state.notices.push(...pendingNotices)
|
|
184
272
|
const result = await change(state)
|
|
273
|
+
const returned = result === undefined ? undefined : structuredClone(result)
|
|
185
274
|
retainRuns(state, historyLimit)
|
|
186
275
|
state.notices = sanitizeNotices(state.notices)
|
|
187
276
|
state.terminalSessions = sanitizeTerminalSessions(state.terminalSessions)
|
|
188
277
|
await persist(state)
|
|
189
|
-
|
|
278
|
+
pendingNotices = []
|
|
279
|
+
return returned
|
|
280
|
+
} catch (error) {
|
|
281
|
+
// A rejected callback or disk write must not leak uncommitted state
|
|
282
|
+
// into the reader cache, including partial rating/budget changes.
|
|
283
|
+
cache = null
|
|
284
|
+
cacheKey = ''
|
|
285
|
+
cacheCorrupt = false
|
|
286
|
+
throw error
|
|
190
287
|
} finally {
|
|
191
288
|
await release()
|
|
192
289
|
}
|
|
@@ -198,9 +295,13 @@ export function createRouterState({ file = defaultStatePath(), maxRuns = MAX_RUN
|
|
|
198
295
|
const read = async () => {
|
|
199
296
|
await chain
|
|
200
297
|
const state = await load()
|
|
201
|
-
if (pendingNotices.length) {
|
|
298
|
+
if (cacheCorrupt || pendingNotices.length) {
|
|
202
299
|
const notices = [...pendingNotices]
|
|
203
|
-
try { await update(() => {}) }
|
|
300
|
+
try { await update(() => {}) }
|
|
301
|
+
catch {
|
|
302
|
+
return structuredClone({ ...state, notices: [...state.notices, ...notices,
|
|
303
|
+
{ kind: 'state-recovery-pending', at: now(), message: '模型路由状态恢复未完成;原文件保留,请稍后重试。' }] })
|
|
304
|
+
}
|
|
204
305
|
return structuredClone(cache)
|
|
205
306
|
}
|
|
206
307
|
return structuredClone(state)
|
|
@@ -233,10 +334,10 @@ export function createRouterState({ file = defaultStatePath(), maxRuns = MAX_RUN
|
|
|
233
334
|
return entry
|
|
234
335
|
}),
|
|
235
336
|
appendRun: run => update(current => { current.runs.push(run); return run }),
|
|
236
|
-
updateRun: (id, change) => update(current => {
|
|
337
|
+
updateRun: (id, change) => update(async current => {
|
|
237
338
|
const run = current.runs.find(item => item.id === id)
|
|
238
339
|
if (!run) throw new Error(`未找到运行记录 ${id}`)
|
|
239
|
-
change(run)
|
|
340
|
+
await change(run)
|
|
240
341
|
return structuredClone(run)
|
|
241
342
|
}),
|
|
242
343
|
/** Remove one notice (for example after the user read the corrupt-file warning). */
|