@ljwei-stak/dsh-model-router 0.23.1 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dsh-plugin/client.js +83 -34
- package/.dsh-plugin/index.mjs +281 -51
- package/.dsh-plugin/shared/budget-reservations.mjs +254 -0
- package/.dsh-plugin/shared/execution-lifecycle.mjs +141 -0
- package/.dsh-plugin/shared/official-team-runtime.mjs +37 -13
- package/.dsh-plugin/shared/official-tools-runtime.mjs +3 -2
- package/.dsh-plugin/shared/router-state.mjs +23 -8
- package/.dsh-plugin/shared/run-ledger.mjs +6 -1
- package/.dsh-plugin/shared/task-executors.mjs +32 -3
- package/.dsh-plugin/shared/zcode-bundle.mjs +3 -3
- package/CHANGELOG.md +13 -0
- package/README.i18n.yaml +2 -2
- package/README.md +25 -17
- package/README.zh.md +24 -16
- package/docs/assets/workbench-budget-reservations.png +0 -0
- package/docs/releases/0.24.0-validation.md +41 -0
- package/package.json +7 -3
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Durable execution holds. Every transition uses router-state's one file-lock
|
|
3
|
+
* transaction, so another Host cannot approve against an out-of-date balance.
|
|
4
|
+
* A hold is a remaining estimate, never an invoice or proof of a zero charge.
|
|
5
|
+
*/
|
|
6
|
+
import { randomUUID } from 'node:crypto'
|
|
7
|
+
import { budgetCheck, spending } from './run-ledger.mjs'
|
|
8
|
+
|
|
9
|
+
export const MAX_EXECUTION_RESERVATIONS = 100
|
|
10
|
+
const record = value => value !== null && typeof value === 'object' && !Array.isArray(value)
|
|
11
|
+
const amount = value => typeof value === 'number' && Number.isFinite(value) && value >= 0
|
|
12
|
+
const identifier = value => typeof value === 'string' && value.length > 0 && value.length <= 512 && !/[\u0000-\u001f\u007f]/u.test(value)
|
|
13
|
+
const timestamp = value => amount(value) && Number.isFinite(new Date(value).getTime())
|
|
14
|
+
|
|
15
|
+
/** Unknown/corrupt holds make history incomplete; silently losing them is unsafe. */
|
|
16
|
+
export function sanitizeBudgetReservations(value) {
|
|
17
|
+
if (value === undefined) return { reservations: {}, invalid: false }
|
|
18
|
+
if (!record(value)) return { reservations: {}, invalid: true }
|
|
19
|
+
const reservations = {}
|
|
20
|
+
let invalid = false
|
|
21
|
+
for (const [key, entry] of Object.entries(value)) {
|
|
22
|
+
if (!identifier(key) || !record(entry) || entry.id !== key || entry.runId !== key
|
|
23
|
+
|| !identifier(entry.ownerToken) || !Number.isSafeInteger(entry.ownerPid) || entry.ownerPid < 1
|
|
24
|
+
|| !['active', 'interrupted'].includes(entry.status) || !timestamp(entry.createdAt) || !timestamp(entry.updatedAt)
|
|
25
|
+
|| !Array.isArray(entry.packages) || !Array.isArray(entry.completedIds)
|
|
26
|
+
|| entry.completedIds.some(id => !identifier(id))
|
|
27
|
+
|| entry.packages.some(item => !record(item) || !identifier(item.id)
|
|
28
|
+
|| (item.estimatedCostUsd !== null && !amount(item.estimatedCostUsd)) || typeof item.started !== 'boolean')
|
|
29
|
+
|| new Set(entry.packages.map(item => item.id)).size !== entry.packages.length
|
|
30
|
+
|| entry.packages.some(item => entry.completedIds.includes(item.id))) {
|
|
31
|
+
invalid = true
|
|
32
|
+
continue
|
|
33
|
+
}
|
|
34
|
+
reservations[key] = {
|
|
35
|
+
id: key, runId: key, ownerToken: entry.ownerToken, ownerPid: entry.ownerPid,
|
|
36
|
+
status: entry.status, createdAt: entry.createdAt, updatedAt: entry.updatedAt,
|
|
37
|
+
...(timestamp(entry.interruptedAt) ? { interruptedAt: entry.interruptedAt } : {}),
|
|
38
|
+
packages: entry.packages.map(item => ({ id: item.id, estimatedCostUsd: item.estimatedCostUsd, started: item.started })),
|
|
39
|
+
completedIds: [...new Set(entry.completedIds)],
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
return { reservations, invalid }
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** Live and unreconciled interrupted work occupies today's and this month's room. */
|
|
46
|
+
export function reservationSpending(state, { excludeReservationId = null } = {}) {
|
|
47
|
+
let reservedUsd = 0
|
|
48
|
+
let unknownReservations = 0
|
|
49
|
+
for (const entry of Object.values(state?.budgetReservations ?? {})) {
|
|
50
|
+
if (!record(entry) || entry.id === excludeReservationId) continue
|
|
51
|
+
for (const item of Array.isArray(entry.packages) ? entry.packages : []) {
|
|
52
|
+
if (amount(item.estimatedCostUsd)) reservedUsd += item.estimatedCostUsd
|
|
53
|
+
if (!amount(item.estimatedCostUsd) || (entry.status === 'interrupted' && item.started)) unknownReservations += 1
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
return { today: reservedUsd, month: reservedUsd, unknownToday: unknownReservations,
|
|
57
|
+
unknownMonth: unknownReservations, reservedUsd, unknownReservations }
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function normalizedEstimates(value, run) {
|
|
61
|
+
const values = value ?? (run?.packages ?? []).map(item => ({ id: item.id, estimatedCostUsd: item.estimatedCost ?? null }))
|
|
62
|
+
if (!Array.isArray(values) || values.some(item => !record(item) || !identifier(item.id))) {
|
|
63
|
+
throw new TypeError('工作包预占必须包含有效的工作包 ID。')
|
|
64
|
+
}
|
|
65
|
+
const seen = new Set()
|
|
66
|
+
return values.map(item => {
|
|
67
|
+
if (seen.has(item.id)) throw new TypeError('工作包预占 ID 不能重复。')
|
|
68
|
+
seen.add(item.id)
|
|
69
|
+
const estimatedCostUsd = Object.hasOwn(item, 'estimatedCostUsd') ? item.estimatedCostUsd : item.estimatedCost ?? null
|
|
70
|
+
if (estimatedCostUsd !== null && !amount(estimatedCostUsd)) throw new TypeError('工作包预估费用必须是非负有限数字或 null。')
|
|
71
|
+
return { id: item.id, estimatedCostUsd, started: item.started === true }
|
|
72
|
+
})
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function checkedRun(run, id = run?.id) {
|
|
76
|
+
if (!record(run) || !identifier(id) || run.id !== id || !Array.isArray(run.packages)) {
|
|
77
|
+
throw new TypeError('执行检查点必须包含对应的完整运行记录。')
|
|
78
|
+
}
|
|
79
|
+
return structuredClone(run)
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function upsertRun(current, run) {
|
|
83
|
+
const index = current.runs.findIndex(item => item.id === run.id)
|
|
84
|
+
if (index < 0) current.runs.push(run)
|
|
85
|
+
else current.runs[index] = run
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function ownedReservation(current, reservationId, ownerToken) {
|
|
89
|
+
const entry = current.budgetReservations?.[reservationId]
|
|
90
|
+
if (!entry || entry.ownerToken !== ownerToken || entry.status !== 'active') {
|
|
91
|
+
const error = new Error('执行预占不存在、已中断或属于其他执行;原记录未修改。')
|
|
92
|
+
error.code = 'ROUTER_RESERVATION_OWNERSHIP'
|
|
93
|
+
throw error
|
|
94
|
+
}
|
|
95
|
+
return entry
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function estimateTotal(packages) {
|
|
99
|
+
if (packages.some(item => item.estimatedCostUsd === null)) return null
|
|
100
|
+
const total = packages.reduce((sum, item) => sum + item.estimatedCostUsd, 0)
|
|
101
|
+
return Number.isFinite(total) ? total : null
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function currentBudget(current, at, packages, options, excludeReservationId = null) {
|
|
105
|
+
const recorded = spending(current.runs, at, current.archivedSpending)
|
|
106
|
+
const holds = reservationSpending(current, { excludeReservationId })
|
|
107
|
+
const spent = { ...recorded, today: recorded.today + holds.today, month: recorded.month + holds.month,
|
|
108
|
+
unknownToday: recorded.unknownToday + holds.unknownToday, unknownMonth: recorded.unknownMonth + holds.unknownMonth }
|
|
109
|
+
const budget = budgetCheck({ estimateUsd: estimateTotal(packages), spent,
|
|
110
|
+
dailyLimitUsd: options.dailyLimitUsd, monthlyLimitUsd: options.monthlyLimitUsd })
|
|
111
|
+
const unknownBudget = budget.limited && (!budget.estimateKnown
|
|
112
|
+
|| (options.dailyLimitUsd > 0 && spent.unknownToday > 0)
|
|
113
|
+
|| (options.monthlyLimitUsd > 0 && spent.unknownMonth > 0))
|
|
114
|
+
if (unknownBudget && !budget.exceeded) budget.message = '存在未核对费用或缺少单价,无法保证本次执行符合预算;请先核对账单或明确确认继续。'
|
|
115
|
+
return { budget, spent, recorded, unknownBudget, ...holds }
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** false means definitely dead; permission failures and PID reuse stay conservative. */
|
|
119
|
+
function processAlive(pid) {
|
|
120
|
+
try { process.kill(pid, 0); return true }
|
|
121
|
+
catch (error) { return error?.code === 'ESRCH' ? false : null }
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
export function createBudgetReservations({ state, now = Date.now, pid = process.pid, isProcessAlive = processAlive,
|
|
125
|
+
maxReservations = MAX_EXECUTION_RESERVATIONS } = {}) {
|
|
126
|
+
if (!state || typeof state.update !== 'function' || typeof state.read !== 'function') throw new TypeError('需要持久状态存储。')
|
|
127
|
+
if (!Number.isSafeInteger(pid) || pid < 1) throw new TypeError('需要有效的执行进程 ID。')
|
|
128
|
+
if (!Number.isInteger(maxReservations) || maxReservations < 1 || maxReservations > MAX_EXECUTION_RESERVATIONS) throw new RangeError('执行预占数量上限无效。')
|
|
129
|
+
const snapshot = async () => reservationSpending(await state.read())
|
|
130
|
+
const beginExecution = options => {
|
|
131
|
+
const run = checkedRun(options?.run)
|
|
132
|
+
const ownerToken = options.ownerToken ?? randomUUID()
|
|
133
|
+
if (!identifier(ownerToken)) throw new TypeError('需要有效的执行所有权标记。')
|
|
134
|
+
const packages = normalizedEstimates(options.packageEstimates, run)
|
|
135
|
+
return state.update(current => {
|
|
136
|
+
const checks = currentBudget(current, now(), packages, options)
|
|
137
|
+
if (current.historyIncomplete) return { accepted: false, reason: 'history-incomplete', ...checks }
|
|
138
|
+
current.budgetReservations ??= {}
|
|
139
|
+
if (current.budgetReservations[run.id]) return { accepted: false, reason: 'active-run', ...checks }
|
|
140
|
+
if (Object.keys(current.budgetReservations).length >= maxReservations) return { accepted: false, reason: 'reservation-capacity', ...checks }
|
|
141
|
+
if (checks.budget.exceeded && options.confirmOverBudget !== true) return { accepted: false, reason: 'budget', ...checks }
|
|
142
|
+
if (checks.unknownBudget && options.confirmOverBudget !== true) return { accepted: false, reason: 'budget-unknown', ...checks }
|
|
143
|
+
const at = now()
|
|
144
|
+
const entry = { id: run.id, runId: run.id, ownerToken, ownerPid: pid, status: 'active',
|
|
145
|
+
createdAt: at, updatedAt: at, packages, completedIds: [] }
|
|
146
|
+
current.budgetReservations[run.id] = entry
|
|
147
|
+
upsertRun(current, run)
|
|
148
|
+
return { accepted: true, reservationId: entry.id, ownerToken, run, ...checks }
|
|
149
|
+
})
|
|
150
|
+
}
|
|
151
|
+
const checkpointExecution = ({ reservationId, ownerToken, run: nextRun, completedIds = [], startedIds = [],
|
|
152
|
+
checkBudget = false, budgetRun: affordabilityRun, ...options }) => {
|
|
153
|
+
const run = checkedRun(nextRun, reservationId)
|
|
154
|
+
const budgetRun = affordabilityRun === undefined ? run : checkedRun(affordabilityRun, reservationId)
|
|
155
|
+
if (![completedIds, startedIds].every(ids => Array.isArray(ids) && ids.every(identifier))) throw new TypeError('检查点工作包 ID 无效。')
|
|
156
|
+
return state.update(current => {
|
|
157
|
+
const entry = ownedReservation(current, reservationId, ownerToken)
|
|
158
|
+
const completed = new Set([...entry.completedIds, ...completedIds])
|
|
159
|
+
const started = new Set(startedIds)
|
|
160
|
+
const known = new Set([...entry.completedIds, ...entry.packages.map(item => item.id)])
|
|
161
|
+
if ([...completed, ...started].some(id => !known.has(id))) throw new TypeError('检查点包含未预占的工作包。')
|
|
162
|
+
const remaining = entry.packages.filter(item => !completed.has(item.id)).map(item => ({ ...item, started: item.started || started.has(item.id) }))
|
|
163
|
+
if (checkBudget) {
|
|
164
|
+
// Receipts may exceed their original estimates, or another execution
|
|
165
|
+
// may have settled meanwhile. Recheck the full remaining occupancy at
|
|
166
|
+
// this dispatch boundary without double-counting our previous hold.
|
|
167
|
+
const prospective = { ...current, runs: current.runs.map(item => item.id === run.id ? budgetRun : item) }
|
|
168
|
+
const checks = currentBudget(prospective, now(), remaining, options, entry.id)
|
|
169
|
+
if (current.historyIncomplete) return { accepted: false, reason: 'history-incomplete', ...checks }
|
|
170
|
+
if (checks.budget.exceeded && options.confirmOverBudget !== true) return { accepted: false, reason: 'budget', ...checks }
|
|
171
|
+
if (checks.unknownBudget && options.confirmOverBudget !== true) return { accepted: false, reason: 'budget-unknown', ...checks }
|
|
172
|
+
}
|
|
173
|
+
entry.packages = remaining
|
|
174
|
+
entry.completedIds = [...completed]
|
|
175
|
+
entry.updatedAt = now()
|
|
176
|
+
upsertRun(current, run)
|
|
177
|
+
return { accepted: true, reservationId, run, ...reservationSpending(current) }
|
|
178
|
+
})
|
|
179
|
+
}
|
|
180
|
+
const resizeExecution = options => {
|
|
181
|
+
const packages = normalizedEstimates(options.packageEstimates)
|
|
182
|
+
return state.update(current => {
|
|
183
|
+
const entry = ownedReservation(current, options.reservationId, options.ownerToken)
|
|
184
|
+
const checks = currentBudget(current, now(), packages, options, entry.id)
|
|
185
|
+
if (current.historyIncomplete) return { accepted: false, reason: 'history-incomplete', ...checks }
|
|
186
|
+
if (checks.budget.exceeded && options.confirmOverBudget !== true) return { accepted: false, reason: 'budget', ...checks }
|
|
187
|
+
if (checks.unknownBudget && options.confirmOverBudget !== true) return { accepted: false, reason: 'budget-unknown', ...checks }
|
|
188
|
+
if (packages.some(item => entry.completedIds.includes(item.id))) throw new TypeError('已完成工作包不能重新加入剩余预占。')
|
|
189
|
+
// An already-started call may still incur charges; resizing cannot erase it.
|
|
190
|
+
for (const previous of entry.packages.filter(item => item.started)) {
|
|
191
|
+
const next = packages.find(item => item.id === previous.id)
|
|
192
|
+
if (!next) throw new TypeError('执行中的工作包必须先保存结果,才能释放其预占。')
|
|
193
|
+
next.started = true
|
|
194
|
+
}
|
|
195
|
+
entry.packages = packages
|
|
196
|
+
entry.updatedAt = now()
|
|
197
|
+
return { accepted: true, reservationId: entry.id, ...checks }
|
|
198
|
+
})
|
|
199
|
+
}
|
|
200
|
+
const finishExecution = ({ reservationId, ownerToken, run: nextRun }) => {
|
|
201
|
+
const run = checkedRun(nextRun, reservationId)
|
|
202
|
+
return state.update(current => {
|
|
203
|
+
ownedReservation(current, reservationId, ownerToken)
|
|
204
|
+
upsertRun(current, run)
|
|
205
|
+
delete current.budgetReservations[reservationId]
|
|
206
|
+
return { reservationId, run, ...reservationSpending(current) }
|
|
207
|
+
})
|
|
208
|
+
}
|
|
209
|
+
const markInterrupted = (current, entry, run, at) => {
|
|
210
|
+
entry.status = 'interrupted'
|
|
211
|
+
entry.interruptedAt = at
|
|
212
|
+
entry.updatedAt = at
|
|
213
|
+
if (run) {
|
|
214
|
+
run.status = 'interrupted'
|
|
215
|
+
run.finishedAt = at
|
|
216
|
+
for (const pending of entry.packages.filter(item => item.started)) {
|
|
217
|
+
const item = [...run.packages, ...(run.reviews ?? [])].find(item => item.id === pending.id)
|
|
218
|
+
if (!item || (!item.inFlight && !['pending', 'running'].includes(item.status))
|
|
219
|
+
|| amount(item.costUsd) || amount(item.knownCostUsd) || Array.isArray(item.qualityAttempts)) continue
|
|
220
|
+
// We know a request began, but never observed its receipt. Keep that
|
|
221
|
+
// uncertainty in the ledger as well as the durable estimate hold.
|
|
222
|
+
item.status = 'interrupted'
|
|
223
|
+
item.inFlight = false
|
|
224
|
+
item.ran = true
|
|
225
|
+
item.costUsd = null
|
|
226
|
+
item.costSource = 'interrupted-usage-unknown'
|
|
227
|
+
item.billing = item.billing === 'subscription' ? 'subscription' : 'api'
|
|
228
|
+
item.finishedAt = at
|
|
229
|
+
}
|
|
230
|
+
run.interruption = { at, unreconciled: true, pendingPackageIds: entry.packages.map(item => item.id),
|
|
231
|
+
startedPackageIds: entry.packages.filter(item => item.started).map(item => item.id) }
|
|
232
|
+
} else current.historyIncomplete = true
|
|
233
|
+
}
|
|
234
|
+
const interruptExecution = ({ reservationId, ownerToken, run: nextRun }) => {
|
|
235
|
+
const run = checkedRun(nextRun, reservationId)
|
|
236
|
+
return state.update(current => {
|
|
237
|
+
const entry = ownedReservation(current, reservationId, ownerToken)
|
|
238
|
+
markInterrupted(current, entry, run, now())
|
|
239
|
+
upsertRun(current, run)
|
|
240
|
+
return { reservationId, run, ...reservationSpending(current) }
|
|
241
|
+
})
|
|
242
|
+
}
|
|
243
|
+
const recoverInterruptedExecutions = () => state.update(async current => {
|
|
244
|
+
const recovered = []
|
|
245
|
+
for (const entry of Object.values(current.budgetReservations ?? {})) {
|
|
246
|
+
if (entry.status !== 'active' || await isProcessAlive(entry.ownerPid) !== false) continue
|
|
247
|
+
const run = current.runs.find(item => item.id === entry.runId)
|
|
248
|
+
markInterrupted(current, entry, run, now())
|
|
249
|
+
recovered.push({ reservationId: entry.id, runId: entry.runId, ...entry })
|
|
250
|
+
}
|
|
251
|
+
return { recovered, ...reservationSpending(current) }
|
|
252
|
+
})
|
|
253
|
+
return { beginExecution, checkpointExecution, resizeExecution, finishExecution, interruptExecution, recoverInterruptedExecutions, snapshot }
|
|
254
|
+
}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
/** Required durable execution state, independent of advisory UI observers. */
|
|
2
|
+
import { randomUUID } from 'node:crypto'
|
|
3
|
+
import { createBudgetReservations } from './budget-reservations.mjs'
|
|
4
|
+
|
|
5
|
+
const receiptTimestamp = value => typeof value === 'number' && Number.isFinite(value) && value >= 0
|
|
6
|
+
&& Number.isFinite(new Date(value).getTime())
|
|
7
|
+
|
|
8
|
+
export async function beginExecutionLifecycle({ state, buildRun, packageEstimates, settings = {}, confirmOverBudget = false }) {
|
|
9
|
+
const ownerToken = randomUUID()
|
|
10
|
+
const manager = createBudgetReservations({ state })
|
|
11
|
+
await manager.recoverInterruptedExecutions()
|
|
12
|
+
const started = new Map(), results = new Map(), completed = new Set()
|
|
13
|
+
const estimates = new Map(packageEstimates.map(item => [item.id, item.estimatedCostUsd ?? item.estimatedCost ?? null]))
|
|
14
|
+
let workspace = {}, poisoned = null, closed = false
|
|
15
|
+
const current = (status = 'running') => {
|
|
16
|
+
const receipts = [...started].map(([id, item]) => results.get(id) ?? {
|
|
17
|
+
...item, id, ok: false, notStarted: true, answer: '', channel: item.channel ?? null,
|
|
18
|
+
})
|
|
19
|
+
for (const [id, result] of results) if (!started.has(id)) receipts.push(result)
|
|
20
|
+
const run = buildRun({ status, results: receipts, workspace })
|
|
21
|
+
if (status === 'running' || status === 'interrupted') {
|
|
22
|
+
run.status = status
|
|
23
|
+
if (status === 'running') run.finishedAt = null
|
|
24
|
+
for (const item of run.packages ?? []) {
|
|
25
|
+
if (!started.has(item.id) || completed.has(item.id)) continue
|
|
26
|
+
item.status = 'running'; item.inFlight = true
|
|
27
|
+
if (!results.has(item.id)) Object.assign(item, { ran: false, costUsd: null, costSource: 'pending-receipt' })
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
run.checkpoint = { at: Date.now(), completedIds: [...completed], startedIds: [...started.keys()] }
|
|
31
|
+
return run
|
|
32
|
+
}
|
|
33
|
+
const initial = current()
|
|
34
|
+
const accepted = await manager.beginExecution({ run: initial, packageEstimates, ownerToken, ...settings, confirmOverBudget })
|
|
35
|
+
if (!accepted.accepted) return accepted
|
|
36
|
+
const identity = { reservationId: accepted.reservationId, ownerToken }
|
|
37
|
+
const rejectBudget = changed => {
|
|
38
|
+
const error = new Error(changed.budget?.message || '费用或预占已变化,已暂停启动后续步骤,请重新核对预算。')
|
|
39
|
+
error.code = 'ROUTER_BUDGET_CHANGED'
|
|
40
|
+
error.budget = { ...changed.budget, paused: true, exceeded: changed.budget?.exceeded ?? 'unknown' }
|
|
41
|
+
error.runId = initial.id
|
|
42
|
+
error.reason = changed.reason
|
|
43
|
+
throw error
|
|
44
|
+
}
|
|
45
|
+
const required = async operation => {
|
|
46
|
+
if (poisoned) throw poisoned
|
|
47
|
+
try { return await operation() }
|
|
48
|
+
catch (cause) {
|
|
49
|
+
poisoned = new Error('执行检查点保存失败,已停止启动后续步骤;请核对历史记录与费用。', { cause })
|
|
50
|
+
poisoned.code = 'ROUTER_CHECKPOINT_FAILED'
|
|
51
|
+
throw poisoned
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
const save = run => required(() => manager.checkpointExecution({ ...identity, run,
|
|
55
|
+
completedIds: [...completed], startedIds: [...started.keys()] }))
|
|
56
|
+
const preserveReceiptTimes = run => {
|
|
57
|
+
for (const item of run.packages ?? []) {
|
|
58
|
+
const receipt = results.get(item.id)
|
|
59
|
+
if (receiptTimestamp(receipt?.finishedAt)) item.finishedAt = receipt.finishedAt
|
|
60
|
+
}
|
|
61
|
+
return run
|
|
62
|
+
}
|
|
63
|
+
return {
|
|
64
|
+
...accepted,
|
|
65
|
+
id: initial.id,
|
|
66
|
+
current,
|
|
67
|
+
results: () => [...results.values()],
|
|
68
|
+
activePackageId: () => [...started.keys()].findLast(id => !completed.has(id)),
|
|
69
|
+
async onCheckpointStart(item) {
|
|
70
|
+
const previous = started.get(item.id)
|
|
71
|
+
const alreadyStarted = started.has(item.id)
|
|
72
|
+
started.set(item.id, item)
|
|
73
|
+
const changed = await required(() => manager.checkpointExecution({ ...identity, run: current(),
|
|
74
|
+
completedIds: [...completed], startedIds: [...started.keys()], checkBudget: true, ...settings, confirmOverBudget }))
|
|
75
|
+
if (changed.accepted === false) {
|
|
76
|
+
if (alreadyStarted) started.set(item.id, previous)
|
|
77
|
+
else started.delete(item.id)
|
|
78
|
+
rejectBudget(changed)
|
|
79
|
+
}
|
|
80
|
+
},
|
|
81
|
+
async onCheckpointResult(result) {
|
|
82
|
+
// Mutate the executor's receipt so later aggregate/final builds use the
|
|
83
|
+
// same observed settlement time, even after another day or month starts.
|
|
84
|
+
if (!receiptTimestamp(result.finishedAt)) result.finishedAt = Date.now()
|
|
85
|
+
results.set(result.id, result); completed.add(result.id); await save(current())
|
|
86
|
+
},
|
|
87
|
+
async onWorkspace(value) { workspace = { ...workspace, ...value }; await save(current()) },
|
|
88
|
+
async partialResult(result, { checkBudget = false } = {}) {
|
|
89
|
+
// The next attempt's unknown receipt is durable dispatch evidence, not
|
|
90
|
+
// an already-observed unknown invoice. Authorize against the previous
|
|
91
|
+
// observed receipt plus its newly reserved estimate, then save both in
|
|
92
|
+
// the same locked transaction before starting that attempt.
|
|
93
|
+
const budgetRun = checkBudget ? current() : undefined
|
|
94
|
+
const hadResult = results.has(result.id), previous = results.get(result.id)
|
|
95
|
+
if (!receiptTimestamp(result.finishedAt)) result.finishedAt = Date.now()
|
|
96
|
+
results.set(result.id, result)
|
|
97
|
+
const changed = await required(() => manager.checkpointExecution({ ...identity, run: current(), budgetRun,
|
|
98
|
+
completedIds: [...completed], startedIds: [...started.keys()], checkBudget, ...settings, confirmOverBudget: false }))
|
|
99
|
+
if (changed.accepted === false) {
|
|
100
|
+
if (hadResult) results.set(result.id, previous)
|
|
101
|
+
else results.delete(result.id)
|
|
102
|
+
rejectBudget(changed)
|
|
103
|
+
}
|
|
104
|
+
},
|
|
105
|
+
async saveRun(run, { completedIds = [], startedIds = [] } = {}) {
|
|
106
|
+
const previousCompleted = new Set(completed), previousStarted = new Map(started)
|
|
107
|
+
for (const id of completedIds) completed.add(id)
|
|
108
|
+
for (const id of startedIds) if (!started.has(id)) started.set(id, { id })
|
|
109
|
+
const changed = await required(() => manager.checkpointExecution({ ...identity, run,
|
|
110
|
+
completedIds: [...completed], startedIds: [...started.keys()], checkBudget: startedIds.length > 0,
|
|
111
|
+
...settings, confirmOverBudget: false }))
|
|
112
|
+
if (changed.accepted === false) {
|
|
113
|
+
completed.clear(); for (const id of previousCompleted) completed.add(id)
|
|
114
|
+
started.clear(); for (const [id, item] of previousStarted) started.set(id, item)
|
|
115
|
+
rejectBudget(changed)
|
|
116
|
+
}
|
|
117
|
+
},
|
|
118
|
+
async reserveExtra(id, estimatedCostUsd, { confirmOverBudget: extraConfirmation = false } = {}) {
|
|
119
|
+
return required(async () => {
|
|
120
|
+
const next = new Map(estimates); next.set(id, estimatedCostUsd)
|
|
121
|
+
const changed = await manager.resizeExecution({ ...identity,
|
|
122
|
+
packageEstimates: [...next].filter(([key]) => !completed.has(key)).map(([key, value]) => ({ id: key, estimatedCostUsd: value })),
|
|
123
|
+
...settings, confirmOverBudget: extraConfirmation === true })
|
|
124
|
+
if (changed.accepted) { estimates.clear(); for (const [key, value] of next) estimates.set(key, value) }
|
|
125
|
+
return changed
|
|
126
|
+
})
|
|
127
|
+
},
|
|
128
|
+
async finish(run) {
|
|
129
|
+
if (poisoned) throw poisoned
|
|
130
|
+
const value = await required(() => manager.finishExecution({ ...identity, run: preserveReceiptTimes(run) }))
|
|
131
|
+
closed = true
|
|
132
|
+
return value
|
|
133
|
+
},
|
|
134
|
+
async interrupt(run) {
|
|
135
|
+
if (closed) return
|
|
136
|
+
// Keep the reservation when storage itself remains unavailable. It is
|
|
137
|
+
// never safe to turn an unconfirmed paid receipt into free capacity.
|
|
138
|
+
try { await manager.interruptExecution({ ...identity, run: preserveReceiptTimes(run) }) } catch { /* original hold stays durable */ }
|
|
139
|
+
},
|
|
140
|
+
}
|
|
141
|
+
}
|
|
@@ -11,7 +11,7 @@ import { tmpdir } from 'node:os'
|
|
|
11
11
|
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'
|
|
12
12
|
import { runOfficialTool, officialToolExecutionCapabilities, officialToolReadiness } from './official-tool-executor.mjs'
|
|
13
13
|
import { toolForProvider } from './official-tool-registry.mjs'
|
|
14
|
-
import { observeExecutionProgress, redactDiagnostic, verifiedDiagnostic } from './task-executors.mjs'
|
|
14
|
+
import { awaitExecutionCheckpoint, observeExecutionProgress, redactDiagnostic, verifiedDiagnostic } from './task-executors.mjs'
|
|
15
15
|
import { unsupportedCliReasoning } from './reasoning-execution.mjs'
|
|
16
16
|
|
|
17
17
|
const MAX_GIT_OUTPUT = 64_000_000
|
|
@@ -313,8 +313,10 @@ function packagePrompt(task, item, completed) {
|
|
|
313
313
|
}
|
|
314
314
|
|
|
315
315
|
/** Preflight every assignment before any model request or workspace write. */
|
|
316
|
-
export function executableTeamPackages(plan, installedIds = [], mode = 'read-only', executionEnvironment = 'harness'
|
|
317
|
-
|
|
316
|
+
export function executableTeamPackages(plan, installedIds = [], mode = 'read-only', executionEnvironment = 'harness',
|
|
317
|
+
executionCapabilities = officialToolExecutionCapabilities({ executionEnvironment })) {
|
|
318
|
+
if (!Array.isArray(executionCapabilities)) throw new TypeError('runtime capabilities must be an array')
|
|
319
|
+
const capabilities = new Map(executionCapabilities.map(item => [item.id, item]))
|
|
318
320
|
const packages = Array.isArray(plan?.team?.workPackages) ? plan.team.workPackages : []
|
|
319
321
|
const blocking = []
|
|
320
322
|
const assigned = packages.map(item => {
|
|
@@ -415,7 +417,7 @@ async function seedWorktree(isolated, seedFrom, signal) {
|
|
|
415
417
|
}
|
|
416
418
|
|
|
417
419
|
/** Sequential execution preserves DAG dependencies and avoids edit conflicts. */
|
|
418
|
-
export async function runOfficialTeam({ plan, task, workspace, allowedRoot, mode = 'read-only', executionEnvironment = 'harness', installedIds = [], cliModels, signal, sandbox, runtime, previous = [], onlyIds = null, seedFrom = null, onPackageStart, onPackage, onStage }) {
|
|
420
|
+
export async function runOfficialTeam({ plan, task, workspace, allowedRoot, mode = 'read-only', executionEnvironment = 'harness', installedIds = [], cliModels, signal, sandbox, runtime, previous = [], onlyIds = null, seedFrom = null, onPackageStart, onPackage, onStage, onCheckpointStart, onCheckpointResult, onWorkspace }) {
|
|
419
421
|
await observeExecutionProgress(onStage, 'preparing')
|
|
420
422
|
if (!['harness', 'cli-native'].includes(executionEnvironment)
|
|
421
423
|
|| (executionEnvironment === 'cli-native' && mode !== 'workspace-write')) throw new TypeError('invalid team CLI execution environment')
|
|
@@ -427,7 +429,10 @@ export async function runOfficialTeam({ plan, task, workspace, allowedRoot, mode
|
|
|
427
429
|
if (retry && mode !== 'read-only' && !(seedFrom && typeof seedFrom.workspace === 'string' && typeof seedFrom.baseCommit === 'string')) {
|
|
428
430
|
throw new TypeError('an editable team retry needs the earlier isolated worktree (seedFrom)')
|
|
429
431
|
}
|
|
430
|
-
|
|
432
|
+
// An injected adapter can declare its own capabilities as well as readiness
|
|
433
|
+
// and execution. The Host supplies no runtime override and always checks the
|
|
434
|
+
// actual platform registry before launching any official CLI.
|
|
435
|
+
const selected = executableTeamPackages(plan, installedIds, mode, executionEnvironment, runtime?.capabilities)
|
|
431
436
|
const assigned = retry ? selected.assigned.filter(item => onlyIds.includes(item.id)) : selected.assigned
|
|
432
437
|
const blocking = retry ? selected.blocking.filter(item => onlyIds.includes(item.id)) : selected.blocking
|
|
433
438
|
if (assigned.length === 0) return { status: 'blocked', blocking: [{ reason: '计划没有工作包。' }], results: [] }
|
|
@@ -448,6 +453,11 @@ export async function runOfficialTeam({ plan, task, workspace, allowedRoot, mode
|
|
|
448
453
|
if (signal?.aborted) return { status: 'cancelled', workspace: source, results: [] }
|
|
449
454
|
const isolated = mode === 'workspace-write' ? await isolatedWorktree(source, signal, allowedRoot) : null
|
|
450
455
|
const runWorkspace = isolated?.workspace ?? source
|
|
456
|
+
const completed = []
|
|
457
|
+
const partial = () => ({ status: signal?.aborted ? 'cancelled' : 'incomplete', workspace: runWorkspace,
|
|
458
|
+
results: [...completed], ...(isolated ? { baseCommit: isolated.baseCommit } : {}) })
|
|
459
|
+
await awaitExecutionCheckpoint(onWorkspace, { workspace: runWorkspace, sourceWorkspace: source,
|
|
460
|
+
isolatedWorkspace: isolated?.workspace ?? null, baseCommit: isolated?.baseCommit ?? null }, partial, 'workspace')
|
|
451
461
|
let seeded = null
|
|
452
462
|
if (isolated && retry) {
|
|
453
463
|
try { seeded = await seedWorktree(isolated, seedFrom, signal) }
|
|
@@ -456,7 +466,6 @@ export async function runOfficialTeam({ plan, task, workspace, allowedRoot, mode
|
|
|
456
466
|
return { status: 'blocked', blocking: [{ reason: String(error?.message ?? error) }], results: [] }
|
|
457
467
|
}
|
|
458
468
|
}
|
|
459
|
-
const completed = []
|
|
460
469
|
const cancelledRun = () => ({ status: 'cancelled', workspace: runWorkspace, results: completed,
|
|
461
470
|
...(isolated ? { baseCommit: isolated.baseCommit } : {}), ...(seeded ? { seeded } : {}) })
|
|
462
471
|
const context = retry ? (Array.isArray(previous) ? previous : []).filter(item => item?.id && typeof item.finalText === 'string') : []
|
|
@@ -466,8 +475,11 @@ export async function runOfficialTeam({ plan, task, workspace, allowedRoot, mode
|
|
|
466
475
|
const cliModel = teamCliModel(item, configuredCliModels)
|
|
467
476
|
const prompt = packagePrompt(task, item, [...context, ...completed])
|
|
468
477
|
const startedAt = Date.now()
|
|
469
|
-
|
|
470
|
-
model: item.recommendedModel, requestedCliModel: cliModel, startedAt }
|
|
478
|
+
const starting = { ...item, provider: item.recommendedProvider,
|
|
479
|
+
model: item.recommendedModel, requestedCliModel: cliModel, startedAt }
|
|
480
|
+
await observeExecutionProgress(onPackageStart, starting)
|
|
481
|
+
if (signal?.aborted) return cancelledRun()
|
|
482
|
+
await awaitExecutionCheckpoint(onCheckpointStart, starting, partial, 'start')
|
|
471
483
|
if (signal?.aborted) return cancelledRun()
|
|
472
484
|
let result
|
|
473
485
|
try {
|
|
@@ -518,6 +530,7 @@ export async function runOfficialTeam({ plan, task, workspace, allowedRoot, mode
|
|
|
518
530
|
error: modelMismatch ? '官方 CLI 实际模型与指定模型不一致。'
|
|
519
531
|
: (result.error || result.reason) ? executionDiagnostic(result.error || result.reason) : null,
|
|
520
532
|
outputTail: result.status === 'succeeded' ? null : executionDiagnostic(verifiedDiagnostic(result)) || null })
|
|
533
|
+
await awaitExecutionCheckpoint(onCheckpointResult, completed.at(-1), partial, 'result')
|
|
521
534
|
await observeExecutionProgress(onPackage, completed.at(-1))
|
|
522
535
|
if (signal?.aborted) return cancelledRun()
|
|
523
536
|
if (packageStatus !== 'succeeded') return { status: packageStatus === 'cancelled' ? 'cancelled' : 'incomplete', workspace: runWorkspace, results: completed,
|
|
@@ -544,7 +557,7 @@ export async function runOfficialTeam({ plan, task, workspace, allowedRoot, mode
|
|
|
544
557
|
}
|
|
545
558
|
|
|
546
559
|
/** One editable CLI task also uses the same isolated workspace and merge gate. */
|
|
547
|
-
export async function runOfficialTask({ toolId, task, workspace, allowedRoot, modelId, mode = 'read-only', signal, sandbox, runtime }) {
|
|
560
|
+
export async function runOfficialTask({ toolId, task, workspace, allowedRoot, modelId, mode = 'read-only', signal, sandbox, runtime, onCheckpointStart, onCheckpointResult, onWorkspace }) {
|
|
548
561
|
const source = await approvedSource(workspace, allowedRoot)
|
|
549
562
|
if (signal?.aborted) return { toolId, status: 'cancelled', notStarted: true, finalText: '' }
|
|
550
563
|
if (typeof sandbox?.confine !== 'function') return { toolId, status: 'unsupported', reason: '官方 Harness 进程沙箱不可用。' }
|
|
@@ -555,18 +568,29 @@ export async function runOfficialTask({ toolId, task, workspace, allowedRoot, mo
|
|
|
555
568
|
reason: readiness.reason ?? readiness.modeReasons?.[mode] ?? `官方 CLI 尚未就绪,或不支持 ${mode} 模式。` }
|
|
556
569
|
if (signal?.aborted) return { toolId, status: 'cancelled', notStarted: true, finalText: '' }
|
|
557
570
|
const isolated = mode === 'workspace-write' ? await isolatedWorktree(source, signal, allowedRoot) : null
|
|
571
|
+
const runWorkspace = isolated?.workspace ?? source
|
|
572
|
+
const location = isolated ? { isolatedWorkspace: isolated.workspace, baseCommit: isolated.baseCommit } : {}
|
|
558
573
|
let result
|
|
574
|
+
const partial = () => ({ id: 'direct', toolId, status: signal?.aborted ? 'cancelled' : 'failed',
|
|
575
|
+
notStarted: !result, finalText: '', ...result, ...location })
|
|
576
|
+
await awaitExecutionCheckpoint(onWorkspace, { workspace: runWorkspace, sourceWorkspace: source,
|
|
577
|
+
isolatedWorkspace: isolated?.workspace ?? null, baseCommit: isolated?.baseCommit ?? null }, partial, 'workspace')
|
|
578
|
+
if (signal?.aborted) return { ...partial(), status: 'cancelled', notStarted: true }
|
|
579
|
+
await awaitExecutionCheckpoint(onCheckpointStart, { id: 'direct', toolId, model: modelId ?? null,
|
|
580
|
+
workspace: runWorkspace, startedAt: Date.now() }, partial, 'start')
|
|
581
|
+
if (signal?.aborted) return { ...partial(), status: 'cancelled', notStarted: true }
|
|
559
582
|
try {
|
|
560
|
-
result = await (runtime?.runTool ?? runOfficialTool)({ toolId, task, modelId, workspace:
|
|
583
|
+
result = await (runtime?.runTool ?? runOfficialTool)({ toolId, task, modelId, workspace: runWorkspace,
|
|
561
584
|
mode, isolatedRoot: isolated?.isolatedRoot, signal, sandbox })
|
|
562
585
|
if (!result || typeof result.status !== 'string') throw new TypeError('官方 CLI 执行器没有返回有效状态。')
|
|
563
586
|
} catch (error) {
|
|
564
587
|
result = executionException(error, signal)
|
|
565
588
|
}
|
|
566
589
|
result = withFailureDiagnostic(result)
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
if (
|
|
590
|
+
result = { ...result, ...location, ...(signal?.aborted ? { status: 'cancelled' } : {}) }
|
|
591
|
+
await awaitExecutionCheckpoint(onCheckpointResult, { ...result, id: 'direct' }, partial, 'result')
|
|
592
|
+
if (signal?.aborted) return { ...result, status: 'cancelled' }
|
|
593
|
+
if (!isolated || result.status !== 'succeeded') return result
|
|
570
594
|
try {
|
|
571
595
|
const integration = await integrateWorktree(source, isolated.workspace, isolated.baseCommit, signal)
|
|
572
596
|
return { ...result,
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
import { spawn } from 'node:child_process'
|
|
15
15
|
import { mkdir, mkdtemp, readFile, realpath, rename, rm, stat, writeFile } from 'node:fs/promises'
|
|
16
16
|
import { homedir, tmpdir } from 'node:os'
|
|
17
|
-
import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'
|
|
17
|
+
import { dirname, isAbsolute, join, posix, relative, resolve, sep, win32 } from 'node:path'
|
|
18
18
|
import {
|
|
19
19
|
DEFAULT_NPM_REGISTRY,
|
|
20
20
|
defaultInstallMethod,
|
|
@@ -1042,7 +1042,8 @@ let prefixPromise = null
|
|
|
1042
1042
|
/** Add one directory to this process's PATH unless it is already there. */
|
|
1043
1043
|
export function appendToProcessPath(directory, { prepend = false, env = process.env, platform = process.platform } = {}) {
|
|
1044
1044
|
const target = String(directory ?? '').trim()
|
|
1045
|
-
|
|
1045
|
+
const paths = platform === 'win32' ? win32 : posix
|
|
1046
|
+
if (!target || !paths.isAbsolute(target)) return
|
|
1046
1047
|
const separator = platform === 'win32' ? ';' : ':'
|
|
1047
1048
|
const key = Object.keys(env).find(name => name.toUpperCase() === 'PATH') ?? 'PATH'
|
|
1048
1049
|
const parts = String(env[key] ?? '').split(separator).filter(Boolean)
|
|
@@ -17,9 +17,11 @@ import { mkdir, open, readFile, rename, stat, unlink, writeFile } from 'node:fs/
|
|
|
17
17
|
import { homedir } from 'node:os'
|
|
18
18
|
import { dirname, isAbsolute, join, resolve } from 'node:path'
|
|
19
19
|
import { archiveSpending, sanitizeArchivedSpending } from './run-ledger.mjs'
|
|
20
|
+
import { sanitizeBudgetReservations } from './budget-reservations.mjs'
|
|
20
21
|
|
|
21
22
|
export const MAX_RUNS = 200
|
|
22
|
-
|
|
23
|
+
// Older Hosts must refuse v2 instead of dropping durable execution holds.
|
|
24
|
+
const STATE_VERSION = 2
|
|
23
25
|
|
|
24
26
|
/** Same DSH home rule as the Host: DSH_HOME (with ~ expansion) or ~/.dsh. */
|
|
25
27
|
export function resolveStateHome(env = process.env, home = homedir(), cwd = process.cwd()) {
|
|
@@ -41,7 +43,7 @@ const LOCK_STALE_MS = 15_000
|
|
|
41
43
|
const LOCK_TIMEOUT_MS = 10_000
|
|
42
44
|
|
|
43
45
|
function emptyState() {
|
|
44
|
-
return { version: STATE_VERSION, historyIncomplete: false, onboarding: { completedAt: null }, health: null, quota: {}, authFailures: {}, runs: [], dynamicData: { liveBench: null, pricing: null, status: {}, revision: '' }, archivedSpending: sanitizeArchivedSpending(), notices: [], terminalSessions: [] }
|
|
46
|
+
return { version: STATE_VERSION, historyIncomplete: false, onboarding: { completedAt: null }, health: null, quota: {}, authFailures: {}, runs: [], budgetReservations: {}, dynamicData: { liveBench: null, pricing: null, status: {}, revision: '' }, archivedSpending: sanitizeArchivedSpending(), notices: [], terminalSessions: [] }
|
|
45
47
|
}
|
|
46
48
|
|
|
47
49
|
function sanitizeNotices(value) {
|
|
@@ -51,9 +53,16 @@ function sanitizeNotices(value) {
|
|
|
51
53
|
}
|
|
52
54
|
|
|
53
55
|
function retainRuns(state, maxRuns) {
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
56
|
+
// A partial run must stay addressable until its hold is reconciled. Archiving
|
|
57
|
+
// it and later upserting a checkpoint would count its receipts twice.
|
|
58
|
+
const protectedIds = new Set(Object.values(state.budgetReservations ?? {}).map(entry => entry.runId))
|
|
59
|
+
let remove = Math.max(0, state.runs.length - Math.max(maxRuns, protectedIds.size))
|
|
60
|
+
const evicted = []
|
|
61
|
+
state.runs = state.runs.filter(run => {
|
|
62
|
+
if (remove > 0 && !protectedIds.has(run.id)) { remove -= 1; evicted.push(run); return false }
|
|
63
|
+
return true
|
|
64
|
+
})
|
|
65
|
+
state.archivedSpending = archiveSpending(evicted, state.archivedSpending)
|
|
57
66
|
return state
|
|
58
67
|
}
|
|
59
68
|
|
|
@@ -94,19 +103,21 @@ function invalidRunSpending(run) {
|
|
|
94
103
|
}
|
|
95
104
|
|
|
96
105
|
function sanitize(value, maxRuns) {
|
|
97
|
-
if (!value || typeof value !== 'object' || Array.isArray(value) || value.version
|
|
106
|
+
if (!value || typeof value !== 'object' || Array.isArray(value) || ![1, STATE_VERSION].includes(value.version)
|
|
98
107
|
|| !Array.isArray(value.runs)) {
|
|
99
108
|
const error = new Error('模型路由状态版本或结构不受支持;原文件未修改,请先恢复或升级兼容版本。')
|
|
100
109
|
error.code = 'ROUTER_STATE_UNSUPPORTED'
|
|
101
110
|
throw error
|
|
102
111
|
}
|
|
103
112
|
const runs = value.runs.filter(run => run && typeof run.id === 'string')
|
|
113
|
+
const held = sanitizeBudgetReservations(value.budgetReservations)
|
|
104
114
|
return retainRuns({
|
|
105
115
|
version: STATE_VERSION,
|
|
106
116
|
// Unlike bounded notices, an incomplete-history marker cannot silently
|
|
107
117
|
// expire. Restoring a verified state file is required to recover budgets.
|
|
108
118
|
historyIncomplete: value.historyIncomplete === true || runs.length !== value.runs.length
|
|
109
|
-
|| invalidArchivedSpending(value.archivedSpending) || runs.some(invalidRunSpending)
|
|
119
|
+
|| invalidArchivedSpending(value.archivedSpending) || runs.some(invalidRunSpending) || held.invalid
|
|
120
|
+
|| Object.values(held.reservations).some(entry => !runs.some(run => run.id === entry.runId))
|
|
110
121
|
|| sanitizeNotices(value.notices).some(notice => notice.kind === 'state-corrupt'),
|
|
111
122
|
onboarding: { completedAt: Number.isFinite(value.onboarding?.completedAt) ? value.onboarding.completedAt : null },
|
|
112
123
|
health: value.health && Array.isArray(value.health.tools) ? value.health : null,
|
|
@@ -114,6 +125,7 @@ function sanitize(value, maxRuns) {
|
|
|
114
125
|
authFailures: Object.fromEntries(Object.entries(value.authFailures && typeof value.authFailures === 'object' && !Array.isArray(value.authFailures) ? value.authFailures : {})
|
|
115
126
|
.filter(([, entry]) => Number.isFinite(entry?.at))),
|
|
116
127
|
runs,
|
|
128
|
+
budgetReservations: held.reservations,
|
|
117
129
|
dynamicData: value.dynamicData && typeof value.dynamicData === 'object' && !Array.isArray(value.dynamicData) ? value.dynamicData : { liveBench: null, pricing: null, status: {}, revision: '' },
|
|
118
130
|
archivedSpending: sanitizeArchivedSpending(value.archivedSpending),
|
|
119
131
|
notices: sanitizeNotices(value.notices),
|
|
@@ -172,7 +184,10 @@ async function acquireLock(lockFile, { staleMs = LOCK_STALE_MS, timeoutMs = LOCK
|
|
|
172
184
|
if (await readFile(lockFile, 'utf8').catch(() => null) === owner) await unlink(lockFile).catch(() => {})
|
|
173
185
|
}
|
|
174
186
|
} catch (error) {
|
|
175
|
-
|
|
187
|
+
// Windows can report a sharing/delete-pending race as EPERM while
|
|
188
|
+
// another owner closes and removes its lock. Retry within the same
|
|
189
|
+
// bounded wait; never treat this as permission to remove a live lock.
|
|
190
|
+
if (error?.code !== 'EEXIST' && !(process.platform === 'win32' && error?.code === 'EPERM')) throw error
|
|
176
191
|
const info = await stat(lockFile).catch(() => null)
|
|
177
192
|
const owner = await readFile(lockFile, 'utf8').catch(() => null)
|
|
178
193
|
if (info && owner !== null && now() - info.mtimeMs > staleMs && ownerIsDead(ownerPid(owner))) {
|