@thready-web/dsh-bot-plugin 0.1.0-beta.3 → 0.1.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +78 -1
- package/dist/approvals.js +160 -0
- package/dist/authorization.js +225 -0
- package/dist/context.js +11 -1
- package/dist/cua-access.js +79 -0
- package/dist/http-plugin.js +8 -2
- package/dist/http.js +206 -74
- package/dist/inbox.js +21 -35
- package/dist/index.js +53 -8
- package/dist/live-snapshot.js +2 -15
- package/dist/log/canonical.js +37 -0
- package/dist/log/plan.js +62 -0
- package/dist/log/routes.js +144 -0
- package/dist/log/snapshot-service.js +76 -0
- package/dist/log/source.js +173 -0
- package/dist/permissions.js +86 -0
- package/dist/questions.js +407 -0
- package/dist/work-admit.js +65 -15
- package/dist/work-child.js +8 -3
- package/dist/work-http.js +29 -0
- package/dist/work-ledger.js +7 -0
- package/dist/work-mock-host.js +1 -0
- package/dist/work-mock.js +6 -0
- package/dist/work-tickets.js +212 -16
- package/dist/work-tools.js +2 -2
- package/dist/work-worker.js +693 -134
- package/dist/workspace-image.js +91 -0
- package/fixtures/mock-llm.mjs +17 -2
- package/package.json +9 -3
- package/public/assets/index-Dczis232.css +1 -0
- package/public/assets/index-oExHCCsZ.js +63 -0
- package/public/index.html +2 -2
- package/public/assets/index-D0tTt7E4.js +0 -61
- package/public/assets/index-PyqSrdBb.css +0 -1
package/dist/work-worker.js
CHANGED
|
@@ -2,7 +2,7 @@ import { randomUUID } from 'node:crypto';
|
|
|
2
2
|
import { sha256 } from "./fs-safe.js";
|
|
3
3
|
import { toolError } from "./errors.js";
|
|
4
4
|
import { createTicketBook } from "./work-tickets.js";
|
|
5
|
-
import { decideAdmission, budgetTrip, estimateWorkerBudget, DEFAULT_WORK_BUDGET } from "./work-admit.js";
|
|
5
|
+
import { decideAdmission, budgetTrip, estimateWorkerBudget, DEFAULT_WORK_BUDGET, effectiveWorkLimits, } from "./work-admit.js";
|
|
6
6
|
import { messageIdOf, pushInput, pushNotice, reasonKindOf, reconcileInputs, reconcileNotices, renderNotice, } from "./work-ledger.js";
|
|
7
7
|
const KINDS = new Set(['progress', 'question', 'artifact', 'result', 'failure']);
|
|
8
8
|
const LABEL = {
|
|
@@ -26,10 +26,15 @@ export const BUDGET_NOTICE_PREFIX = 'foundations-budget:';
|
|
|
26
26
|
export function createWorkRuntime(store, engine, options = {}) {
|
|
27
27
|
const now = options.now ?? (() => new Date());
|
|
28
28
|
const limits = { ...DEFAULT_WORK_BUDGET, ...options.budget };
|
|
29
|
+
const slots = effectiveWorkLimits(limits);
|
|
29
30
|
const pid = options.pid ?? process.pid;
|
|
30
31
|
const resident = options.resident !== false;
|
|
31
32
|
const book = createTicketBook(store.db, store.root, now);
|
|
32
33
|
const inflight = new Set();
|
|
34
|
+
/** Ticket ids currently in ensureChild / flush, so a tick cannot start the same ticket twice. */
|
|
35
|
+
const starting = new Set();
|
|
36
|
+
/** In-flight startReserved promises, so a later promoteQueued can join them. */
|
|
37
|
+
const startingPromises = new Map();
|
|
33
38
|
/** @type {Map<string, { messageId: string, turn: number | null, seq: number | null }>} */
|
|
34
39
|
const turns = new Map();
|
|
35
40
|
/** @type {Map<string, string>} */
|
|
@@ -38,6 +43,8 @@ export function createWorkRuntime(store, engine, options = {}) {
|
|
|
38
43
|
const arms = new Map();
|
|
39
44
|
/** @type {Map<string, number>} */
|
|
40
45
|
const holds = new Map();
|
|
46
|
+
const admitting = new Set();
|
|
47
|
+
const budgetWaiters = new Map();
|
|
41
48
|
const schedule = options.schedule ??
|
|
42
49
|
((fn, ms) => {
|
|
43
50
|
const timer = setTimeout(fn, ms);
|
|
@@ -75,9 +82,11 @@ export function createWorkRuntime(store, engine, options = {}) {
|
|
|
75
82
|
releaseHold,
|
|
76
83
|
tick,
|
|
77
84
|
admit,
|
|
85
|
+
promoteQueued,
|
|
78
86
|
observeSession,
|
|
79
87
|
observeClaimed,
|
|
80
88
|
gateStep,
|
|
89
|
+
waitForBudget,
|
|
81
90
|
isWorkerSession,
|
|
82
91
|
summaries: (owner) => book.summaries(owner),
|
|
83
92
|
detail: (id) => book.detail(id),
|
|
@@ -92,6 +101,171 @@ export function createWorkRuntime(store, engine, options = {}) {
|
|
|
92
101
|
function isWorkerSession(sessionId) {
|
|
93
102
|
return book.attemptForChild(sessionId) != null;
|
|
94
103
|
}
|
|
104
|
+
/**
|
|
105
|
+
* A running slot is free when both the per-bot and global running caps have room.
|
|
106
|
+
* needs_user does not occupy a running slot.
|
|
107
|
+
* @param {string} owner
|
|
108
|
+
*/
|
|
109
|
+
function hasSlot(owner) {
|
|
110
|
+
return book.countSlots(owner) < slots.perBot && book.countGlobalSlots() < slots.global;
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Queue room is separate from running slots. The combined unfinished cap is
|
|
114
|
+
* slots + queue, so parked needs_user tickets cannot grow without bound.
|
|
115
|
+
* @param {string} owner
|
|
116
|
+
*/
|
|
117
|
+
function canQueue(owner) {
|
|
118
|
+
return (book.countQueued(owner) < slots.perBotQueue &&
|
|
119
|
+
book.countGlobalQueued() < slots.globalQueue &&
|
|
120
|
+
book.countActive(owner) < slots.perBot + slots.perBotQueue &&
|
|
121
|
+
book.countGlobal() < slots.global + slots.globalQueue);
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* @param {string} owner
|
|
125
|
+
* @returns {'start' | 'queue' | 'refuse'}
|
|
126
|
+
*/
|
|
127
|
+
function underCombinedCap(owner) {
|
|
128
|
+
return (book.countActive(owner) < slots.perBot + slots.perBotQueue &&
|
|
129
|
+
book.countGlobal() < slots.global + slots.globalQueue);
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* @param {string} owner
|
|
133
|
+
* @returns {'start' | 'queue' | 'refuse'}
|
|
134
|
+
*/
|
|
135
|
+
function decideDispatch(owner) {
|
|
136
|
+
// needs_user does not occupy a running slot, but it still counts as unfinished.
|
|
137
|
+
// A free slot must not admit a new ticket once the combined cap is full.
|
|
138
|
+
if (!underCombinedCap(owner))
|
|
139
|
+
return 'refuse';
|
|
140
|
+
if (hasSlot(owner))
|
|
141
|
+
return 'start';
|
|
142
|
+
if (canQueue(owner))
|
|
143
|
+
return 'queue';
|
|
144
|
+
return 'refuse';
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Create the child if needed and flush the current briefing or supply.
|
|
148
|
+
* @param {Record<string, any>} ticket
|
|
149
|
+
*/
|
|
150
|
+
async function startReserved(ticket) {
|
|
151
|
+
if (!resident) {
|
|
152
|
+
return { ...publicDispatch(ticket), flush: 'pending', resident: false, message: 'Only a resident host continues' };
|
|
153
|
+
}
|
|
154
|
+
if (starting.has(ticket.id)) {
|
|
155
|
+
return startingPromises.get(ticket.id) ?? publicDispatch(book.ticket(ticket.id) ?? ticket, { flush: 'pending' });
|
|
156
|
+
}
|
|
157
|
+
starting.add(ticket.id);
|
|
158
|
+
const run = (async () => {
|
|
159
|
+
const current = book.ticket(ticket.id);
|
|
160
|
+
if (!current)
|
|
161
|
+
return toolError('NOT_FOUND', 'No such ticket.');
|
|
162
|
+
if (current.status === 'cancelled' || current.status === 'failed' || current.status === 'delivered') {
|
|
163
|
+
return publicDispatch(current);
|
|
164
|
+
}
|
|
165
|
+
const created = await ensureChild(current);
|
|
166
|
+
if (created.error)
|
|
167
|
+
return created.error;
|
|
168
|
+
if (created.rejected)
|
|
169
|
+
return publicDispatch(created.ticket ?? book.ticket(ticket.id) ?? current);
|
|
170
|
+
const liveAfterStart = book.ticket(ticket.id);
|
|
171
|
+
if (!liveAfterStart ||
|
|
172
|
+
liveAfterStart.status === 'cancelled' ||
|
|
173
|
+
liveAfterStart.status === 'failed' ||
|
|
174
|
+
liveAfterStart.status === 'delivered') {
|
|
175
|
+
return publicDispatch(liveAfterStart ?? current);
|
|
176
|
+
}
|
|
177
|
+
const input = await store.write('work', () => {
|
|
178
|
+
const live = book.ticket(ticket.id);
|
|
179
|
+
const liveAttempt = book.attempt(live.active_attempt_id);
|
|
180
|
+
if (liveAttempt?.current_input_id) {
|
|
181
|
+
const existing = book.input(liveAttempt.current_input_id);
|
|
182
|
+
if (existing)
|
|
183
|
+
return existing;
|
|
184
|
+
}
|
|
185
|
+
const text = book.readBriefing(live.briefing_hash);
|
|
186
|
+
return book.insertInput({
|
|
187
|
+
ticketId: live.id,
|
|
188
|
+
attemptId: liveAttempt.id,
|
|
189
|
+
revision: live.briefing_revision,
|
|
190
|
+
kind: 'briefing',
|
|
191
|
+
text,
|
|
192
|
+
});
|
|
193
|
+
});
|
|
194
|
+
if (input.flush_status === 'accepted') {
|
|
195
|
+
return publicDispatch(book.ticket(ticket.id), { flush: 'accepted', nativeMessageId: input.native_message_id });
|
|
196
|
+
}
|
|
197
|
+
const flushed = await pushInput(book, input, deps);
|
|
198
|
+
return publicDispatch(book.ticket(ticket.id), {
|
|
199
|
+
flush: flushWord(flushed),
|
|
200
|
+
nativeMessageId: input.native_message_id,
|
|
201
|
+
});
|
|
202
|
+
})();
|
|
203
|
+
startingPromises.set(ticket.id, run);
|
|
204
|
+
try {
|
|
205
|
+
return await run;
|
|
206
|
+
}
|
|
207
|
+
finally {
|
|
208
|
+
starting.delete(ticket.id);
|
|
209
|
+
startingPromises.delete(ticket.id);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
/**
|
|
213
|
+
* Start waiting tickets that now fit a slot. Awaits only starts this call
|
|
214
|
+
* launched, so a hung createChild from an earlier call cannot deadlock
|
|
215
|
+
* later ticks or cancel. The earlier promise stays owned until it finishes
|
|
216
|
+
* and is then attached or cleaned up.
|
|
217
|
+
*/
|
|
218
|
+
async function promoteQueued() {
|
|
219
|
+
if (!resident)
|
|
220
|
+
return { started: [] };
|
|
221
|
+
const started = [];
|
|
222
|
+
const launched = [];
|
|
223
|
+
const launch = (ticket) => {
|
|
224
|
+
if (!ticket?.id || starting.has(ticket.id))
|
|
225
|
+
return;
|
|
226
|
+
started.push(ticket.id);
|
|
227
|
+
launched.push(startReserved(ticket).catch(error => {
|
|
228
|
+
options.log?.('warn', `session-foundations: startReserved failed ticket=${ticket.id} ${error instanceof Error ? error.message : String(error)}`);
|
|
229
|
+
return error;
|
|
230
|
+
}));
|
|
231
|
+
};
|
|
232
|
+
for (const ticket of book.heldStarts())
|
|
233
|
+
launch(ticket);
|
|
234
|
+
for (;;) {
|
|
235
|
+
const claimed = await store.write('work', () => {
|
|
236
|
+
for (const ticket of book.nextPromotable()) {
|
|
237
|
+
if (book.slotHeld(ticket))
|
|
238
|
+
continue;
|
|
239
|
+
if (ticket.status !== 'accepted' && ticket.status !== 'needs_user')
|
|
240
|
+
continue;
|
|
241
|
+
if (!hasSlot(ticket.owner_session_id))
|
|
242
|
+
continue;
|
|
243
|
+
return book.reserveSlot(ticket.id);
|
|
244
|
+
}
|
|
245
|
+
return null;
|
|
246
|
+
});
|
|
247
|
+
if (!claimed)
|
|
248
|
+
break;
|
|
249
|
+
launch(claimed);
|
|
250
|
+
}
|
|
251
|
+
await Promise.all(launched);
|
|
252
|
+
// Join in-flight starts that are still allowed to attach. Do not join a
|
|
253
|
+
// hung createChild after the ticket has already been cancelled, failed, or
|
|
254
|
+
// expired, so a later tick can free the slot and start the next waiter.
|
|
255
|
+
const pending = [...startingPromises.entries()]
|
|
256
|
+
.filter(([id]) => {
|
|
257
|
+
const ticket = book.ticket(id);
|
|
258
|
+
return (ticket != null && ticket.status !== 'failed' && ticket.status !== 'cancelled' && ticket.status !== 'delivered');
|
|
259
|
+
})
|
|
260
|
+
.map(([, promise]) => promise);
|
|
261
|
+
await Promise.all(pending);
|
|
262
|
+
return { started };
|
|
263
|
+
}
|
|
264
|
+
function schedulePromote() {
|
|
265
|
+
void promoteQueued().catch(error => {
|
|
266
|
+
options.log?.('warn', `session-foundations: promoteQueued failed ${error instanceof Error ? error.message : String(error)}`);
|
|
267
|
+
});
|
|
268
|
+
}
|
|
95
269
|
/**
|
|
96
270
|
* @param {string} owner
|
|
97
271
|
* @param {Record<string, unknown>} args
|
|
@@ -119,50 +293,40 @@ export function createWorkRuntime(store, engine, options = {}) {
|
|
|
119
293
|
return { replay: existing };
|
|
120
294
|
}
|
|
121
295
|
}
|
|
122
|
-
|
|
296
|
+
const decision = decideDispatch(owner);
|
|
297
|
+
if (decision === 'refuse') {
|
|
123
298
|
return {
|
|
124
|
-
error: toolError('BUDGET', 'Unfinished tickets already hit the host
|
|
299
|
+
error: toolError('BUDGET', 'Unfinished tickets already hit the host slot and queue limits, so no row was created.'),
|
|
125
300
|
};
|
|
126
301
|
}
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
302
|
+
const created = book.insertAccepted({
|
|
303
|
+
owner,
|
|
304
|
+
briefing,
|
|
305
|
+
idempotencyKey: key,
|
|
306
|
+
correlationId: correlationOf(owner, args.correlationId),
|
|
307
|
+
budget,
|
|
308
|
+
});
|
|
309
|
+
if (decision === 'start')
|
|
310
|
+
book.reserveSlot(created.ticket.id);
|
|
311
|
+
return { created, start: decision === 'start' };
|
|
136
312
|
});
|
|
137
313
|
if (prepared.error)
|
|
138
314
|
return prepared.error;
|
|
139
315
|
if (prepared.replay)
|
|
140
316
|
return publicDispatch(prepared.replay, { replayed: true });
|
|
141
317
|
const ticket = prepared.created.ticket;
|
|
318
|
+
if (!prepared.start) {
|
|
319
|
+
return publicDispatch(book.ticket(ticket.id), {
|
|
320
|
+
flush: 'pending',
|
|
321
|
+
queued: true,
|
|
322
|
+
childSessionId: null,
|
|
323
|
+
nativeMessageId: null,
|
|
324
|
+
});
|
|
325
|
+
}
|
|
142
326
|
if (!resident) {
|
|
143
327
|
return { ...publicDispatch(ticket), flush: 'pending', resident: false, message: 'Only a resident host continues' };
|
|
144
328
|
}
|
|
145
|
-
|
|
146
|
-
if (created.error)
|
|
147
|
-
return created.error;
|
|
148
|
-
const input = await store.write('work', () => {
|
|
149
|
-
const current = book.ticket(ticket.id);
|
|
150
|
-
const attempt = book.attempt(current.active_attempt_id);
|
|
151
|
-
if (attempt?.current_input_id)
|
|
152
|
-
return book.input(attempt.current_input_id);
|
|
153
|
-
return book.insertInput({
|
|
154
|
-
ticketId: current.id,
|
|
155
|
-
attemptId: attempt.id,
|
|
156
|
-
revision: current.briefing_revision,
|
|
157
|
-
kind: 'briefing',
|
|
158
|
-
text: briefing,
|
|
159
|
-
});
|
|
160
|
-
});
|
|
161
|
-
const flushed = await pushInput(book, input, deps);
|
|
162
|
-
return publicDispatch(book.ticket(ticket.id), {
|
|
163
|
-
flush: flushWord(flushed),
|
|
164
|
-
nativeMessageId: input.native_message_id,
|
|
165
|
-
});
|
|
329
|
+
return startReserved(ticket);
|
|
166
330
|
}
|
|
167
331
|
/**
|
|
168
332
|
* @param {string} owner
|
|
@@ -193,9 +357,28 @@ export function createWorkRuntime(store, engine, options = {}) {
|
|
|
193
357
|
if (found.error)
|
|
194
358
|
return found.error;
|
|
195
359
|
const expected = revisionOf(args.expectedRevision);
|
|
196
|
-
const advanced = await store.write('work', () =>
|
|
360
|
+
const advanced = await store.write('work', () => {
|
|
361
|
+
const result = book.advanceGeneration(found.ticket.id, expected, kind, text);
|
|
362
|
+
if (result.error)
|
|
363
|
+
return result;
|
|
364
|
+
// A supply into needs_user must take a running slot. If none is free, keep
|
|
365
|
+
// the input durable and start it when promoteQueued reserves one.
|
|
366
|
+
if (kind === 'supply' && result.ticket.status === 'needs_user' && !hasSlot(owner)) {
|
|
367
|
+
return { ...result, queued: true };
|
|
368
|
+
}
|
|
369
|
+
if (kind === 'supply' && result.ticket.status === 'needs_user')
|
|
370
|
+
book.reserveSlot(result.ticket.id);
|
|
371
|
+
return result;
|
|
372
|
+
});
|
|
197
373
|
if (advanced.error)
|
|
198
374
|
return toolError(advanced.error, advanced.message ?? 'Changing the generation failed.');
|
|
375
|
+
if (advanced.queued) {
|
|
376
|
+
return publicDispatch(book.ticket(found.ticket.id), {
|
|
377
|
+
flush: 'pending',
|
|
378
|
+
queued: true,
|
|
379
|
+
nativeMessageId: advanced.input.native_message_id,
|
|
380
|
+
});
|
|
381
|
+
}
|
|
199
382
|
if (!resident)
|
|
200
383
|
return { ...publicDispatch(advanced.ticket), flush: 'pending', message: 'Only a resident host continues' };
|
|
201
384
|
const flushed = await pushInput(book, advanced.input, deps);
|
|
@@ -220,12 +403,15 @@ export function createWorkRuntime(store, engine, options = {}) {
|
|
|
220
403
|
if (cancelled.error)
|
|
221
404
|
return toolError(cancelled.error, cancelled.message ?? 'Cancel failed.');
|
|
222
405
|
const child = cancelled.ticket.child_session_id;
|
|
406
|
+
if (child)
|
|
407
|
+
closeBudgetWait(child);
|
|
223
408
|
if (resident && child && options.cancel) {
|
|
224
409
|
await options.cancel(child, { keepInbox: note !== '', reason: 'ticket-cancel' });
|
|
225
410
|
}
|
|
226
411
|
if (resident && note !== '' && cancelled.input)
|
|
227
412
|
await pushInput(book, cancelled.input, deps);
|
|
228
413
|
await removeUnclaimed(owner, cancelled.ticket.id);
|
|
414
|
+
schedulePromote();
|
|
229
415
|
return {
|
|
230
416
|
ok: true,
|
|
231
417
|
ticketId: cancelled.ticket.id,
|
|
@@ -247,29 +433,88 @@ export function createWorkRuntime(store, engine, options = {}) {
|
|
|
247
433
|
const failed = await store.write('work', () => book.failRunning(found.ticket.id, 'grant-denied'));
|
|
248
434
|
const child = failed?.child_session_id;
|
|
249
435
|
const state = child ? arms.get(child) : null;
|
|
436
|
+
if (child)
|
|
437
|
+
closeBudgetWait(child);
|
|
250
438
|
if (state) {
|
|
251
439
|
state.closing = true;
|
|
252
440
|
state.clear?.();
|
|
253
441
|
}
|
|
254
442
|
if (resident && child && options.cancel)
|
|
255
443
|
await options.cancel(child, { keepInbox: false, reason: 'grant-denied' });
|
|
444
|
+
schedulePromote();
|
|
256
445
|
return { ok: true, ticketId: found.ticket.id, ticketStatus: 'failed', denied: true };
|
|
257
446
|
}
|
|
447
|
+
const live = found.ticket.child_session_id ? arms.get(found.ticket.child_session_id) : null;
|
|
448
|
+
if (live?.role === 'worker')
|
|
449
|
+
saveQuota(found.ticket.child_session_id, live);
|
|
258
450
|
const added = await store.write('work', () => book.extendBudget(found.ticket.id, {
|
|
259
451
|
wallMs: typeof args.wallMs === 'number' ? args.wallMs : undefined,
|
|
260
452
|
maxTokens: typeof args.maxTokens === 'number' ? args.maxTokens : undefined,
|
|
261
453
|
}, { wallMs: limits.workerWallMs, maxTokens: limits.workerMaxTokens }));
|
|
262
454
|
if (added.error)
|
|
263
|
-
return toolError(added.error, added.message ?? 'No budget was added.');
|
|
455
|
+
return { ...toolError(added.error, added.message ?? 'No budget was added.'), ...quotaFeedback(found.ticket.id) };
|
|
264
456
|
const child = added.ticket.child_session_id;
|
|
265
457
|
if (child)
|
|
266
458
|
releaseArm(child, added.budget);
|
|
459
|
+
let resumedInput = null;
|
|
460
|
+
const quota = quotaOf(added.ticket.id);
|
|
461
|
+
if (child &&
|
|
462
|
+
!arms.has(child) &&
|
|
463
|
+
quota?.waiting &&
|
|
464
|
+
options.resumePaused &&
|
|
465
|
+
resident &&
|
|
466
|
+
options.push &&
|
|
467
|
+
['started', 'enqueued'].includes(added.ticket.status) &&
|
|
468
|
+
added.budget.maxTokens > quota.tokens &&
|
|
469
|
+
added.budget.wallMs > quota.activeMs &&
|
|
470
|
+
quota.tokens < limits.workerMaxTokens &&
|
|
471
|
+
quota.activeMs < limits.workerWallMs &&
|
|
472
|
+
!(await isChildLive(child))) {
|
|
473
|
+
resumedInput = await store.write('work', () => {
|
|
474
|
+
const fresh = book.ticket(added.ticket.id);
|
|
475
|
+
if (!fresh ||
|
|
476
|
+
!['started', 'enqueued'].includes(fresh.status) ||
|
|
477
|
+
fresh.active_attempt_id !== added.ticket.active_attempt_id)
|
|
478
|
+
return null;
|
|
479
|
+
const current = quotaOf(fresh.id);
|
|
480
|
+
if (!current?.waiting)
|
|
481
|
+
return null;
|
|
482
|
+
if (current.resumeInputId)
|
|
483
|
+
return book.input(current.resumeInputId);
|
|
484
|
+
// A detached native turn ended (possibly disposed). Advance only the
|
|
485
|
+
// input generation, not the attempt/fence/session, so replay of that
|
|
486
|
+
// old turn/end cannot finalize a new pending result.
|
|
487
|
+
store.db
|
|
488
|
+
.prepare('UPDATE work_tickets SET revision = revision + 1, briefing_revision = briefing_revision + 1, updated_at = ? WHERE id = ?')
|
|
489
|
+
.run(now().toISOString(), fresh.id);
|
|
490
|
+
const input = book.insertInput({
|
|
491
|
+
ticketId: fresh.id,
|
|
492
|
+
attemptId: fresh.active_attempt_id,
|
|
493
|
+
revision: fresh.briefing_revision + 1,
|
|
494
|
+
kind: 'supply',
|
|
495
|
+
text: '[foundations:budget-resume] Budget was explicitly granted. Continue the SAME ticket from durable history; do not repeat completed effects. Inspect current state before continuing. This is a new host turn after detachment, not a new task.',
|
|
496
|
+
});
|
|
497
|
+
writeQuota(fresh.id, { ...current, resumeInputId: input.id });
|
|
498
|
+
return input;
|
|
499
|
+
});
|
|
500
|
+
if (resumedInput)
|
|
501
|
+
await pushInput(book, resumedInput, deps);
|
|
502
|
+
}
|
|
267
503
|
return {
|
|
268
504
|
ok: true,
|
|
269
505
|
ticketId: added.ticket.id,
|
|
270
|
-
ticketStatus: added.ticket.status,
|
|
506
|
+
ticketStatus: book.ticket(added.ticket.id)?.status ?? added.ticket.status,
|
|
507
|
+
briefingRevision: book.ticket(added.ticket.id)?.briefing_revision,
|
|
271
508
|
wallMs: added.budget.wallMs,
|
|
272
509
|
maxTokens: added.budget.maxTokens,
|
|
510
|
+
hostCeilingTokens: limits.workerMaxTokens,
|
|
511
|
+
hostCeilingWallMs: limits.workerWallMs,
|
|
512
|
+
...quotaFeedback(added.ticket.id),
|
|
513
|
+
continuation: resumedInput
|
|
514
|
+
? 'same-session-new-turn'
|
|
515
|
+
: child && arms.has(child)
|
|
516
|
+
? 'same-turn'
|
|
517
|
+
: 'awaiting-host-turn',
|
|
273
518
|
};
|
|
274
519
|
}
|
|
275
520
|
async function retry(owner, args) {
|
|
@@ -279,19 +524,27 @@ export function createWorkRuntime(store, engine, options = {}) {
|
|
|
279
524
|
if (found.error)
|
|
280
525
|
return found.error;
|
|
281
526
|
const expected = revisionOf(args.expectedRevision);
|
|
282
|
-
const retried = await store.write('work', () =>
|
|
527
|
+
const retried = await store.write('work', () => {
|
|
528
|
+
const result = book.insertRetry(found.ticket.id, expected);
|
|
529
|
+
if (result.error)
|
|
530
|
+
return result;
|
|
531
|
+
if (!hasSlot(owner))
|
|
532
|
+
return { ...result, queued: true };
|
|
533
|
+
book.reserveSlot(result.ticket.id);
|
|
534
|
+
return result;
|
|
535
|
+
});
|
|
283
536
|
if (retried.error)
|
|
284
537
|
return toolError(retried.error, retried.message ?? 'Retry failed.');
|
|
538
|
+
if (retried.queued) {
|
|
539
|
+
return publicDispatch(book.ticket(retried.ticket.id), {
|
|
540
|
+
flush: 'pending',
|
|
541
|
+
queued: true,
|
|
542
|
+
nativeMessageId: retried.input?.native_message_id ?? null,
|
|
543
|
+
});
|
|
544
|
+
}
|
|
285
545
|
if (!resident)
|
|
286
546
|
return { ...publicDispatch(retried.ticket), flush: 'pending', message: 'Only a resident host continues' };
|
|
287
|
-
|
|
288
|
-
if (ensured.error)
|
|
289
|
-
return ensured.error;
|
|
290
|
-
const flushed = await pushInput(book, retried.input, deps);
|
|
291
|
-
return publicDispatch(book.ticket(retried.ticket.id), {
|
|
292
|
-
flush: flushWord(flushed),
|
|
293
|
-
nativeMessageId: retried.input.native_message_id,
|
|
294
|
-
});
|
|
547
|
+
return startReserved(book.ticket(retried.ticket.id));
|
|
295
548
|
}
|
|
296
549
|
/**
|
|
297
550
|
* @param {string} owner
|
|
@@ -428,9 +681,9 @@ export function createWorkRuntime(store, engine, options = {}) {
|
|
|
428
681
|
async function tick() {
|
|
429
682
|
await reconcileInputs(book, deps);
|
|
430
683
|
await reconcileNotices(book, deps);
|
|
431
|
-
await
|
|
432
|
-
|
|
433
|
-
|
|
684
|
+
await renewLiveLeases();
|
|
685
|
+
const expired = await store.write('work', () => book.expireQuiet());
|
|
686
|
+
await stopExpiredChildren(expired);
|
|
434
687
|
for (const owner of book.ownerIds()) {
|
|
435
688
|
if (engine.isBot(owner))
|
|
436
689
|
await admit(owner);
|
|
@@ -439,11 +692,137 @@ export function createWorkRuntime(store, engine, options = {}) {
|
|
|
439
692
|
await reconcileChild(child);
|
|
440
693
|
for (const owner of book.ownerIds())
|
|
441
694
|
await reconcileOwner(owner);
|
|
695
|
+
await promoteQueued();
|
|
696
|
+
}
|
|
697
|
+
/**
|
|
698
|
+
* Renew the lease of a started worker that is still alive, so a silent
|
|
699
|
+
* model or tool turn longer than leaseMs does not false-expire it.
|
|
700
|
+
*/
|
|
701
|
+
async function renewLiveLeases() {
|
|
702
|
+
const renewals = [];
|
|
703
|
+
for (const child of book.childIds()) {
|
|
704
|
+
const attempt = book.attemptForChild(child);
|
|
705
|
+
if (!attempt || (attempt.status !== 'started' && attempt.status !== 'enqueued'))
|
|
706
|
+
continue;
|
|
707
|
+
if (!(await isChildLive(child)))
|
|
708
|
+
continue;
|
|
709
|
+
renewals.push(attempt.id);
|
|
710
|
+
}
|
|
711
|
+
if (renewals.length === 0)
|
|
712
|
+
return;
|
|
713
|
+
await store.write('work', () => {
|
|
714
|
+
const until = new Date(now().getTime() + limits.leaseMs).toISOString();
|
|
715
|
+
for (const id of renewals) {
|
|
716
|
+
const attempt = book.attempt(id);
|
|
717
|
+
if (!attempt || (attempt.status !== 'started' && attempt.status !== 'enqueued'))
|
|
718
|
+
continue;
|
|
719
|
+
book.touchLease(id, until, typeof attempt.lease_observed_seq === 'number' ? attempt.lease_observed_seq : null);
|
|
720
|
+
}
|
|
721
|
+
});
|
|
722
|
+
}
|
|
723
|
+
/**
|
|
724
|
+
* @param {string} sessionId
|
|
725
|
+
*/
|
|
726
|
+
async function isChildLive(sessionId) {
|
|
727
|
+
if (options.childPresent) {
|
|
728
|
+
try {
|
|
729
|
+
if (!(await options.childPresent(sessionId)))
|
|
730
|
+
return false;
|
|
731
|
+
}
|
|
732
|
+
catch {
|
|
733
|
+
return false;
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
else if (!options.snapshot) {
|
|
737
|
+
return false;
|
|
738
|
+
}
|
|
739
|
+
if (!options.snapshot)
|
|
740
|
+
return true;
|
|
741
|
+
try {
|
|
742
|
+
const events = await options.snapshot(sessionId);
|
|
743
|
+
return !latestTurnEnded(events);
|
|
744
|
+
}
|
|
745
|
+
catch {
|
|
746
|
+
return false;
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
/**
|
|
750
|
+
* @param {Array<{ ticketId: string, reason: string, childSessionId?: string | null }>} expired
|
|
751
|
+
*/
|
|
752
|
+
async function stopExpiredChildren(expired) {
|
|
753
|
+
if (!Array.isArray(expired) || expired.length === 0)
|
|
754
|
+
return;
|
|
755
|
+
for (const item of expired) {
|
|
756
|
+
const child = item.childSessionId;
|
|
757
|
+
if (!child || !options.cancel)
|
|
758
|
+
continue;
|
|
759
|
+
const state = arms.get(child);
|
|
760
|
+
if (state) {
|
|
761
|
+
state.closing = true;
|
|
762
|
+
state.clear?.();
|
|
763
|
+
}
|
|
764
|
+
try {
|
|
765
|
+
await options.cancel(child, { keepInbox: false, reason: `worker-expire:${item.reason}` });
|
|
766
|
+
}
|
|
767
|
+
catch {
|
|
768
|
+
/* best-effort stop; the ledger already failed the ticket */
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
/**
|
|
773
|
+
* @param {Record<string, any>} event
|
|
774
|
+
*/
|
|
775
|
+
function eventText(event) {
|
|
776
|
+
if (typeof event.text === 'string' && event.text !== '')
|
|
777
|
+
return event.text;
|
|
778
|
+
return book.detail(event.ticket_id)?.events?.find(item => item.event_id === event.event_id)?.text ?? '';
|
|
779
|
+
}
|
|
780
|
+
/**
|
|
781
|
+
* @param {Record<string, any>} event
|
|
782
|
+
*/
|
|
783
|
+
function isExtensionAsk(event) {
|
|
784
|
+
return event.kind === 'progress' && eventText(event).startsWith('[foundations:extension]');
|
|
785
|
+
}
|
|
786
|
+
/**
|
|
787
|
+
* Extension asks are decision-required: they may starve-bypass notice-open,
|
|
788
|
+
* user-pending, and delivery-open. Ordinary progress does not.
|
|
789
|
+
* @param {Record<string, any>} event
|
|
790
|
+
*/
|
|
791
|
+
function isDecisionRequiredEvent(event) {
|
|
792
|
+
return event.kind === 'question' || event.kind === 'result' || event.kind === 'failure' || isExtensionAsk(event);
|
|
793
|
+
}
|
|
794
|
+
/**
|
|
795
|
+
* @param {Record<string, any>} ticket
|
|
796
|
+
*/
|
|
797
|
+
function isFinishedTicket(ticket) {
|
|
798
|
+
const status = String(ticket.status);
|
|
799
|
+
return status === 'failed' || status === 'cancelled' || status === 'delivered';
|
|
442
800
|
}
|
|
443
801
|
/**
|
|
444
802
|
* @param {string} owner
|
|
445
803
|
*/
|
|
446
804
|
async function admit(owner) {
|
|
805
|
+
if (admitting.has(owner))
|
|
806
|
+
return { admitted: false, reason: 'admission-inflight' };
|
|
807
|
+
admitting.add(owner);
|
|
808
|
+
try {
|
|
809
|
+
return await admitOnce(owner);
|
|
810
|
+
}
|
|
811
|
+
finally {
|
|
812
|
+
admitting.delete(owner);
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
function currentExtension(event) {
|
|
816
|
+
const ticket = book.ticket(event.ticket_id);
|
|
817
|
+
if (!ticket || !['started', 'enqueued'].includes(ticket.status))
|
|
818
|
+
return false;
|
|
819
|
+
const budget = JSON.parse(ticket.budget_json);
|
|
820
|
+
if (budget.extension !== 'open')
|
|
821
|
+
return false;
|
|
822
|
+
const latest = book.detail(ticket.id)?.events.filter(isExtensionAsk).at(-1);
|
|
823
|
+
return latest?.event_id === event.event_id;
|
|
824
|
+
}
|
|
825
|
+
async function admitOnce(owner) {
|
|
447
826
|
const wake = book.pendingWake(owner);
|
|
448
827
|
if (wake.events.length === 0 && wake.cancels.length === 0)
|
|
449
828
|
return { admitted: false, reason: 'empty' };
|
|
@@ -456,15 +835,17 @@ export function createWorkRuntime(store, engine, options = {}) {
|
|
|
456
835
|
}
|
|
457
836
|
const terminalTimes = [];
|
|
458
837
|
for (const event of wake.events) {
|
|
459
|
-
if (
|
|
460
|
-
event
|
|
838
|
+
if (event.apply_status === 'applied' &&
|
|
839
|
+
isDecisionRequiredEvent(event) &&
|
|
840
|
+
(!isExtensionAsk(event) || currentExtension(event))) {
|
|
461
841
|
terminalTimes.push(Date.parse(event.created_at));
|
|
462
842
|
}
|
|
463
843
|
}
|
|
464
844
|
for (const ticket of wake.cancels)
|
|
465
845
|
terminalTimes.push(Date.parse(ticket.updated_at));
|
|
846
|
+
const urgent = wake.events.filter(event => isExtensionAsk(event) && currentExtension(event));
|
|
466
847
|
const heldUntil = holds.get(owner);
|
|
467
|
-
if (heldUntil != null && now().getTime() < heldUntil)
|
|
848
|
+
if (!urgent.length && heldUntil != null && now().getTime() < heldUntil)
|
|
468
849
|
return { admitted: false, reason: 'interrupt-hold' };
|
|
469
850
|
if (heldUntil != null)
|
|
470
851
|
holds.delete(owner);
|
|
@@ -479,21 +860,24 @@ export function createWorkRuntime(store, engine, options = {}) {
|
|
|
479
860
|
terminalSince: terminalTimes.length === 0 ? null : Math.min(...terminalTimes),
|
|
480
861
|
starveMs: limits.starveMs,
|
|
481
862
|
});
|
|
482
|
-
if (!decision.admit)
|
|
863
|
+
if (!urgent.length && !decision.admit)
|
|
483
864
|
return { admitted: false, ...decision };
|
|
484
865
|
const nativeId = randomUUID();
|
|
485
866
|
const items = [];
|
|
486
|
-
for (const event of wake.events) {
|
|
867
|
+
for (const event of urgent.length ? urgent : wake.events) {
|
|
487
868
|
const ticket = book.ticket(event.ticket_id);
|
|
488
869
|
if (!ticket)
|
|
489
870
|
continue;
|
|
871
|
+
// A grant cannot revive a finished ticket, so drop a stale extension ask.
|
|
872
|
+
if (isExtensionAsk(event) && !currentExtension(event))
|
|
873
|
+
continue;
|
|
490
874
|
items.push({
|
|
491
875
|
ticketId: ticket.id,
|
|
492
876
|
eventId: event.event_id,
|
|
493
877
|
line: noticeLine(ticket, event),
|
|
494
878
|
});
|
|
495
879
|
}
|
|
496
|
-
for (const ticket of wake.cancels) {
|
|
880
|
+
for (const ticket of urgent.length ? [] : wake.cancels) {
|
|
497
881
|
items.push({
|
|
498
882
|
ticketId: ticket.id,
|
|
499
883
|
eventId: `cancel:${ticket.cancel_requested_revision}`,
|
|
@@ -503,10 +887,24 @@ export function createWorkRuntime(store, engine, options = {}) {
|
|
|
503
887
|
if (items.length === 0)
|
|
504
888
|
return { admitted: false, reason: 'empty' };
|
|
505
889
|
await store.write('work', () => {
|
|
506
|
-
|
|
890
|
+
// Revalidate after the write queue: a grant/cancel may have raced admission.
|
|
891
|
+
const valid = items.filter(item => item.eventId.startsWith('cancel:') ||
|
|
892
|
+
(!book.event(item.eventId)?.notice_id &&
|
|
893
|
+
(!isExtensionAsk(book.event(item.eventId) ?? {}) || currentExtension(book.event(item.eventId)))));
|
|
894
|
+
items.splice(0, items.length, ...valid);
|
|
895
|
+
if (items.length)
|
|
896
|
+
book.insertNotices(nativeId, owner, items);
|
|
507
897
|
});
|
|
898
|
+
if (!items.length)
|
|
899
|
+
return { admitted: false, reason: 'superseded' };
|
|
508
900
|
const flushed = await pushNotice(book, nativeId, owner, renderNotice(items.map(item => item.line)), 0, deps);
|
|
509
|
-
return {
|
|
901
|
+
return {
|
|
902
|
+
...decision,
|
|
903
|
+
admitted: true,
|
|
904
|
+
reason: urgent.length ? 'budget-decision' : decision.reason,
|
|
905
|
+
nativeMessageId: nativeId,
|
|
906
|
+
flush: flushWord(flushed),
|
|
907
|
+
};
|
|
510
908
|
}
|
|
511
909
|
/**
|
|
512
910
|
* Progress stays a status line. Result and failure carry the body so the
|
|
@@ -519,6 +917,8 @@ export function createWorkRuntime(store, engine, options = {}) {
|
|
|
519
917
|
const head = `${ticket.id} ${status} (${LABEL[status] ?? status}) rev ${ticket.briefing_revision} ${event.kind}`;
|
|
520
918
|
const text = book.detail(ticket.id)?.events?.find(item => item.event_id === event.event_id)?.text ?? '';
|
|
521
919
|
if (text.startsWith('[foundations:extension]')) {
|
|
920
|
+
if (isFinishedTicket(ticket))
|
|
921
|
+
return head;
|
|
522
922
|
return [head, `eventId: ${event.event_id}`, text].join('\n');
|
|
523
923
|
}
|
|
524
924
|
if (event.kind !== 'result' && event.kind !== 'failure')
|
|
@@ -563,8 +963,15 @@ export function createWorkRuntime(store, engine, options = {}) {
|
|
|
563
963
|
wakeTurns.set(`${sessionId}:${event.data.turn}`, nativeId);
|
|
564
964
|
}
|
|
565
965
|
}
|
|
566
|
-
if (event.type === 'turn/end')
|
|
966
|
+
if (event.type === 'turn/end') {
|
|
967
|
+
if (isWorkerSession(sessionId) && event.data?.reason?.reason?.kind === 'user') {
|
|
968
|
+
const attempt = book.attemptForChild(sessionId);
|
|
969
|
+
if (attempt)
|
|
970
|
+
book.markCancelled(attempt.ticket_id, null, 'Worker cancelled by user.');
|
|
971
|
+
closeBudgetWait(sessionId);
|
|
972
|
+
}
|
|
567
973
|
applyTurnEnd(sessionId, event.data?.turn ?? null, reasonKindOf(event));
|
|
974
|
+
}
|
|
568
975
|
if (isWorkerSession(sessionId) && typeof event.seq === 'number') {
|
|
569
976
|
const attempt = book.attemptForChild(sessionId);
|
|
570
977
|
if (attempt && (attempt.status === 'started' || attempt.status === 'enqueued')) {
|
|
@@ -581,6 +988,8 @@ export function createWorkRuntime(store, engine, options = {}) {
|
|
|
581
988
|
if (book.inputByNative(messageId))
|
|
582
989
|
turns.set(sessionId, { messageId, turn, seq: null });
|
|
583
990
|
book.setNoticeObserve(messageId, 'claimed');
|
|
991
|
+
if (turn != null && book.noticeIds(sessionId).has(messageId))
|
|
992
|
+
wakeTurns.set(`${sessionId}:${turn}`, messageId);
|
|
584
993
|
}
|
|
585
994
|
/**
|
|
586
995
|
* @param {Record<string, any>} ticket
|
|
@@ -591,17 +1000,29 @@ export function createWorkRuntime(store, engine, options = {}) {
|
|
|
591
1000
|
if (!options.createChild)
|
|
592
1001
|
return { error: publicDispatch(ticket, { flush: 'pending' }) };
|
|
593
1002
|
const sessionId = `worker-${ticket.id}`;
|
|
1003
|
+
const attemptId = ticket.active_attempt_id;
|
|
1004
|
+
const fencingToken = book.attempt(attemptId)?.fencing_token ?? 1;
|
|
594
1005
|
try {
|
|
595
1006
|
const created = await options.createChild({
|
|
596
1007
|
sessionId,
|
|
597
1008
|
ownerSessionId: ticket.owner_session_id,
|
|
598
1009
|
ticketId: ticket.id,
|
|
599
|
-
fencingToken
|
|
1010
|
+
fencingToken,
|
|
600
1011
|
});
|
|
601
1012
|
const child = typeof created === 'string' && created !== '' ? created : sessionId;
|
|
1013
|
+
const saved = await store.write('work', () => book.setChildIfEmpty(ticket.id, child, { attemptId, fencingToken }));
|
|
1014
|
+
if (!saved?.attached) {
|
|
1015
|
+
try {
|
|
1016
|
+
if (options.cancel)
|
|
1017
|
+
await options.cancel(child, { keepInbox: false, reason: 'stale-start' });
|
|
1018
|
+
}
|
|
1019
|
+
catch {
|
|
1020
|
+
/* best-effort cleanup of a late or fenced child */
|
|
1021
|
+
}
|
|
1022
|
+
return { rejected: true, ticket: saved?.ticket ?? book.ticket(ticket.id) ?? ticket };
|
|
1023
|
+
}
|
|
602
1024
|
engine.ensureSession(child);
|
|
603
|
-
|
|
604
|
-
return { ticket: saved };
|
|
1025
|
+
return { ticket: saved.ticket };
|
|
605
1026
|
}
|
|
606
1027
|
catch {
|
|
607
1028
|
return {
|
|
@@ -678,11 +1099,13 @@ export function createWorkRuntime(store, engine, options = {}) {
|
|
|
678
1099
|
briefingRevision: fresh.briefing_revision,
|
|
679
1100
|
status: dispatched && flush === 'accepted' ? 'enqueued' : 'accepted',
|
|
680
1101
|
ticketStatus: fresh.status,
|
|
1102
|
+
...quotaFeedback(fresh.id),
|
|
681
1103
|
childSessionId: extra.childSessionId === null ? null : fresh.child_session_id,
|
|
682
1104
|
fencingToken: attempt?.fencing_token ?? null,
|
|
683
1105
|
nativeMessageId: extra.nativeMessageId === undefined ? (input?.native_message_id ?? null) : extra.nativeMessageId,
|
|
684
1106
|
flush,
|
|
685
1107
|
replayed: extra.replayed === true,
|
|
1108
|
+
queued: extra.queued === true || (fresh.status === 'accepted' && !book.slotHeld(fresh)),
|
|
686
1109
|
};
|
|
687
1110
|
}
|
|
688
1111
|
function project(ticket) {
|
|
@@ -721,6 +1144,7 @@ export function createWorkRuntime(store, engine, options = {}) {
|
|
|
721
1144
|
const found = book.detail(input.ticket_id);
|
|
722
1145
|
if (!found)
|
|
723
1146
|
return;
|
|
1147
|
+
let released = false;
|
|
724
1148
|
for (const event of found.events) {
|
|
725
1149
|
if (event.attempt_id !== input.attempt_id)
|
|
726
1150
|
continue;
|
|
@@ -729,7 +1153,10 @@ export function createWorkRuntime(store, engine, options = {}) {
|
|
|
729
1153
|
if (event.apply_status !== 'pending_completion')
|
|
730
1154
|
continue;
|
|
731
1155
|
book.applyTerminal(event.event_id, reason);
|
|
1156
|
+
released = true;
|
|
732
1157
|
}
|
|
1158
|
+
if (released)
|
|
1159
|
+
void promoteQueued().catch(() => { });
|
|
733
1160
|
}
|
|
734
1161
|
/**
|
|
735
1162
|
* @param {string} sessionId
|
|
@@ -760,7 +1187,7 @@ export function createWorkRuntime(store, engine, options = {}) {
|
|
|
760
1187
|
const nativeId = byTurn.get(event.data?.turn);
|
|
761
1188
|
if (!nativeId)
|
|
762
1189
|
continue;
|
|
763
|
-
|
|
1190
|
+
// Historical turn endings must not overwrite the latest live input binding.
|
|
764
1191
|
const input = book.inputByNative(nativeId);
|
|
765
1192
|
if (!input)
|
|
766
1193
|
continue;
|
|
@@ -874,6 +1301,7 @@ export function createWorkRuntime(store, engine, options = {}) {
|
|
|
874
1301
|
state.tokens += total;
|
|
875
1302
|
if (seq != null)
|
|
876
1303
|
state.lastSeq = seq;
|
|
1304
|
+
saveQuota(sessionId, state);
|
|
877
1305
|
consider(sessionId, state, false);
|
|
878
1306
|
}
|
|
879
1307
|
return;
|
|
@@ -1028,11 +1456,12 @@ export function createWorkRuntime(store, engine, options = {}) {
|
|
|
1028
1456
|
}
|
|
1029
1457
|
: workerCaps(sessionId);
|
|
1030
1458
|
const state = {
|
|
1459
|
+
attemptId: role === 'worker' ? (book.attemptForChild(sessionId)?.id ?? null) : null,
|
|
1031
1460
|
turn,
|
|
1032
|
-
startedAt: previous?.startedAt ?? now().getTime(),
|
|
1461
|
+
startedAt: role === 'worker' ? now().getTime() : (previous?.startedAt ?? now().getTime()),
|
|
1033
1462
|
steps: 0,
|
|
1034
|
-
tokens: previous?.tokens ?? 0,
|
|
1035
|
-
lastSeq: previous?.lastSeq ?? null,
|
|
1463
|
+
tokens: role === 'worker' ? 0 : (previous?.tokens ?? 0),
|
|
1464
|
+
lastSeq: role === 'worker' ? null : (previous?.lastSeq ?? null),
|
|
1036
1465
|
role,
|
|
1037
1466
|
tight: false,
|
|
1038
1467
|
limits: caps,
|
|
@@ -1040,18 +1469,37 @@ export function createWorkRuntime(store, engine, options = {}) {
|
|
|
1040
1469
|
budgetNotice: false,
|
|
1041
1470
|
extensionOpen: false,
|
|
1042
1471
|
closing: false,
|
|
1472
|
+
pausedAt: null,
|
|
1473
|
+
atBoundary: false,
|
|
1043
1474
|
clear: () => { },
|
|
1044
1475
|
};
|
|
1045
|
-
|
|
1476
|
+
if (role === 'worker') {
|
|
1477
|
+
const attempt = book.attemptForChild(sessionId);
|
|
1478
|
+
const quota = attempt ? quotaOf(attempt.ticket_id) : null;
|
|
1479
|
+
if (quota) {
|
|
1480
|
+
state.tokens = quota.tokens ?? 0;
|
|
1481
|
+
state.lastSeq = quota.lastSeq ?? null;
|
|
1482
|
+
const elapsed = (quota.activeMs ?? 0) + (quota.runningSince == null ? 0 : Math.max(0, now().getTime() - quota.runningSince));
|
|
1483
|
+
state.startedAt = now().getTime() - elapsed;
|
|
1484
|
+
if (quota.waiting) {
|
|
1485
|
+
state.pausedAt = now().getTime();
|
|
1486
|
+
state.atBoundary = true;
|
|
1487
|
+
}
|
|
1488
|
+
}
|
|
1489
|
+
}
|
|
1490
|
+
const elapsed = Math.max(0, (state.pausedAt != null && state.atBoundary ? state.pausedAt : now().getTime()) - state.startedAt);
|
|
1046
1491
|
state.extensionOpen = previous?.extensionOpen === true || caps.extensionOpen === true;
|
|
1047
|
-
const target =
|
|
1048
|
-
state.
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1492
|
+
const target = caps.wallMs;
|
|
1493
|
+
if (state.pausedAt == null)
|
|
1494
|
+
state.clear = schedule(() => {
|
|
1495
|
+
const live = arms.get(sessionId);
|
|
1496
|
+
if (!live || live !== state || live.closing)
|
|
1497
|
+
return;
|
|
1498
|
+
consider(sessionId, live, true);
|
|
1499
|
+
}, Math.max(1, target - elapsed));
|
|
1054
1500
|
arms.set(sessionId, state);
|
|
1501
|
+
if (role === 'worker')
|
|
1502
|
+
saveQuota(sessionId, state);
|
|
1055
1503
|
return state;
|
|
1056
1504
|
}
|
|
1057
1505
|
/**
|
|
@@ -1104,26 +1552,14 @@ export function createWorkRuntime(store, engine, options = {}) {
|
|
|
1104
1552
|
Promise.resolve(options.cancel?.(sessionId, { keepInbox: true, reason: trip.cause })).catch(() => { });
|
|
1105
1553
|
return;
|
|
1106
1554
|
}
|
|
1107
|
-
|
|
1108
|
-
if (state.tokens >= state.limits.ceilingTokens || elapsed >= state.limits.ceilingWall) {
|
|
1109
|
-
finishWorker(sessionId, state, state.tokens >= state.limits.ceilingTokens && elapsed < state.limits.ceilingWall ? 'tokens' : 'wall');
|
|
1110
|
-
return;
|
|
1111
|
-
}
|
|
1112
|
-
if (state.extensionOpen)
|
|
1555
|
+
if (state.pausedAt != null)
|
|
1113
1556
|
return;
|
|
1557
|
+
const elapsed = Math.max(0, now().getTime() - state.startedAt);
|
|
1114
1558
|
const overWall = fromTimer || elapsed >= state.limits.wallMs;
|
|
1115
1559
|
const overTokens = state.tokens >= state.limits.maxTokens;
|
|
1116
1560
|
if (!overWall && !overTokens)
|
|
1117
1561
|
return;
|
|
1118
|
-
|
|
1119
|
-
askForQuota(sessionId, state, 'wall', elapsed);
|
|
1120
|
-
return;
|
|
1121
|
-
}
|
|
1122
|
-
if (overTokens && state.limits.maxTokens < state.limits.ceilingTokens) {
|
|
1123
|
-
askForQuota(sessionId, state, 'tokens', elapsed);
|
|
1124
|
-
return;
|
|
1125
|
-
}
|
|
1126
|
-
finishWorker(sessionId, state, overTokens ? 'tokens' : 'wall');
|
|
1562
|
+
askForQuota(sessionId, state, overTokens ? 'tokens' : 'wall', elapsed);
|
|
1127
1563
|
}
|
|
1128
1564
|
/**
|
|
1129
1565
|
* @param {string} sessionId
|
|
@@ -1134,13 +1570,15 @@ export function createWorkRuntime(store, engine, options = {}) {
|
|
|
1134
1570
|
function askForQuota(sessionId, state, reason, elapsed) {
|
|
1135
1571
|
state.extensionOpen = true;
|
|
1136
1572
|
state.clear?.();
|
|
1137
|
-
|
|
1573
|
+
state.pausedAt = now().getTime();
|
|
1574
|
+
state.atBoundary = false;
|
|
1575
|
+
saveQuota(sessionId, state, reason);
|
|
1138
1576
|
const attempt = book.attemptForChild(sessionId);
|
|
1139
1577
|
if (!attempt)
|
|
1140
1578
|
return;
|
|
1141
1579
|
const ticket = book.ticket(attempt.ticket_id);
|
|
1142
1580
|
if (!ticket || ticket.status === 'failed' || ticket.status === 'cancelled' || ticket.status === 'delivered') {
|
|
1143
|
-
|
|
1581
|
+
closeBudgetWait(sessionId);
|
|
1144
1582
|
return;
|
|
1145
1583
|
}
|
|
1146
1584
|
const marked = book.markExtensionOpen(ticket.id);
|
|
@@ -1153,7 +1591,10 @@ export function createWorkRuntime(store, engine, options = {}) {
|
|
|
1153
1591
|
`granted wall ms: ${state.limits.wallMs}`,
|
|
1154
1592
|
`tokens used: ${state.tokens}`,
|
|
1155
1593
|
`granted tokens: ${state.limits.maxTokens}`,
|
|
1156
|
-
|
|
1594
|
+
`host ceiling tokens: ${state.limits.ceilingTokens}; host ceiling active wall ms: ${state.limits.ceilingWall}`,
|
|
1595
|
+
`required total tokens: ${state.tokens + 1}; required total active wall ms: ${elapsed + 1}`,
|
|
1596
|
+
'Worker is waiting for budget at the next safe loop boundary. It will not dispatch another step until sufficient budget is granted. Budget wait does not expire.',
|
|
1597
|
+
'Call foundations_grant with amounts to ADD, sufficient to exceed current cumulative usage. Host ceilings cannot be raised by this tool; at a host ceiling an explicit deny/cancel is required. Pass deny=true to stop. Do not redispatch a replacement ticket.',
|
|
1157
1598
|
].join('\n');
|
|
1158
1599
|
book.insertEvent({
|
|
1159
1600
|
ticketId: ticket.id,
|
|
@@ -1165,74 +1606,192 @@ export function createWorkRuntime(store, engine, options = {}) {
|
|
|
1165
1606
|
applyStatus: 'applied',
|
|
1166
1607
|
});
|
|
1167
1608
|
}
|
|
1168
|
-
/**
|
|
1169
|
-
* After a quota request, the worker keeps running until the host ceiling.
|
|
1170
|
-
* An early timer is rearmed. It does not fail the ticket by itself.
|
|
1171
|
-
* @param {string} sessionId
|
|
1172
|
-
* @param {{ startedAt: number, limits: { ceilingWall: number }, closing: boolean, clear: () => void }} state
|
|
1173
|
-
*/
|
|
1174
|
-
function watchHostCeiling(sessionId, state) {
|
|
1175
|
-
const elapsed = Math.max(0, now().getTime() - state.startedAt);
|
|
1176
|
-
state.clear = schedule(() => {
|
|
1177
|
-
const live = arms.get(sessionId);
|
|
1178
|
-
if (!live || live !== state || live.closing)
|
|
1179
|
-
return;
|
|
1180
|
-
const at = Math.max(0, now().getTime() - live.startedAt);
|
|
1181
|
-
if (at >= live.limits.ceilingWall) {
|
|
1182
|
-
finishWorker(sessionId, live, 'wall');
|
|
1183
|
-
return;
|
|
1184
|
-
}
|
|
1185
|
-
watchHostCeiling(sessionId, live);
|
|
1186
|
-
}, Math.max(1000, state.limits.ceilingWall - elapsed));
|
|
1187
|
-
}
|
|
1188
|
-
/**
|
|
1189
|
-
* @param {string} sessionId
|
|
1190
|
-
* @param {{ closing: boolean, clear: () => void }} state
|
|
1191
|
-
* @param {string} reason
|
|
1192
|
-
*/
|
|
1193
|
-
function finishWorker(sessionId, state, reason) {
|
|
1194
|
-
if (state.closing)
|
|
1195
|
-
return;
|
|
1196
|
-
state.closing = true;
|
|
1197
|
-
state.clear?.();
|
|
1198
|
-
Promise.resolve(options.cancel?.(sessionId, { keepInbox: false, reason: `worker-budget:${reason}` })).catch(() => { });
|
|
1199
|
-
const attempt = book.attemptForChild(sessionId);
|
|
1200
|
-
if (attempt)
|
|
1201
|
-
book.failRunning(attempt.ticket_id, reason);
|
|
1202
|
-
}
|
|
1203
1609
|
/**
|
|
1204
1610
|
* @param {string} sessionId
|
|
1205
1611
|
* @param {{ wallMs: number, maxTokens: number }} budget
|
|
1206
1612
|
*/
|
|
1207
1613
|
function releaseArm(sessionId, budget) {
|
|
1208
1614
|
const state = arms.get(sessionId);
|
|
1209
|
-
|
|
1615
|
+
const attempt = book.attemptForChild(sessionId);
|
|
1616
|
+
if (!attempt)
|
|
1210
1617
|
return;
|
|
1211
|
-
|
|
1618
|
+
const quota = quotaOf(attempt.ticket_id);
|
|
1619
|
+
if (!state || state.role !== 'worker' || state.closing) {
|
|
1620
|
+
// A detached turn stays parked until the grant path queues a durable continuation.
|
|
1621
|
+
if (quota)
|
|
1622
|
+
writeQuota(attempt.ticket_id, { ...quota, waiting: true, runningSince: null });
|
|
1623
|
+
return;
|
|
1624
|
+
}
|
|
1212
1625
|
state.limits.wallMs = budget.wallMs;
|
|
1213
1626
|
state.limits.maxTokens = budget.maxTokens;
|
|
1627
|
+
const elapsed = Math.max(0, (state.pausedAt != null && state.atBoundary ? state.pausedAt : now().getTime()) - state.startedAt);
|
|
1628
|
+
if (state.tokens >= budget.maxTokens ||
|
|
1629
|
+
elapsed >= budget.wallMs ||
|
|
1630
|
+
state.tokens >= state.limits.ceilingTokens ||
|
|
1631
|
+
elapsed >= state.limits.ceilingWall) {
|
|
1632
|
+
state.extensionOpen = true;
|
|
1633
|
+
if (state.pausedAt == null)
|
|
1634
|
+
state.pausedAt = now().getTime();
|
|
1635
|
+
state.clear();
|
|
1636
|
+
saveQuota(sessionId, state);
|
|
1637
|
+
book.markExtensionOpen(attempt.ticket_id);
|
|
1638
|
+
return;
|
|
1639
|
+
}
|
|
1640
|
+
state.extensionOpen = false;
|
|
1641
|
+
state.startedAt = now().getTime() - elapsed;
|
|
1642
|
+
state.pausedAt = null;
|
|
1643
|
+
state.atBoundary = false;
|
|
1214
1644
|
state.clear?.();
|
|
1215
|
-
|
|
1645
|
+
saveQuota(sessionId, state);
|
|
1216
1646
|
state.clear = schedule(() => {
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
return;
|
|
1220
|
-
consider(sessionId, live, true);
|
|
1647
|
+
if (arms.get(sessionId) === state && !state.closing)
|
|
1648
|
+
consider(sessionId, state, true);
|
|
1221
1649
|
}, Math.max(1, budget.wallMs - elapsed));
|
|
1222
|
-
|
|
1223
|
-
|
|
1650
|
+
for (const resolve of budgetWaiters.get(sessionId) ?? [])
|
|
1651
|
+
resolve(true);
|
|
1652
|
+
budgetWaiters.delete(sessionId);
|
|
1653
|
+
}
|
|
1654
|
+
function quotaOf(ticketId) {
|
|
1655
|
+
try {
|
|
1656
|
+
return JSON.parse(book.ticket(ticketId)?.budget_json ?? '{}').quota ?? null;
|
|
1657
|
+
}
|
|
1658
|
+
catch {
|
|
1659
|
+
return null;
|
|
1660
|
+
}
|
|
1661
|
+
}
|
|
1662
|
+
function writeQuota(ticketId, quota) {
|
|
1663
|
+
const ticket = book.ticket(ticketId);
|
|
1664
|
+
if (!ticket)
|
|
1665
|
+
return;
|
|
1666
|
+
const budget = JSON.parse(ticket.budget_json);
|
|
1667
|
+
budget.quota = quota;
|
|
1668
|
+
store.db
|
|
1669
|
+
.prepare('UPDATE work_tickets SET budget_json = ?, updated_at = ? WHERE id = ?')
|
|
1670
|
+
.run(JSON.stringify(budget), now().toISOString(), ticketId);
|
|
1671
|
+
}
|
|
1672
|
+
function saveQuota(sessionId, state, reason) {
|
|
1673
|
+
const attempt = book.attemptForChild(sessionId);
|
|
1674
|
+
if (!attempt || attempt.id !== state.attemptId)
|
|
1675
|
+
return;
|
|
1676
|
+
const activeMs = Math.max(0, (state.pausedAt != null && state.atBoundary ? state.pausedAt : now().getTime()) - state.startedAt);
|
|
1677
|
+
writeQuota(attempt.ticket_id, {
|
|
1678
|
+
tokens: state.tokens,
|
|
1679
|
+
lastSeq: state.lastSeq,
|
|
1680
|
+
activeMs,
|
|
1681
|
+
runningSince: state.pausedAt == null ? now().getTime() : null,
|
|
1682
|
+
waiting: state.pausedAt != null,
|
|
1683
|
+
phase: state.atBoundary ? 'waiting' : 'requested',
|
|
1684
|
+
reason: state.pausedAt == null
|
|
1685
|
+
? undefined
|
|
1686
|
+
: state.tokens >= state.limits.ceilingTokens || activeMs >= state.limits.ceilingWall
|
|
1687
|
+
? 'host-ceiling'
|
|
1688
|
+
: (reason ?? quotaOf(attempt.ticket_id)?.reason),
|
|
1689
|
+
});
|
|
1690
|
+
}
|
|
1691
|
+
function quotaFeedback(ticketId) {
|
|
1692
|
+
const quota = quotaOf(ticketId);
|
|
1693
|
+
const budget = JSON.parse(book.ticket(ticketId)?.budget_json ?? '{}');
|
|
1694
|
+
return {
|
|
1695
|
+
waitingBudget: quota?.waiting === true,
|
|
1696
|
+
budgetReason: quota?.reason ?? null,
|
|
1697
|
+
usedTokens: quota?.tokens ?? 0,
|
|
1698
|
+
activeElapsedMs: quota?.activeMs ?? 0,
|
|
1699
|
+
requiredMaxTokens: (quota?.tokens ?? 0) + 1,
|
|
1700
|
+
requiredWallMs: (quota?.activeMs ?? 0) + 1,
|
|
1701
|
+
minimumAdditionalTokens: Math.max(0, (quota?.tokens ?? 0) + 1 - (budget.maxTokens ?? 0)),
|
|
1702
|
+
minimumAdditionalWallMs: Math.max(0, (quota?.activeMs ?? 0) + 1 - (budget.wallMs ?? 0)),
|
|
1703
|
+
hostTokenHeadroom: Math.max(0, limits.workerMaxTokens - (quota?.tokens ?? 0)),
|
|
1704
|
+
hostWallHeadroomMs: Math.max(0, limits.workerWallMs - (quota?.activeMs ?? 0)),
|
|
1705
|
+
};
|
|
1706
|
+
}
|
|
1707
|
+
function closeBudgetWait(sessionId) {
|
|
1708
|
+
const state = arms.get(sessionId);
|
|
1709
|
+
if (state) {
|
|
1710
|
+
state.closing = true;
|
|
1711
|
+
state.clear();
|
|
1712
|
+
}
|
|
1713
|
+
for (const resolve of budgetWaiters.get(sessionId) ?? [])
|
|
1714
|
+
resolve(false);
|
|
1715
|
+
budgetWaiters.delete(sessionId);
|
|
1716
|
+
}
|
|
1717
|
+
/** Awaiting this official pre-step waterfall suspends admission, not the turn. */
|
|
1718
|
+
async function waitForBudget(sessionId, signal) {
|
|
1719
|
+
if (!isWorkerSession(sessionId))
|
|
1720
|
+
return true;
|
|
1721
|
+
const attempt = book.attemptForChild(sessionId);
|
|
1722
|
+
const ticket = attempt ? book.ticket(attempt.ticket_id) : null;
|
|
1723
|
+
if (!ticket || isFinishedTicket(ticket) || signal?.aborted)
|
|
1724
|
+
return false;
|
|
1725
|
+
const state = arms.get(sessionId) ?? arm(sessionId, 0, 'worker');
|
|
1726
|
+
if (state.closing)
|
|
1727
|
+
return false;
|
|
1728
|
+
consider(sessionId, state, false);
|
|
1729
|
+
if (state.pausedAt == null)
|
|
1730
|
+
return true;
|
|
1731
|
+
if (!state.atBoundary) {
|
|
1732
|
+
// The already-admitted request/tool batch may finish; account for that
|
|
1733
|
+
// active time before freezing the clock at this safe loop boundary.
|
|
1734
|
+
state.pausedAt = now().getTime();
|
|
1735
|
+
state.atBoundary = true;
|
|
1736
|
+
saveQuota(sessionId, state);
|
|
1737
|
+
}
|
|
1738
|
+
const caps = workerCaps(sessionId);
|
|
1739
|
+
releaseArm(sessionId, caps);
|
|
1740
|
+
if (state.pausedAt == null)
|
|
1741
|
+
return true;
|
|
1742
|
+
return new Promise(resolve => {
|
|
1743
|
+
const waiters = budgetWaiters.get(sessionId) ?? new Set();
|
|
1744
|
+
const done = (allowed) => {
|
|
1745
|
+
signal?.removeEventListener('abort', aborted);
|
|
1746
|
+
waiters.delete(done);
|
|
1747
|
+
if (!waiters.size)
|
|
1748
|
+
budgetWaiters.delete(sessionId);
|
|
1749
|
+
resolve(allowed);
|
|
1750
|
+
};
|
|
1751
|
+
const aborted = () => done(false);
|
|
1752
|
+
waiters.add(done);
|
|
1753
|
+
budgetWaiters.set(sessionId, waiters);
|
|
1754
|
+
signal?.addEventListener('abort', aborted, { once: true });
|
|
1755
|
+
if (signal?.aborted)
|
|
1756
|
+
done(false);
|
|
1757
|
+
});
|
|
1224
1758
|
}
|
|
1225
1759
|
/**
|
|
1226
1760
|
* @param {string} sessionId
|
|
1227
1761
|
*/
|
|
1228
1762
|
function clearBudget(sessionId) {
|
|
1229
1763
|
const state = arms.get(sessionId);
|
|
1764
|
+
if (state?.role === 'worker')
|
|
1765
|
+
saveQuota(sessionId, state);
|
|
1230
1766
|
if (!state)
|
|
1231
1767
|
return;
|
|
1232
1768
|
state.clear?.();
|
|
1233
1769
|
arms.delete(sessionId);
|
|
1234
1770
|
}
|
|
1235
1771
|
}
|
|
1772
|
+
/**
|
|
1773
|
+
* A later turn/end for every started turn means the child is idle.
|
|
1774
|
+
* No turn at all, or an open turn, means it may still be running.
|
|
1775
|
+
* @param {Array<Record<string, any>>} events
|
|
1776
|
+
*/
|
|
1777
|
+
function latestTurnEnded(events) {
|
|
1778
|
+
const open = new Set();
|
|
1779
|
+
let sawTurn = false;
|
|
1780
|
+
for (const event of events) {
|
|
1781
|
+
const turn = event.data?.turn;
|
|
1782
|
+
if (turn == null)
|
|
1783
|
+
continue;
|
|
1784
|
+
if (event.type === 'turn/start' || event.type === 'user/message') {
|
|
1785
|
+
sawTurn = true;
|
|
1786
|
+
open.add(turn);
|
|
1787
|
+
}
|
|
1788
|
+
else if (event.type === 'turn/end') {
|
|
1789
|
+
sawTurn = true;
|
|
1790
|
+
open.delete(turn);
|
|
1791
|
+
}
|
|
1792
|
+
}
|
|
1793
|
+
return sawTurn && open.size === 0;
|
|
1794
|
+
}
|
|
1236
1795
|
/**
|
|
1237
1796
|
* @param {unknown} value
|
|
1238
1797
|
*/
|