libfx 0.0.13-dev.1472.g2c7a4f83f06f → 0.0.13-dev.1510.ga878d4900a6a
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +60 -23
- package/browser.js +1 -1
- package/durable/local.mjs +55 -55
- package/durable/vercel.mjs +64 -64
- package/durable/world.mjs +1 -1
- package/durable.js +504 -104
- package/fx-core.wasm +0 -0
- package/fx-harness.js +20 -4
- package/fx-sdk.js +92 -36
- package/fx-term.wasm +0 -0
- package/libfx.darwin-arm64.node +0 -0
- package/libfx.darwin-x64.node +0 -0
- package/libfx.linux-arm64.node +0 -0
- package/libfx.linux-x64.node +0 -0
- package/mcp.js +3 -0
- package/node.cjs +446 -125
- package/node.js +22 -6
- package/package.json +1 -1
- package/types/browser.d.ts +5 -1
- package/types/durable-vercel.d.ts +8 -2
- package/types/durable-world.d.ts +16 -0
- package/types/libfx.d.cts +62 -8
- package/types/mcp.d.ts +3 -2
- package/types/node.d.cts +6 -1
- package/types/skills-node.d.ts +3 -3
- package/types/skills.d.ts +4 -3
package/durable.js
CHANGED
|
@@ -49,10 +49,18 @@ const defaultReserveMs = 30_000;
|
|
|
49
49
|
// after its last renewal, not at its function's deadline.
|
|
50
50
|
const defaultLeaseMs = 15_000;
|
|
51
51
|
|
|
52
|
+
// Whether the queue shows whether a worker lives, delivering the message its
|
|
53
|
+
// delivery held again once it is gone, so the worker holds its lease to the
|
|
54
|
+
// deadline that bounds it and renews nothing.
|
|
55
|
+
const queueHeld = (backend, deadline) => !backend.livenessKnown && backend.queueLeases === true && deadline !== null;
|
|
56
|
+
// Whether a worker holds its lease to its deadline and renews nothing: when
|
|
57
|
+
// whether it lives is known, or the queue shows it.
|
|
58
|
+
const heldToDeadline = (backend, deadline) => backend.livenessKnown || queueHeld(backend, deadline);
|
|
59
|
+
|
|
52
60
|
// When a lease taken or renewed now runs out: never past the deadline its
|
|
53
|
-
// worker stops at, and
|
|
61
|
+
// worker stops at, and when it is held to that deadline, only there.
|
|
54
62
|
function leaseEnd(backend, deadline) {
|
|
55
|
-
if (backend
|
|
63
|
+
if (heldToDeadline(backend, deadline)) return deadline;
|
|
56
64
|
const renewed = Date.now() + backend.leaseMs;
|
|
57
65
|
return deadline === null ? renewed : Math.min(renewed, deadline);
|
|
58
66
|
}
|
|
@@ -162,6 +170,10 @@ export function foldSessionLog(entries, { now = 0, alive = () => null } = {}) {
|
|
|
162
170
|
// A checkpoint taken where a turn yielded leaves that turn open.
|
|
163
171
|
let openTurn = typeof base?.openTurn?.id === "string" ? { id: base.openTurn.id, at: Number(base.openTurn.at) || 0, context: base.openTurn.context ?? null, settings: base.openTurn.settings ?? null } : null;
|
|
164
172
|
let lastTurnId = base?.lastTurnId ?? null;
|
|
173
|
+
// The agent the session belongs to: the one its first stored input, or the
|
|
174
|
+
// checkpoint it was restored from, names. A session stored before agents
|
|
175
|
+
// had names belongs to the default agent.
|
|
176
|
+
let agent = typeof base?.agent === "string" ? base.agent : base && base.through != null ? defaultAgentName : null;
|
|
165
177
|
let yielded = base?.yielded === true;
|
|
166
178
|
// The context each prompt's or steer's caller gave, and the settings its
|
|
167
179
|
// session object set, which its turn runs with: an untaken steer becomes a
|
|
@@ -193,6 +205,7 @@ export function foldSessionLog(entries, { now = 0, alive = () => null } = {}) {
|
|
|
193
205
|
if (position > maxCursor) maxCursor = position;
|
|
194
206
|
switch (entry?.k) {
|
|
195
207
|
case "input":
|
|
208
|
+
agent ??= typeof entry.agent === "string" ? entry.agent : defaultAgentName;
|
|
196
209
|
if (seenInputs.has(entry.key)) break;
|
|
197
210
|
seenInputs.add(entry.key);
|
|
198
211
|
inputs.push({ ...entry, cursor: position });
|
|
@@ -207,7 +220,7 @@ export function foldSessionLog(entries, { now = 0, alive = () => null } = {}) {
|
|
|
207
220
|
if ((entry.a ?? null) !== head) break;
|
|
208
221
|
head = cursor;
|
|
209
222
|
if (entry.k === "lease") {
|
|
210
|
-
lease = { holder: entry.holder, epoch: entry.epoch, expiresAt: entry.expiresAt ?? null, pid: entry.pid, host: entry.host, cursor };
|
|
223
|
+
lease = { holder: entry.holder, epoch: entry.epoch, expiresAt: entry.expiresAt ?? null, pid: entry.pid, host: entry.host, delivery: entry.delivery, cursor };
|
|
211
224
|
if (Number.isSafeInteger(entry.epoch) && entry.epoch > epoch) epoch = entry.epoch;
|
|
212
225
|
} else if (entry.k === "release") {
|
|
213
226
|
if (lease?.holder === entry.holder) lease = null;
|
|
@@ -309,6 +322,7 @@ export function foldSessionLog(entries, { now = 0, alive = () => null } = {}) {
|
|
|
309
322
|
cutoffs: openTurn === null ? 0 : cut.cutoffs,
|
|
310
323
|
openFailed,
|
|
311
324
|
lastTurnId,
|
|
325
|
+
agent,
|
|
312
326
|
yielded,
|
|
313
327
|
pending,
|
|
314
328
|
steers,
|
|
@@ -334,15 +348,38 @@ function leaseStands(lease, now, alive) {
|
|
|
334
348
|
return true;
|
|
335
349
|
}
|
|
336
350
|
|
|
351
|
+
// Which queue message a delivery carries, by the queue's own id, and which
|
|
352
|
+
// delivery of it this is, when the World reports both.
|
|
353
|
+
function deliveryOf(meta) {
|
|
354
|
+
const messageId = meta?.messageId;
|
|
355
|
+
const attempt = meta?.attempt;
|
|
356
|
+
return typeof messageId === "string" && messageId.length > 0 && Number.isSafeInteger(attempt) && attempt > 0
|
|
357
|
+
? { messageId, attempt }
|
|
358
|
+
: null;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
// Whether `delivery`, in another process, carries the message whose delivery
|
|
362
|
+
// took `lease`. A queue hands a message on only once the delivery holding it
|
|
363
|
+
// let it go unacknowledged: its handler failed, or its process died or lost
|
|
364
|
+
// the message. On Vercel, a function whose process exits mid-delivery runs
|
|
365
|
+
// the same delivery again, attempt and all, in a new process. Either way the
|
|
366
|
+
// holder no longer runs the session, or runs it replaced without knowing,
|
|
367
|
+
// and its next write is fenced.
|
|
368
|
+
function redelivers(delivery, lease) {
|
|
369
|
+
const held = lease?.delivery;
|
|
370
|
+
return Boolean(delivery && held && held.messageId === delivery.messageId &&
|
|
371
|
+
Number.isSafeInteger(held.attempt) && delivery.attempt >= held.attempt);
|
|
372
|
+
}
|
|
373
|
+
|
|
337
374
|
// ---------------------------------------------------------------------------
|
|
338
375
|
// Backends. Each gives sessions a log, a stream, and a queue.
|
|
339
376
|
|
|
340
377
|
function createMemoryBackend({ maxDurationMs, reserveMs } = {}) {
|
|
341
378
|
const sessions = new Map();
|
|
342
379
|
const holders = liveHolders;
|
|
343
|
-
//
|
|
344
|
-
|
|
345
|
-
|
|
380
|
+
// The agents sharing the backend, by name: an agent's copies take the
|
|
381
|
+
// deliveries for its name in turn.
|
|
382
|
+
const handlers = new Map();
|
|
346
383
|
const timers = new Set();
|
|
347
384
|
const sessionFor = (id) => {
|
|
348
385
|
let session = sessions.get(id);
|
|
@@ -356,20 +393,28 @@ function createMemoryBackend({ maxDurationMs, reserveMs } = {}) {
|
|
|
356
393
|
const timer = setTimeout(() => { timers.delete(timer); fn(); }, ms);
|
|
357
394
|
timers.add(timer);
|
|
358
395
|
};
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
396
|
+
// A message keeps its id across its deliveries, as a queue's does.
|
|
397
|
+
const deliver = (message, attempt, messageId = newId("qmsg")) => {
|
|
398
|
+
if (handlers.size === 0) return later(() => deliver(message, attempt, messageId), 10);
|
|
399
|
+
const group = handlers.get(agentOf(message));
|
|
400
|
+
// No agent of that name here yet: tried again, as a failed delivery is.
|
|
401
|
+
if (!group) {
|
|
402
|
+
if (attempt < 48) later(() => deliver(message, attempt + 1, messageId), 1000);
|
|
403
|
+
return;
|
|
404
|
+
}
|
|
405
|
+
const handler = group.members[group.turn++ % group.members.length];
|
|
362
406
|
Promise.resolve()
|
|
363
|
-
.then(() => handler(message, { attempt }))
|
|
407
|
+
.then(() => handler(message, { messageId, attempt }))
|
|
364
408
|
.then((result) => {
|
|
365
|
-
if (Number.isFinite(result?.timeoutSeconds)) later(() => deliver(message, attempt + 1), result.timeoutSeconds * 1000);
|
|
409
|
+
if (Number.isFinite(result?.timeoutSeconds)) later(() => deliver(message, attempt + 1, messageId), result.timeoutSeconds * 1000);
|
|
366
410
|
}, () => {
|
|
367
|
-
if (attempt < 48) later(() => deliver(message, attempt + 1), 1000);
|
|
411
|
+
if (attempt < 48) later(() => deliver(message, attempt + 1, messageId), 1000);
|
|
368
412
|
});
|
|
369
413
|
};
|
|
370
414
|
return {
|
|
371
415
|
name: "memory",
|
|
372
416
|
livenessKnown: true,
|
|
417
|
+
runInRequest: false,
|
|
373
418
|
queueDurable: false,
|
|
374
419
|
pollMs: null,
|
|
375
420
|
reserveMs: reserveMs ?? defaultReserveMs,
|
|
@@ -448,12 +493,11 @@ function createMemoryBackend({ maxDurationMs, reserveMs } = {}) {
|
|
|
448
493
|
later(() => deliver(message, 1), delaySeconds * 1000);
|
|
449
494
|
},
|
|
450
495
|
},
|
|
451
|
-
listen(fn) {
|
|
452
|
-
handlers
|
|
496
|
+
listen(fn, name = defaultAgentName) {
|
|
497
|
+
const group = joinGroup(handlers, name, fn);
|
|
453
498
|
return () => {
|
|
454
|
-
|
|
455
|
-
if (
|
|
456
|
-
if (handlers.length > 0) return;
|
|
499
|
+
leaveGroup(handlers, name, group, fn);
|
|
500
|
+
if (handlers.size > 0) return;
|
|
457
501
|
// Nobody is left to deliver to.
|
|
458
502
|
for (const timer of timers) clearTimeout(timer);
|
|
459
503
|
timers.clear();
|
|
@@ -470,6 +514,12 @@ const logStep = "fx.log";
|
|
|
470
514
|
// "libfx-ui".
|
|
471
515
|
const uiStreamOf = (runId) => `${runId.replace(/^wrun_/, "strm_")}_user_bGliZngtdWk`;
|
|
472
516
|
const queuePrefix = "__libfx_wkf_workflow_";
|
|
517
|
+
// The name of an agent created without one, and of whatever a message or a
|
|
518
|
+
// session from before names belonged to.
|
|
519
|
+
const defaultAgentName = "default";
|
|
520
|
+
const agentNameRule = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
|
|
521
|
+
// The agent a queued message is for.
|
|
522
|
+
const agentOf = (message) => (typeof message?.agent === "string" ? message.agent : defaultAgentName);
|
|
473
523
|
const queueName = `${queuePrefix}session`;
|
|
474
524
|
const eventIdPattern = /^[a-z]+_(\d{26})$/;
|
|
475
525
|
|
|
@@ -482,6 +532,22 @@ function ulid(now = Date.now()) {
|
|
|
482
532
|
return time + random;
|
|
483
533
|
}
|
|
484
534
|
|
|
535
|
+
// The entries that continue a session's chain of claims and records.
|
|
536
|
+
function chainedKind(k) {
|
|
537
|
+
return k === "lease" || k === "release" || k === "record";
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
// A run's in-band count right after its creation, as `@workflow/world`
|
|
541
|
+
// defines it (`IN_BAND_SEQ_AT_RUN_CREATION`): `run_created` counts as the
|
|
542
|
+
// first in-band write.
|
|
543
|
+
const inBandAtRunCreation = 1;
|
|
544
|
+
|
|
545
|
+
// The World's in-band writer fence refused a write: another writer moved the
|
|
546
|
+
// run's in-band count past the one the write expected.
|
|
547
|
+
function inBandSuperseded(error) {
|
|
548
|
+
return error?.name === "InBandSupersededError" || error?.code === "in-band-superseded" || error?.status === 412;
|
|
549
|
+
}
|
|
550
|
+
|
|
485
551
|
function slotOf(event) {
|
|
486
552
|
const match = eventIdPattern.exec(String(event?.eventId ?? ""));
|
|
487
553
|
if (!match) throw new Error(`the World returned event id ${JSON.stringify(event?.eventId ?? null)}, which holds no slot`);
|
|
@@ -512,9 +578,80 @@ const laterPageEvents = 100;
|
|
|
512
578
|
// deliveries in turn, as servers sharing a queue do.
|
|
513
579
|
const listenersOf = new WeakMap();
|
|
514
580
|
|
|
581
|
+
// Adds `member` to the group for `name` in `groups`, a Map of name to
|
|
582
|
+
// `{ members, turn }`, and returns the group; leaveGroup takes it out again.
|
|
583
|
+
function joinGroup(groups, name, member) {
|
|
584
|
+
let group = groups.get(name);
|
|
585
|
+
if (!group) groups.set(name, group = { members: [], turn: 0 });
|
|
586
|
+
group.members.push(member);
|
|
587
|
+
return group;
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
function leaveGroup(groups, name, group, member) {
|
|
591
|
+
const index = group.members.indexOf(member);
|
|
592
|
+
if (index >= 0) group.members.splice(index, 1);
|
|
593
|
+
if (group.members.length === 0 && groups.get(name) === group) groups.delete(name);
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
function noAgentNamed(name) {
|
|
597
|
+
return Object.assign(new Error(`no libfx agent named ${JSON.stringify(name)} runs in this process; the queue delivers this again`), { code: "FX_NO_AGENT", agent: name });
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
// The durable agents in this process, by name, from when each is created
|
|
601
|
+
// until it closes, each as `{ open, durability }`: a route delivers each
|
|
602
|
+
// message to the agent it names, and a function that only receives queue
|
|
603
|
+
// messages has its agents here before any of them is used. Shared by every
|
|
604
|
+
// copy of this module in the process.
|
|
605
|
+
const agentsByName = (globalThis[Symbol.for("libfx.durableAgents")] ??= new Map());
|
|
606
|
+
|
|
607
|
+
// The World backend of an agent named `name` in this process whose
|
|
608
|
+
// durability `matches`, the agent's copies in turn, or null.
|
|
609
|
+
async function agentNamed(name, matches) {
|
|
610
|
+
const group = agentsByName.get(name);
|
|
611
|
+
if (!group) return null;
|
|
612
|
+
const { members } = group;
|
|
613
|
+
const first = group.turn++;
|
|
614
|
+
for (let index = 0; index < members.length; index += 1) {
|
|
615
|
+
const member = members[(first + index) % members.length];
|
|
616
|
+
if (!matches(await member.durability())) continue;
|
|
617
|
+
const created = await member.open();
|
|
618
|
+
if (typeof created.deliverHere === "function") return created;
|
|
619
|
+
}
|
|
620
|
+
return null;
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
// Agents share sessions through the durability they were created with, so
|
|
624
|
+
// one on another durability can hold other sessions.
|
|
625
|
+
const agentSharing = (name, durability) => agentNamed(name, (other) => other === durability);
|
|
626
|
+
|
|
627
|
+
// A route takes its deployment's deliveries, which every route durability
|
|
628
|
+
// of the same kind in the process shares, such as each `vercel()` an app
|
|
629
|
+
// creates: the agent a message names on this durability, or on another of
|
|
630
|
+
// that kind.
|
|
631
|
+
const agentOnRoute = async (name, durability) => (await agentSharing(name, durability)) ??
|
|
632
|
+
agentNamed(name, (other) => other !== durability && other.name === durability.name && typeof other.world === "function" && other.create === undefined);
|
|
633
|
+
|
|
634
|
+
/**
|
|
635
|
+
* The request handler a deployment mounts for queue deliveries, for every
|
|
636
|
+
* durable agent in the process: each delivery goes to the agent its message
|
|
637
|
+
* names. One route and one queue trigger serve any number of agents.
|
|
638
|
+
*/
|
|
639
|
+
export function wakeHandler() {
|
|
640
|
+
return async (request) => {
|
|
641
|
+
for (const group of agentsByName.values()) {
|
|
642
|
+
for (const member of group.members) {
|
|
643
|
+
const created = await member.open();
|
|
644
|
+
if (created.viaRoute === true) return created.queueHandler(request);
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
return new Response("no libfx agent in this process takes queue deliveries from a route", { status: 503 });
|
|
648
|
+
};
|
|
649
|
+
}
|
|
650
|
+
|
|
515
651
|
// libfx starts and closes a World it `owned`, having created it; a World the
|
|
516
652
|
// app passed in stays the app's.
|
|
517
|
-
function createWorldBackend(world,
|
|
653
|
+
function createWorldBackend(world, durability) {
|
|
654
|
+
const { name, reserveMs, pollMs, leaseMs, alive, holderInfo, livenessKnown = false, queueLeases = false, runInRequest = queueLeases, waitUntil, queueDurable = false, maxDurationMs, owned = true } = durability;
|
|
518
655
|
const spec = world.specVersion === undefined ? {} : { specVersion: world.specVersion };
|
|
519
656
|
const created = new Set();
|
|
520
657
|
// Runs being created now: writes that go out together create a run once.
|
|
@@ -530,7 +667,8 @@ function createWorldBackend(world, { name, reserveMs, pollMs, leaseMs, alive, ho
|
|
|
530
667
|
pagination: { sortOrder: "desc", limit, ...(cursor ? { cursor } : {}) },
|
|
531
668
|
resolveData: "all",
|
|
532
669
|
});
|
|
533
|
-
|
|
670
|
+
// `meta` says which queue message this is and which delivery of it.
|
|
671
|
+
const deliverTo = async (handle, message, meta) => {
|
|
534
672
|
const input = message?.input;
|
|
535
673
|
if (!input || typeof input !== "object") return undefined;
|
|
536
674
|
if (!handle) {
|
|
@@ -539,14 +677,29 @@ function createWorldBackend(world, { name, reserveMs, pollMs, leaseMs, alive, ho
|
|
|
539
677
|
if (owned) return undefined;
|
|
540
678
|
throw new Error("no libfx agent is listening; the queue delivers this again");
|
|
541
679
|
}
|
|
542
|
-
const result = await handle(input);
|
|
680
|
+
const result = await handle(input, meta);
|
|
681
|
+
// The queue delivers this same message again, which takes back a lease
|
|
682
|
+
// its worker could not let go.
|
|
683
|
+
if (result?.redeliver === true) throw new Error("libfx asks the queue to deliver this message again");
|
|
543
684
|
return Number.isFinite(result?.timeoutSeconds) ? { timeoutSeconds: result.timeoutSeconds } : undefined;
|
|
544
685
|
};
|
|
545
|
-
// The route a deployment mounts delivers to this
|
|
546
|
-
|
|
686
|
+
// The route a deployment mounts delivers to the agent in this process the
|
|
687
|
+
// message names, whichever agent's route it is.
|
|
688
|
+
const queueHandler = world.createQueueHandler(queuePrefix, async (message, meta) => {
|
|
689
|
+
const named = agentOf(message?.input);
|
|
690
|
+
const agent = await agentOnRoute(named, durability);
|
|
691
|
+
if (!agent) throw noAgentNamed(named);
|
|
692
|
+
return agent.deliverHere(message, meta);
|
|
693
|
+
});
|
|
547
694
|
const backend = {
|
|
548
695
|
name,
|
|
549
696
|
livenessKnown,
|
|
697
|
+
queueLeases,
|
|
698
|
+
// A prompt runs in the request that took it, held by its backstop's
|
|
699
|
+
// message, which the queue keeps until after the request's deadline.
|
|
700
|
+
runInRequest: queueLeases && runInRequest,
|
|
701
|
+
// Keeps the request alive for a turn it runs after its response.
|
|
702
|
+
waitUntil: waitUntil ?? null,
|
|
550
703
|
queueDurable,
|
|
551
704
|
pollMs: pollMs ?? 1000,
|
|
552
705
|
reserveMs: reserveMs ?? defaultReserveMs,
|
|
@@ -557,9 +710,14 @@ function createWorldBackend(world, { name, reserveMs, pollMs, leaseMs, alive, ho
|
|
|
557
710
|
validSessionId: (id) => typeof id === "string" && /^wrun_[0-9A-Za-z]{10,64}$/.test(id),
|
|
558
711
|
deadline: () => null,
|
|
559
712
|
holderInfo: holderInfo ?? (() => ({})),
|
|
560
|
-
|
|
713
|
+
// Whether a lease's holder still runs, as `delivery` can tell: a worker
|
|
714
|
+
// of this process runs while it holds it, and the message its delivery
|
|
715
|
+
// held, delivered to this process, means it no longer does.
|
|
716
|
+
alive(lease, delivery = null) {
|
|
717
|
+
if (liveHolders.has(lease.holder)) return true;
|
|
561
718
|
const mine = holderInfo?.() ?? {};
|
|
562
|
-
if (mine.pid !== undefined && lease.pid === mine.pid && lease.host === mine.host) return
|
|
719
|
+
if (mine.pid !== undefined && lease.pid === mine.pid && lease.host === mine.host) return false;
|
|
720
|
+
if (redelivers(delivery, lease)) return false;
|
|
563
721
|
return alive ? alive(lease) : null;
|
|
564
722
|
},
|
|
565
723
|
holding(holder, held) {
|
|
@@ -584,6 +742,28 @@ function createWorldBackend(world, { name, reserveMs, pollMs, leaseMs, alive, ho
|
|
|
584
742
|
async session(runId) {
|
|
585
743
|
// The newest slot this worker has seen.
|
|
586
744
|
let known = 0;
|
|
745
|
+
// Where the World has the in-band writer fence, chained entries are
|
|
746
|
+
// written in-band, each expecting the World's count of in-band writes
|
|
747
|
+
// at the entry it continues, so the World refuses one that continues a
|
|
748
|
+
// superseded head and nothing of it lands. `inBandAt` maps a chained
|
|
749
|
+
// entry's cursor (`null` for an empty chain) to the count right after
|
|
750
|
+
// it: from the snapshot a read returns, or from the write itself. The
|
|
751
|
+
// check after a write stays, since a World may count per process.
|
|
752
|
+
const inBandAt = new Map();
|
|
753
|
+
let inBand = world.capabilities?.inBandFence === true;
|
|
754
|
+
const noteInBand = (cursor, count) => {
|
|
755
|
+
inBandAt.delete(cursor);
|
|
756
|
+
inBandAt.set(cursor, count);
|
|
757
|
+
if (inBandAt.size > 16) inBandAt.delete(inBandAt.keys().next().value);
|
|
758
|
+
};
|
|
759
|
+
// A snapshot is read before its listing, so it gives the count at the
|
|
760
|
+
// chain's head only when it covers the head's slot. It never replaces
|
|
761
|
+
// a count this worker's own write reported.
|
|
762
|
+
const noteSnapshot = (snapshot, cursor) => {
|
|
763
|
+
if (!inBand || cursor === undefined || inBandAt.has(cursor) || !Number.isSafeInteger(snapshot?.seqInBand)) return;
|
|
764
|
+
if (cursor !== "null" && !(Number(cursor) <= snapshot.seq)) return;
|
|
765
|
+
noteInBand(cursor, snapshot.seqInBand);
|
|
766
|
+
};
|
|
587
767
|
const ensureRun = () => {
|
|
588
768
|
if (created.has(runId)) return Promise.resolve();
|
|
589
769
|
let pending = creating.get(runId);
|
|
@@ -616,6 +796,16 @@ function createWorldBackend(world, { name, reserveMs, pollMs, leaseMs, alive, ho
|
|
|
616
796
|
const newestFirst = async (stopBelow, stopAtCheckpoint) => {
|
|
617
797
|
const events = [];
|
|
618
798
|
let through = null;
|
|
799
|
+
let snapshot;
|
|
800
|
+
// The newest chained entry read: the chain's head, unless the read
|
|
801
|
+
// stopped before reaching any.
|
|
802
|
+
let newestChained;
|
|
803
|
+
// Ends the read. `head` is where the chain's head is when the read
|
|
804
|
+
// found no chained entry, or undefined when the read does not show.
|
|
805
|
+
const done = (head) => {
|
|
806
|
+
noteSnapshot(snapshot, newestChained ?? head);
|
|
807
|
+
return { events, through };
|
|
808
|
+
};
|
|
619
809
|
// A session's newest checkpoint is usually among its last few
|
|
620
810
|
// entries, so a read starts with a small page and fetches more only
|
|
621
811
|
// while it has not reached where it stops; every page carries its
|
|
@@ -625,23 +815,29 @@ function createWorldBackend(world, { name, reserveMs, pollMs, leaseMs, alive, ho
|
|
|
625
815
|
try {
|
|
626
816
|
page = await listEvents(runId, cursor, limit);
|
|
627
817
|
} catch (error) {
|
|
628
|
-
if (missingRun(error))
|
|
629
|
-
|
|
818
|
+
if (!missingRun(error)) throw error;
|
|
819
|
+
// The run's creation starts its count.
|
|
820
|
+
if (stopBelow === 0) noteSnapshot({ seq: 0, seqInBand: inBandAtRunCreation }, "null");
|
|
821
|
+
return { events, through };
|
|
630
822
|
}
|
|
823
|
+
snapshot ??= page.snapshot;
|
|
631
824
|
for (const event of page.data) {
|
|
632
825
|
const slot = slotOf(event);
|
|
633
826
|
if (slot > known) known = slot;
|
|
634
|
-
if (slot <= stopBelow) return
|
|
635
|
-
|
|
827
|
+
if (slot <= stopBelow) return done(undefined);
|
|
828
|
+
// The checkpoint stands for the entries through its `through`,
|
|
829
|
+
// the last of them the chain's head.
|
|
830
|
+
if (through !== null && slot <= through) return done(through === 0 ? "null" : String(through));
|
|
636
831
|
const entry = logEntryOf(event);
|
|
637
832
|
if (!entry) continue;
|
|
638
833
|
created.add(runId);
|
|
834
|
+
if (newestChained === undefined && chainedKind(entry.k)) newestChained = String(slot);
|
|
639
835
|
events.push({ cursor: String(slot), entry });
|
|
640
836
|
if (stopAtCheckpoint && through === null && entry.k === "checkpoint") {
|
|
641
837
|
through = entry.through === null || entry.through === undefined ? 0 : Number(entry.through);
|
|
642
838
|
}
|
|
643
839
|
}
|
|
644
|
-
if (!page.hasMore) return
|
|
840
|
+
if (!page.hasMore) return done(stopBelow > 0 ? undefined : through ? String(through) : "null");
|
|
645
841
|
cursor = page.cursor;
|
|
646
842
|
}
|
|
647
843
|
};
|
|
@@ -656,17 +852,55 @@ function createWorldBackend(world, { name, reserveMs, pollMs, leaseMs, alive, ho
|
|
|
656
852
|
},
|
|
657
853
|
async append(entry) {
|
|
658
854
|
await ensureRun();
|
|
659
|
-
const chained = entry.k
|
|
855
|
+
const chained = chainedKind(entry.k);
|
|
660
856
|
// A chained entry asks the World to report every event since the
|
|
661
857
|
// one it continues, so a competing continuation is seen.
|
|
662
858
|
const after = chained ? (entry.a === null || entry.a === undefined ? 0 : Number(entry.a)) : known;
|
|
663
|
-
const
|
|
859
|
+
const continues = entry.a === null || entry.a === undefined ? "null" : String(entry.a);
|
|
860
|
+
let expected = chained && inBand ? inBandAt.get(continues) : undefined;
|
|
861
|
+
const event = {
|
|
664
862
|
eventType: "step_created",
|
|
665
863
|
correlationId: `step_${ulid()}`,
|
|
666
864
|
eventData: { stepName: logStep, input: encoder.encode(JSON.stringify(entry)) },
|
|
667
|
-
}
|
|
865
|
+
};
|
|
866
|
+
let result;
|
|
867
|
+
for (let recounted = false; ;) {
|
|
868
|
+
try {
|
|
869
|
+
result = await world.events.create(runId, event, expected === undefined ? { eventCount: after } : { eventCount: after, inBand: true, expectedSeqInBand: expected });
|
|
870
|
+
break;
|
|
871
|
+
} catch (error) {
|
|
872
|
+
if (expected === undefined) throw error;
|
|
873
|
+
if (inBandSuperseded(error)) {
|
|
874
|
+
// The next read maps the head afresh.
|
|
875
|
+
inBandAt.delete(continues);
|
|
876
|
+
// A World counts a write after it publishes it, so a read
|
|
877
|
+
// between the two gave this head the count before its own
|
|
878
|
+
// write. When nothing else continued the head, the write goes
|
|
879
|
+
// again once at the count the World reports.
|
|
880
|
+
if (!recounted && Number.isSafeInteger(error?.seqInBand)) {
|
|
881
|
+
const { events: since } = await newestFirst(after, false);
|
|
882
|
+
const rivals = since.filter(({ entry: other }) => chainedKind(other.k) && (other.a ?? null) === (entry.a ?? null));
|
|
883
|
+
// Our own earlier write of this entry landed first.
|
|
884
|
+
const ours = rivals.find(({ entry: other }) => other.key !== undefined && other.key === entry.key && other.k === entry.k);
|
|
885
|
+
if (ours) return ours.cursor;
|
|
886
|
+
if (rivals.length === 0) {
|
|
887
|
+
recounted = true;
|
|
888
|
+
expected = error.seqInBand;
|
|
889
|
+
continue;
|
|
890
|
+
}
|
|
891
|
+
}
|
|
892
|
+
throw new FencedError(`another worker continued session ${runId}`);
|
|
893
|
+
}
|
|
894
|
+
// A World may refuse in-band writes on a run it stamped before
|
|
895
|
+
// it had the fence; the write goes out-of-band, as before.
|
|
896
|
+
if (!(error?.status >= 400 && error?.status < 500) || error?.status === 409) throw error;
|
|
897
|
+
inBand = false;
|
|
898
|
+
expected = undefined;
|
|
899
|
+
}
|
|
900
|
+
}
|
|
668
901
|
const slot = slotOf(result.event);
|
|
669
902
|
if (slot > known) known = slot;
|
|
903
|
+
if (expected !== undefined && result.allocated !== undefined) noteInBand(String(slot), expected + result.allocated);
|
|
670
904
|
changed(runId);
|
|
671
905
|
if (!chained || slot === after + 1) return String(slot);
|
|
672
906
|
const reported = Array.isArray(result.events) ? result.events : (await newestFirst(after, false)).events.map((item) => ({ eventId: `evnt_${String(item.cursor).padStart(26, "0")}`, ...item }));
|
|
@@ -674,7 +908,7 @@ function createWorldBackend(world, { name, reserveMs, pollMs, leaseMs, alive, ho
|
|
|
674
908
|
const other = event.entry ?? logEntryOf(event);
|
|
675
909
|
const otherSlot = event.entry ? Number(event.cursor) : slotOf(event);
|
|
676
910
|
if (!other || otherSlot <= after || otherSlot >= slot) continue;
|
|
677
|
-
const otherChained = other.k
|
|
911
|
+
const otherChained = chainedKind(other.k);
|
|
678
912
|
if (!otherChained || (other.a ?? null) !== (entry.a ?? null)) continue;
|
|
679
913
|
// Our own earlier write of this entry landed first.
|
|
680
914
|
if (other.key !== undefined && other.key === entry.key && other.k === entry.k) return String(otherSlot);
|
|
@@ -744,33 +978,49 @@ function createWorldBackend(world, { name, reserveMs, pollMs, leaseMs, alive, ho
|
|
|
744
978
|
};
|
|
745
979
|
},
|
|
746
980
|
queue: {
|
|
981
|
+
// Returns the queue's id for the message, when the World gives one.
|
|
747
982
|
async send(message, { delaySeconds = 0, idempotencyKey } = {}) {
|
|
748
|
-
await world.queue(queueName, { runId: message.sessionId, requestId: message.messageId, input: message }, {
|
|
983
|
+
const sent = await world.queue(queueName, { runId: message.sessionId, requestId: message.messageId, input: message }, {
|
|
749
984
|
...(idempotencyKey ? { idempotencyKey } : {}),
|
|
750
985
|
...(delaySeconds > 0 ? { delaySeconds } : {}),
|
|
751
986
|
});
|
|
987
|
+
return typeof sent?.messageId === "string" && sent.messageId.length > 0 ? { messageId: sent.messageId } : {};
|
|
752
988
|
},
|
|
753
989
|
},
|
|
754
990
|
// The route a deployment mounts for queue deliveries, until the World
|
|
755
991
|
// can deliver to an in-process handler itself.
|
|
756
992
|
queueHandler,
|
|
757
|
-
|
|
993
|
+
durability,
|
|
994
|
+
// Whether the queue delivers to queueHandler, mounted by the deployment.
|
|
995
|
+
viaRoute: route,
|
|
996
|
+
// Delivers a routed message to this backend's agent.
|
|
997
|
+
deliverHere: (message, meta) => deliverTo(handler, message, meta),
|
|
998
|
+
listen(fn, name = defaultAgentName) {
|
|
758
999
|
handler = fn;
|
|
759
1000
|
if (route) return () => { if (handler === fn) handler = null; };
|
|
1001
|
+
// The agents on this World, by name: an agent's copies take the
|
|
1002
|
+
// deliveries for its name in turn. A name none of them has goes to an
|
|
1003
|
+
// agent of that name on the same durability elsewhere in the process,
|
|
1004
|
+
// as each agent has a World of its own, and one the process has none
|
|
1005
|
+
// of is left for the queue to deliver again.
|
|
760
1006
|
let listeners = listenersOf.get(world);
|
|
761
1007
|
if (!listeners) {
|
|
762
|
-
listeners =
|
|
1008
|
+
listeners = new Map();
|
|
763
1009
|
listenersOf.set(world, listeners);
|
|
764
|
-
world.registerHandler(queuePrefix, world.createQueueHandler(queuePrefix, (message) => {
|
|
765
|
-
const
|
|
766
|
-
|
|
1010
|
+
world.registerHandler(queuePrefix, world.createQueueHandler(queuePrefix, async (message, meta) => {
|
|
1011
|
+
const named = agentOf(message?.input);
|
|
1012
|
+
const group = listeners.get(named);
|
|
1013
|
+
if (group) return deliverTo(group.members[group.turn++ % group.members.length], message, meta);
|
|
1014
|
+
const agent = await agentSharing(named, durability);
|
|
1015
|
+
if (agent) return agent.deliverHere(message, meta);
|
|
1016
|
+
if (listeners.size === 0) return deliverTo(null, message, meta);
|
|
1017
|
+
throw noAgentNamed(named);
|
|
767
1018
|
}));
|
|
768
1019
|
}
|
|
769
|
-
listeners
|
|
1020
|
+
const group = joinGroup(listeners, name, fn);
|
|
770
1021
|
return () => {
|
|
771
1022
|
if (handler === fn) handler = null;
|
|
772
|
-
|
|
773
|
-
if (index >= 0) listeners.handlers.splice(index, 1);
|
|
1023
|
+
leaveGroup(listeners, name, group, fn);
|
|
774
1024
|
};
|
|
775
1025
|
},
|
|
776
1026
|
// Closing is best effort: a runtime whose HTTP agent cannot close, as
|
|
@@ -830,8 +1080,9 @@ const isFenced = (error) => error?.code === "FX_FENCED";
|
|
|
830
1080
|
// Nothing the log already holds. Pure.
|
|
831
1081
|
function messageEntries(state, message) {
|
|
832
1082
|
const entries = [];
|
|
1083
|
+
const named = typeof message.agent === "string" ? { agent: message.agent } : {};
|
|
833
1084
|
if (message.seed && state.checkpoint === null && state.records.length === 0) {
|
|
834
|
-
entries.push({ k: "checkpoint", through: null, data: message.seed });
|
|
1085
|
+
entries.push({ k: "checkpoint", through: null, data: message.seed, ...named });
|
|
835
1086
|
}
|
|
836
1087
|
const key = `${message.type}:${message.messageId}`;
|
|
837
1088
|
const known = state.inputKeys.has(key) || state.consumed.has(message.messageId);
|
|
@@ -841,6 +1092,7 @@ function messageEntries(state, message) {
|
|
|
841
1092
|
key,
|
|
842
1093
|
type: message.type,
|
|
843
1094
|
messageId: message.messageId,
|
|
1095
|
+
...named,
|
|
844
1096
|
...(message.input === undefined ? {} : { input: message.input }),
|
|
845
1097
|
...(typeof message.target === "string" ? { target: message.target } : {}),
|
|
846
1098
|
...((message.type === "prompt" || message.type === "steer") && message.context !== undefined ? { context: message.context } : {}),
|
|
@@ -907,6 +1159,14 @@ const durableInternals = Symbol.for("libfx.durableInternals");
|
|
|
907
1159
|
// process is delivered again, as a backstop for the running worker.
|
|
908
1160
|
const runningBackstopSeconds = 2;
|
|
909
1161
|
|
|
1162
|
+
// When a delivery that starts now must stop: its function's deadline, or
|
|
1163
|
+
// `maxDurationMs` after now; null when nothing stops it.
|
|
1164
|
+
const deliveryDeadline = (backend) => backend.deadline() ?? (backend.maxDurationMs === null ? null : Date.now() + backend.maxDurationMs);
|
|
1165
|
+
|
|
1166
|
+
// How long past the request's own deadline the queue keeps a prompt that
|
|
1167
|
+
// request runs itself, before it delivers it as a backstop.
|
|
1168
|
+
const inlineBackstopMarginSeconds = 5;
|
|
1169
|
+
|
|
910
1170
|
class SessionWorker {
|
|
911
1171
|
constructor(agent, backend, sessionId) {
|
|
912
1172
|
this.agent = agent;
|
|
@@ -919,33 +1179,46 @@ class SessionWorker {
|
|
|
919
1179
|
this.holder = null;
|
|
920
1180
|
}
|
|
921
1181
|
|
|
922
|
-
|
|
923
|
-
|
|
1182
|
+
// The log folded as `delivery` judges whether a lease's holder lives.
|
|
1183
|
+
fold(entries, delivery = null) {
|
|
1184
|
+
return foldSessionLog(entries, { now: Date.now(), alive: (lease) => this.backend.alive(lease, delivery) });
|
|
924
1185
|
}
|
|
925
1186
|
|
|
926
|
-
// One queue delivery
|
|
1187
|
+
// One queue delivery, `delivery` its message's id and attempt when the
|
|
1188
|
+
// queue reports them. Its input reaches the log at once, even while this
|
|
927
1189
|
// process runs the session, so a steer or cancel reaches the running turn.
|
|
928
1190
|
// Then it runs the session when no live worker holds it. Returns
|
|
929
1191
|
// `{ timeoutSeconds }` to have the same message delivered again later.
|
|
930
|
-
async consume(message) {
|
|
1192
|
+
async consume(message, delivery = null) {
|
|
931
1193
|
const log = await this.backend.session(this.sessionId);
|
|
932
1194
|
await this.backend.refreshDeadline?.();
|
|
933
1195
|
// When this delivery's function stops, if it does.
|
|
934
1196
|
const deadline = this.backend.deadline() ?? (this.backend.maxDurationMs === null ? null : Date.now() + this.backend.maxDurationMs);
|
|
935
1197
|
let entries = await log.read();
|
|
936
|
-
let state = this.fold(entries);
|
|
1198
|
+
let state = this.fold(entries, delivery);
|
|
1199
|
+
// A session runs only on the agent it belongs to, which a new one is the
|
|
1200
|
+
// agent that sent its first message. A message that reached another
|
|
1201
|
+
// agent, as one sent to another agent's session does, goes on to it.
|
|
1202
|
+
const owner = state.agent ?? agentOf(message);
|
|
1203
|
+
if (owner !== this.agent.name) {
|
|
1204
|
+
await this.backend.queue.send({ ...message, agent: owner }, {
|
|
1205
|
+
idempotencyKey: `${message.sessionId}:${message.type}:${message.messageId}:to:${owner}`,
|
|
1206
|
+
});
|
|
1207
|
+
this.agent.emit("session.forwarded", { sessionId: this.sessionId, agent: owner });
|
|
1208
|
+
return undefined;
|
|
1209
|
+
}
|
|
937
1210
|
for (let claims = 0; claims < 8; claims += 1) {
|
|
938
1211
|
// A session with no open turn starts its next prompt at once: the
|
|
939
1212
|
// first model request goes out while the input and the lease are
|
|
940
1213
|
// written, and nothing else is written until both land.
|
|
941
|
-
const early = this.running || state.lease ? null : earlyStart(entries, state, message, (items) => this.fold(items));
|
|
1214
|
+
const early = this.running || state.lease ? null : earlyStart(entries, state, message, (items) => this.fold(items, delivery));
|
|
942
1215
|
let outcome;
|
|
943
1216
|
if (early) {
|
|
944
|
-
outcome = await this.claim(log, state, deadline, early);
|
|
1217
|
+
outcome = await this.claim(log, state, deadline, early, delivery);
|
|
945
1218
|
} else {
|
|
946
1219
|
if (await recordMessage(log, state, message) > 0) {
|
|
947
1220
|
entries = await log.read();
|
|
948
|
-
state = this.fold(entries);
|
|
1221
|
+
state = this.fold(entries, delivery);
|
|
949
1222
|
}
|
|
950
1223
|
if (this.running) return { timeoutSeconds: runningBackstopSeconds };
|
|
951
1224
|
if (!state.hasWork) {
|
|
@@ -958,12 +1231,15 @@ class SessionWorker {
|
|
|
958
1231
|
const left = state.lease.expiresAt ? Math.ceil((state.lease.expiresAt - Date.now()) / 1000) + 1 : 5;
|
|
959
1232
|
return { timeoutSeconds: Math.max(1, Math.min(maxBackstopSeconds, left)) };
|
|
960
1233
|
}
|
|
961
|
-
outcome = await this.claim(log, state, deadline, null);
|
|
1234
|
+
outcome = await this.claim(log, state, deadline, null, delivery);
|
|
962
1235
|
}
|
|
963
1236
|
if (outcome === "yielded") return { timeoutSeconds: 0 };
|
|
964
1237
|
// A write that failed for any other reason: the same message runs the
|
|
965
|
-
// session again shortly.
|
|
966
|
-
|
|
1238
|
+
// session again shortly. A lease held to the deadline, which a release
|
|
1239
|
+
// that failed may leave standing, is let go only by a later delivery of
|
|
1240
|
+
// this message, and a queue may send `{ timeoutSeconds }` on as a new
|
|
1241
|
+
// one, so the queue is asked for this one again.
|
|
1242
|
+
if (outcome === "failed") return queueHeld(this.backend, deadline) ? { redeliver: true } : { timeoutSeconds: 1 };
|
|
967
1243
|
if (outcome === "fenced") {
|
|
968
1244
|
// Another worker took the session over; this one stops.
|
|
969
1245
|
this.agent.emit("session.fenced", { sessionId: this.sessionId, epoch: state.lastEpoch + 1 });
|
|
@@ -972,7 +1248,7 @@ class SessionWorker {
|
|
|
972
1248
|
// The turns ran, or another worker's write came before this claim:
|
|
973
1249
|
// read the log again.
|
|
974
1250
|
entries = await log.read();
|
|
975
|
-
state = this.fold(entries);
|
|
1251
|
+
state = this.fold(entries, delivery);
|
|
976
1252
|
}
|
|
977
1253
|
return { timeoutSeconds: 1 };
|
|
978
1254
|
}
|
|
@@ -980,9 +1256,10 @@ class SessionWorker {
|
|
|
980
1256
|
// Claims the session where `state` read it, with a lease that continues
|
|
981
1257
|
// the chain there, and runs it. With `early`, what `earlyStart` returned,
|
|
982
1258
|
// the message's input is written with the lease and the next turn starts
|
|
983
|
-
// before either lands.
|
|
984
|
-
//
|
|
985
|
-
|
|
1259
|
+
// before either lands. The lease names `delivery`, so a later delivery of
|
|
1260
|
+
// the same message can tell it lost its holder. Returns what run()
|
|
1261
|
+
// returns, or "refused" when another worker's write came before the claim.
|
|
1262
|
+
async claim(log, state, deadline, early, delivery = null) {
|
|
986
1263
|
// Claimed before the first await, so a delivery that arrives meanwhile
|
|
987
1264
|
// sees the session running here.
|
|
988
1265
|
this.running = true;
|
|
@@ -994,6 +1271,7 @@ class SessionWorker {
|
|
|
994
1271
|
epoch: state.lastEpoch + 1,
|
|
995
1272
|
expiresAt: leaseEnd(this.backend, deadline),
|
|
996
1273
|
...this.backend.holderInfo(),
|
|
1274
|
+
...(delivery ? { delivery } : {}),
|
|
997
1275
|
};
|
|
998
1276
|
this.backend.holding(this.holder, true);
|
|
999
1277
|
try {
|
|
@@ -1135,15 +1413,16 @@ class SessionWorker {
|
|
|
1135
1413
|
pending: now.unconsumed.filter((input) => input.cursor <= Number(through)),
|
|
1136
1414
|
recent: now.recent,
|
|
1137
1415
|
lastTurnId: now.lastTurnId,
|
|
1416
|
+
...(now.agent === null ? {} : { agent: now.agent }),
|
|
1138
1417
|
...(now.openTurn ? { openTurn: now.openTurn, yielded: now.yielded } : {}),
|
|
1139
|
-
lease: { holder: this.holder, epoch: lease.epoch, expiresAt: leaseExpiresAt, pid: lease.pid, host: lease.host },
|
|
1418
|
+
lease: { holder: this.holder, epoch: lease.epoch, expiresAt: leaseExpiresAt, pid: lease.pid, host: lease.host, ...(lease.delivery ? { delivery: lease.delivery } : {}) },
|
|
1140
1419
|
});
|
|
1141
1420
|
},
|
|
1142
1421
|
};
|
|
1143
1422
|
// The turn running now, which a deadline or a lost lease cancels.
|
|
1144
1423
|
let turnNow = null;
|
|
1145
1424
|
let renew = null;
|
|
1146
|
-
if (!this.backend
|
|
1425
|
+
if (!heldToDeadline(this.backend, deadline)) {
|
|
1147
1426
|
// A lease nothing else shows alive is renewed while its worker runs,
|
|
1148
1427
|
// and never after its release, which would take the session back. A
|
|
1149
1428
|
// renewal fenced out by another worker's write stops the turn at once,
|
|
@@ -1151,7 +1430,7 @@ class SessionWorker {
|
|
|
1151
1430
|
renew = setInterval(() => {
|
|
1152
1431
|
if (released || failed || this.stopping) return;
|
|
1153
1432
|
leaseExpiresAt = leaseEnd(this.backend, deadline);
|
|
1154
|
-
void chained({ k: "lease", holder: this.holder, epoch: lease.epoch, expiresAt: leaseExpiresAt, ...this.backend.holderInfo() })
|
|
1433
|
+
void chained({ k: "lease", holder: this.holder, epoch: lease.epoch, expiresAt: leaseExpiresAt, ...this.backend.holderInfo(), ...(lease.delivery ? { delivery: lease.delivery } : {}) })
|
|
1155
1434
|
.catch((error) => { if (isFenced(error)) turnNow?.cancel({ reason: "handoff" }); });
|
|
1156
1435
|
}, Math.max(1, Math.floor(this.backend.leaseMs / 3)));
|
|
1157
1436
|
}
|
|
@@ -1349,10 +1628,19 @@ class SessionWorker {
|
|
|
1349
1628
|
}
|
|
1350
1629
|
if (state.lastLease?.holder !== this.holder && !failed) return "fenced";
|
|
1351
1630
|
}
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1631
|
+
if (failed) {
|
|
1632
|
+
await harness.close();
|
|
1633
|
+
harness = null;
|
|
1634
|
+
return broken();
|
|
1635
|
+
}
|
|
1636
|
+
// The last turn's records and end have landed, so the worker lets go
|
|
1637
|
+
// before its engine shuts down: a prompt that lands meanwhile starts at
|
|
1638
|
+
// once, here or in another process, instead of waiting for the
|
|
1639
|
+
// shutdown. The engine stores nothing more; the agent waits for it on
|
|
1640
|
+
// close.
|
|
1355
1641
|
await release();
|
|
1642
|
+
agent.track(harness.close().catch(() => {}));
|
|
1643
|
+
harness = null;
|
|
1356
1644
|
return failed ? broken() : "done";
|
|
1357
1645
|
} finally {
|
|
1358
1646
|
if (hardStop) clearTimeout(hardStop);
|
|
@@ -1456,8 +1744,7 @@ const internalEvent = new Set(["turn_start", "turn_resume", "turn_yield", "turn_
|
|
|
1456
1744
|
// replaced worker's lines carry its older lease epoch, so once its successor
|
|
1457
1745
|
// has written, they stay hidden. The same stored stream always shows the
|
|
1458
1746
|
// same lines, from any server and after any refresh.
|
|
1459
|
-
function uiRanker() {
|
|
1460
|
-
let high = 0;
|
|
1747
|
+
function uiRanker(high = 0) {
|
|
1461
1748
|
return (event) => {
|
|
1462
1749
|
if (!Number.isSafeInteger(event?.epoch)) return true;
|
|
1463
1750
|
if (event.epoch < high) return false;
|
|
@@ -1543,8 +1830,10 @@ async function turnEndIn(log, messageId, count) {
|
|
|
1543
1830
|
const retryEndWaitMs = 30_000;
|
|
1544
1831
|
|
|
1545
1832
|
// One turn as its viewer sees it: the session stream from `from` on, its
|
|
1546
|
-
// events only, until its end. `start` queues the turn and returns
|
|
1547
|
-
// stream; a resume learns its turn from the first one the stream shows.
|
|
1833
|
+
// events only, until its end. `start` queues the turn at once and returns
|
|
1834
|
+
// the stream; a resume learns its turn from the first one the stream shows.
|
|
1835
|
+
// The stream is read only once something reads the view, its events,
|
|
1836
|
+
// `readable` or `result`, so a prompt nobody follows holds no read open.
|
|
1548
1837
|
function turnView({ messageId, resumeRequest = null, start }) {
|
|
1549
1838
|
// A retried turn reports the outcome of the turn its id already ran.
|
|
1550
1839
|
let repeated = false;
|
|
@@ -1566,9 +1855,17 @@ function turnView({ messageId, resumeRequest = null, start }) {
|
|
|
1566
1855
|
} else settle.resolve(repeated ? { ...value, repeated: true } : value);
|
|
1567
1856
|
wake();
|
|
1568
1857
|
};
|
|
1569
|
-
|
|
1858
|
+
const started = start();
|
|
1859
|
+
void started.catch(() => {});
|
|
1860
|
+
let following = false;
|
|
1861
|
+
const follow = () => {
|
|
1862
|
+
if (following) return;
|
|
1863
|
+
following = true;
|
|
1864
|
+
void read();
|
|
1865
|
+
};
|
|
1866
|
+
const read = async () => {
|
|
1570
1867
|
try {
|
|
1571
|
-
const { log, from, settled, attaching, endWithinMs, unsure } = await
|
|
1868
|
+
const { log, from, settled, attaching, endWithinMs, unsure } = await started;
|
|
1572
1869
|
// A turn whose end this view finds after its own start may be the
|
|
1573
1870
|
// message's first run, so only an end found before it is a repeat.
|
|
1574
1871
|
repeated = settled?.repeated === true || (endWithinMs !== undefined && !unsure);
|
|
@@ -1629,7 +1926,7 @@ function turnView({ messageId, resumeRequest = null, start }) {
|
|
|
1629
1926
|
} catch (error) {
|
|
1630
1927
|
finish(null, error);
|
|
1631
1928
|
}
|
|
1632
|
-
}
|
|
1929
|
+
};
|
|
1633
1930
|
async function* iterate(raw) {
|
|
1634
1931
|
for (let index = 0; ; index += 1) {
|
|
1635
1932
|
while (index >= events.length) {
|
|
@@ -1649,11 +1946,18 @@ function turnView({ messageId, resumeRequest = null, start }) {
|
|
|
1649
1946
|
}
|
|
1650
1947
|
return {
|
|
1651
1948
|
get messageId() { return id; },
|
|
1652
|
-
result
|
|
1949
|
+
get result() {
|
|
1950
|
+
follow();
|
|
1951
|
+
return result;
|
|
1952
|
+
},
|
|
1653
1953
|
// Dropping a turn cancels nothing; `session.cancel()` does.
|
|
1654
|
-
[Symbol.asyncIterator]() {
|
|
1954
|
+
[Symbol.asyncIterator]() {
|
|
1955
|
+
follow();
|
|
1956
|
+
return iterate(false);
|
|
1957
|
+
},
|
|
1655
1958
|
// Each line is one event with the cursor `stream()` resumes after it.
|
|
1656
1959
|
get readable() {
|
|
1960
|
+
follow();
|
|
1657
1961
|
const iterator = iterate(true);
|
|
1658
1962
|
return new ReadableStream({
|
|
1659
1963
|
async pull(controller) {
|
|
@@ -1728,17 +2032,27 @@ export function createDurableAgentFactory({ harness, defaultDurability, name = "
|
|
|
1728
2032
|
if (!options || typeof options !== "object" || Array.isArray(options)) {
|
|
1729
2033
|
throw new TypeError(`${name}() options must be an object`);
|
|
1730
2034
|
}
|
|
1731
|
-
const { durability, ...harnessOptions } = options;
|
|
2035
|
+
const { durability, name: agentName = defaultAgentName, ...harnessOptions } = options;
|
|
1732
2036
|
if (durability !== undefined && !isDurability(durability)) {
|
|
1733
2037
|
throw new TypeError("durability must come from memory(), local(), vercel() or world()");
|
|
1734
2038
|
}
|
|
2039
|
+
if (typeof agentName !== "string" || !agentNameRule.test(agentName)) {
|
|
2040
|
+
throw new TypeError(`${name}() name must be 1 to 64 letters, digits, ".", "_" or "-", starting with a letter or digit`);
|
|
2041
|
+
}
|
|
1735
2042
|
const plugged = harness(harnessOptions);
|
|
1736
2043
|
const seed = plugged.seed === undefined ? undefined : toBase64(plugged.seed);
|
|
1737
2044
|
const emit = (type, detail = {}) => {
|
|
1738
2045
|
try { options.onEvent?.({ type, timestamp: performance.now(), ...detail }); } catch {}
|
|
1739
2046
|
};
|
|
1740
2047
|
const agent = {
|
|
2048
|
+
name: agentName,
|
|
1741
2049
|
emit,
|
|
2050
|
+
// Work that outlives the delivery that started it, which close() waits
|
|
2051
|
+
// for: an engine that shuts down after its worker let go.
|
|
2052
|
+
track(work) {
|
|
2053
|
+
active.add(work);
|
|
2054
|
+
void work.finally(() => active.delete(work));
|
|
2055
|
+
},
|
|
1742
2056
|
openHarness: (sessionId, store, run, ready) => plugged.open({
|
|
1743
2057
|
sessionId,
|
|
1744
2058
|
store,
|
|
@@ -1749,6 +2063,7 @@ export function createDurableAgentFactory({ harness, defaultDurability, name = "
|
|
|
1749
2063
|
}),
|
|
1750
2064
|
};
|
|
1751
2065
|
let backendPromise = null;
|
|
2066
|
+
let durabilityPromise = null;
|
|
1752
2067
|
let chosenDurability = null;
|
|
1753
2068
|
let unlisten = null;
|
|
1754
2069
|
let closed = false;
|
|
@@ -1757,34 +2072,58 @@ export function createDurableAgentFactory({ harness, defaultDurability, name = "
|
|
|
1757
2072
|
const backend = () => {
|
|
1758
2073
|
if (closed) return Promise.reject(new Error(`${label} is closed`));
|
|
1759
2074
|
backendPromise ??= (async () => {
|
|
1760
|
-
const chosen =
|
|
2075
|
+
const chosen = await durabilityOf();
|
|
1761
2076
|
chosenDurability = chosen;
|
|
1762
2077
|
const created = await createBackend(chosen);
|
|
1763
2078
|
await created.start?.();
|
|
1764
|
-
unlisten = created.listen((message) =>
|
|
1765
|
-
let worker = workers.get(message.sessionId);
|
|
1766
|
-
if (!worker) {
|
|
1767
|
-
worker = new SessionWorker(agent, created, message.sessionId);
|
|
1768
|
-
workers.set(message.sessionId, worker);
|
|
1769
|
-
}
|
|
1770
|
-
worker.deliveries += 1;
|
|
1771
|
-
const work = worker.consume(message);
|
|
1772
|
-
active.add(work);
|
|
1773
|
-
void work.catch((error) => {
|
|
1774
|
-
// Its queue delivers the message again; the host hears why.
|
|
1775
|
-
emit("session.error", { sessionId: message.sessionId, error: errorSummary(error) });
|
|
1776
|
-
}).finally(() => {
|
|
1777
|
-
active.delete(work);
|
|
1778
|
-
worker.deliveries -= 1;
|
|
1779
|
-
// A worker lives while a delivery runs it.
|
|
1780
|
-
if (worker.deliveries === 0 && !worker.running && workers.get(message.sessionId) === worker) workers.delete(message.sessionId);
|
|
1781
|
-
});
|
|
1782
|
-
return work;
|
|
1783
|
-
});
|
|
2079
|
+
unlisten = created.listen((message, meta) => receive(created, message, deliveryOf(meta)), agentName);
|
|
1784
2080
|
return created;
|
|
1785
2081
|
})();
|
|
1786
2082
|
return backendPromise;
|
|
1787
2083
|
};
|
|
2084
|
+
// The durability the agent's sessions live in, chosen once.
|
|
2085
|
+
const durabilityOf = () => (durabilityPromise ??= durability ? Promise.resolve(durability) : defaultDurability());
|
|
2086
|
+
// In the process's agents from now, so a route that receives a message
|
|
2087
|
+
// for this agent finds it before anything uses it.
|
|
2088
|
+
const member = { open: backend, durability: durabilityOf };
|
|
2089
|
+
const registered = joinGroup(agentsByName, agentName, member);
|
|
2090
|
+
// One delivery of `message` to the session's worker here: from the
|
|
2091
|
+
// queue, with `delivery` naming the queue's message and attempt, or with
|
|
2092
|
+
// `inRequest` from the request that took the prompt.
|
|
2093
|
+
const receive = (created, message, delivery, { inRequest = false } = {}) => {
|
|
2094
|
+
let worker = workers.get(message.sessionId);
|
|
2095
|
+
if (!worker) {
|
|
2096
|
+
worker = new SessionWorker(agent, created, message.sessionId);
|
|
2097
|
+
workers.set(message.sessionId, worker);
|
|
2098
|
+
}
|
|
2099
|
+
worker.deliveries += 1;
|
|
2100
|
+
let work = worker.consume(message, delivery);
|
|
2101
|
+
if (inRequest) {
|
|
2102
|
+
// No queue delivered this message, so none delivers it again: a turn
|
|
2103
|
+
// that must go on later continues in a fresh message. One whose
|
|
2104
|
+
// write failed is left to the backstop, which takes its lease over.
|
|
2105
|
+
work = work.then(async (result) => {
|
|
2106
|
+
if (Number.isFinite(result?.timeoutSeconds)) {
|
|
2107
|
+
await created.queue.send(message, {
|
|
2108
|
+
idempotencyKey: `${message.sessionId}:${message.type}:${message.messageId}:${newId("again")}`,
|
|
2109
|
+
...(result.timeoutSeconds > 0 ? { delaySeconds: result.timeoutSeconds } : {}),
|
|
2110
|
+
});
|
|
2111
|
+
}
|
|
2112
|
+
return result;
|
|
2113
|
+
});
|
|
2114
|
+
}
|
|
2115
|
+
active.add(work);
|
|
2116
|
+
void work.catch((error) => {
|
|
2117
|
+
// Its queue delivers the message again; the host hears why.
|
|
2118
|
+
emit("session.error", { sessionId: message.sessionId, error: errorSummary(error) });
|
|
2119
|
+
}).finally(() => {
|
|
2120
|
+
active.delete(work);
|
|
2121
|
+
worker.deliveries -= 1;
|
|
2122
|
+
// A worker lives while a delivery runs it.
|
|
2123
|
+
if (worker.deliveries === 0 && !worker.running && workers.get(message.sessionId) === worker) workers.delete(message.sessionId);
|
|
2124
|
+
});
|
|
2125
|
+
return work;
|
|
2126
|
+
};
|
|
1788
2127
|
|
|
1789
2128
|
const openSession = (requestedId, sessionOptions = {}) => {
|
|
1790
2129
|
if (requestedId !== undefined && !validId(requestedId)) throw new TypeError(`session id must be ${idRule}`);
|
|
@@ -1801,21 +2140,43 @@ export function createDurableAgentFactory({ harness, defaultDurability, name = "
|
|
|
1801
2140
|
return { created, id };
|
|
1802
2141
|
});
|
|
1803
2142
|
void resolved.catch(() => {});
|
|
1804
|
-
|
|
2143
|
+
// With `inRequest`, a prompt runs here, in the request that took it,
|
|
2144
|
+
// where the durability allows (see `runInRequest`).
|
|
2145
|
+
const send = async (message, { inRequest = false } = {}) => {
|
|
1805
2146
|
const { created, id: sessionId } = await resolved;
|
|
1806
2147
|
const extra = {
|
|
1807
2148
|
...(seeded ? {} : { seed }),
|
|
1808
2149
|
...(context === undefined ? {} : { context }),
|
|
1809
2150
|
...(settings === undefined ? {} : { settings }),
|
|
1810
2151
|
};
|
|
1811
|
-
const full = { ...message, ...extra, sessionId };
|
|
2152
|
+
const full = { ...message, ...extra, agent: agentName, sessionId };
|
|
1812
2153
|
// A queue that ends with the process could lose the message, so its
|
|
1813
2154
|
// input reaches the log before the caller hears it was accepted.
|
|
1814
2155
|
if (!created.queueDurable) {
|
|
1815
2156
|
const log = await created.session(sessionId);
|
|
1816
2157
|
await recordMessage(log, foldSessionLog(await log.read()), full);
|
|
1817
2158
|
}
|
|
1818
|
-
|
|
2159
|
+
const idempotencyKey = `${sessionId}:${message.type}:${message.messageId}`;
|
|
2160
|
+
if (inRequest && created.runInRequest) {
|
|
2161
|
+
await created.refreshDeadline?.();
|
|
2162
|
+
const deadline = deliveryDeadline(created);
|
|
2163
|
+
if (deadline !== null) {
|
|
2164
|
+
// This request runs the turn, and the queue keeps the message as
|
|
2165
|
+
// a backstop it delivers only after the request's deadline, by
|
|
2166
|
+
// when the request has stopped. The lease names the backstop, so
|
|
2167
|
+
// if the request died, the backstop takes the session over and
|
|
2168
|
+
// continues the turn.
|
|
2169
|
+
const delaySeconds = Math.ceil(Math.max(0, deadline - Date.now()) / 1000) + inlineBackstopMarginSeconds;
|
|
2170
|
+
const { messageId: backstop } = await created.queue.send(full, { idempotencyKey, delaySeconds }) ?? {};
|
|
2171
|
+
seeded = true;
|
|
2172
|
+
const delivery = backstop ? { messageId: backstop, attempt: 0 } : null;
|
|
2173
|
+
// receive() reports a failure to the host; the backstop retries it.
|
|
2174
|
+
const work = receive(created, full, delivery, { inRequest: true }).then(() => {}, () => {});
|
|
2175
|
+
created.waitUntil?.(work);
|
|
2176
|
+
return;
|
|
2177
|
+
}
|
|
2178
|
+
}
|
|
2179
|
+
await created.queue.send(full, { idempotencyKey });
|
|
1819
2180
|
seeded = true;
|
|
1820
2181
|
};
|
|
1821
2182
|
const startTurn = (type, messageId, input, retried = false, aborted = false) => {
|
|
@@ -1840,7 +2201,7 @@ export function createDurableAgentFactory({ harness, defaultDurability, name = "
|
|
|
1840
2201
|
from = await log.ui.length();
|
|
1841
2202
|
const [state] = await Promise.all([
|
|
1842
2203
|
log.read().then(foldSessionLog),
|
|
1843
|
-
send({ type, messageId, ...(input === undefined ? {} : { input }) }).then(() => accept.resolve({ messageId, sessionId })),
|
|
2204
|
+
send({ type, messageId, ...(input === undefined ? {} : { input }) }, { inRequest: type === "prompt" }).then(() => accept.resolve({ messageId, sessionId })),
|
|
1844
2205
|
]);
|
|
1845
2206
|
// One the log does not know yet is this send's own. When the
|
|
1846
2207
|
// log says it ran and nothing shows how, either this send's
|
|
@@ -1872,7 +2233,7 @@ export function createDurableAgentFactory({ harness, defaultDurability, name = "
|
|
|
1872
2233
|
// The viewer starts where the stream is now, before the
|
|
1873
2234
|
// message can produce anything.
|
|
1874
2235
|
from ??= await log.ui.length();
|
|
1875
|
-
await send({ type, messageId, ...(input === undefined ? {} : { input }) });
|
|
2236
|
+
await send({ type, messageId, ...(input === undefined ? {} : { input }) }, { inRequest: type === "prompt" });
|
|
1876
2237
|
accept.resolve({ messageId, sessionId });
|
|
1877
2238
|
return { log, from, attaching };
|
|
1878
2239
|
} catch (error) {
|
|
@@ -1919,20 +2280,47 @@ export function createDurableAgentFactory({ harness, defaultDurability, name = "
|
|
|
1919
2280
|
},
|
|
1920
2281
|
/**
|
|
1921
2282
|
* The session's events after `cursor`, as NDJSON, from any server.
|
|
1922
|
-
*
|
|
1923
|
-
* a
|
|
2283
|
+
* A cursor is one a line carried, and a line shows only when no line
|
|
2284
|
+
* before it outranks it, so the epoch of the line a cursor names is
|
|
2285
|
+
* the rank every line after it is held to: the read starts there, not
|
|
2286
|
+
* at the stream's start, and lines rank the same however a reader
|
|
2287
|
+
* reconnects. After a line with no epoch it reads from the start.
|
|
1924
2288
|
*/
|
|
1925
2289
|
stream(cursor = 0) {
|
|
1926
2290
|
if (!Number.isSafeInteger(cursor) || cursor < 0) throw new TypeError("stream cursor must be a non-negative integer");
|
|
1927
2291
|
let reader = null;
|
|
1928
2292
|
let buffered = "";
|
|
1929
2293
|
let next = 0;
|
|
1930
|
-
|
|
2294
|
+
let shows = uiRanker();
|
|
2295
|
+
// Opens the read: at the cursor's own line, which sets the rank,
|
|
2296
|
+
// or from the start when it carries none.
|
|
2297
|
+
const open = async (ui) => {
|
|
2298
|
+
if (cursor === 0) return ui.read(0).getReader();
|
|
2299
|
+
const at = ui.read(cursor - 1).getReader();
|
|
2300
|
+
let first = "";
|
|
2301
|
+
for (;;) {
|
|
2302
|
+
const index = first.indexOf("\n");
|
|
2303
|
+
if (index >= 0) {
|
|
2304
|
+
let epoch = null;
|
|
2305
|
+
try { epoch = JSON.parse(first.slice(0, index))?.epoch; } catch {}
|
|
2306
|
+
if (!Number.isSafeInteger(epoch)) break;
|
|
2307
|
+
shows = uiRanker(epoch);
|
|
2308
|
+
buffered = first.slice(index + 1);
|
|
2309
|
+
next = cursor;
|
|
2310
|
+
return at;
|
|
2311
|
+
}
|
|
2312
|
+
const { value, done } = await at.read();
|
|
2313
|
+
if (done) return at;
|
|
2314
|
+
first += decoder.decode(value, { stream: true });
|
|
2315
|
+
}
|
|
2316
|
+
void at.cancel().catch(() => {});
|
|
2317
|
+
return ui.read(0).getReader();
|
|
2318
|
+
};
|
|
1931
2319
|
return new ReadableStream({
|
|
1932
2320
|
async pull(controller) {
|
|
1933
2321
|
if (!reader) {
|
|
1934
2322
|
const { created, id: sessionId } = await resolved;
|
|
1935
|
-
reader = (await created.session(sessionId)).ui
|
|
2323
|
+
reader = await open((await created.session(sessionId)).ui);
|
|
1936
2324
|
}
|
|
1937
2325
|
for (;;) {
|
|
1938
2326
|
const index = buffered.indexOf("\n");
|
|
@@ -1996,7 +2384,14 @@ export function createDurableAgentFactory({ harness, defaultDurability, name = "
|
|
|
1996
2384
|
// deadline would.
|
|
1997
2385
|
[durableInternals]: {
|
|
1998
2386
|
stopAtDeadline: (sessionId) => workers.get(sessionId)?.stopNow?.(),
|
|
2387
|
+
// Waits for what a session's running turn has written so far.
|
|
2388
|
+
settled: async (sessionId) => { await workers.get(sessionId)?.settled?.(); },
|
|
2389
|
+
// Delivers a queue message to this agent as its queue would.
|
|
2390
|
+
deliver: async (message, delivery) => receive(await backend(), message, delivery),
|
|
1999
2391
|
liveWorkers: () => workers.size,
|
|
2392
|
+
// A session's log as one worker holds it, with its own view of the
|
|
2393
|
+
// chain.
|
|
2394
|
+
openLog: async (sessionId) => (await backend()).session(sessionId),
|
|
2000
2395
|
// The newest lease the session's log holds, standing or not.
|
|
2001
2396
|
lastLease: async (sessionId) => {
|
|
2002
2397
|
const log = await (await backend()).session(sessionId);
|
|
@@ -2022,6 +2417,10 @@ export function createDurableAgentFactory({ harness, defaultDurability, name = "
|
|
|
2022
2417
|
async newSessionId() {
|
|
2023
2418
|
return (await backend()).newSessionId();
|
|
2024
2419
|
},
|
|
2420
|
+
/** The agent's name: its sessions' turns run only on agents with this name. */
|
|
2421
|
+
get name() {
|
|
2422
|
+
return agentName;
|
|
2423
|
+
},
|
|
2025
2424
|
/** The agent's own session, for code that holds one conversation. */
|
|
2026
2425
|
get sessionId() {
|
|
2027
2426
|
return defaultSession?.id ?? null;
|
|
@@ -2049,6 +2448,7 @@ export function createDurableAgentFactory({ harness, defaultDurability, name = "
|
|
|
2049
2448
|
async close() {
|
|
2050
2449
|
if (closed) return;
|
|
2051
2450
|
closed = true;
|
|
2451
|
+
leaveGroup(agentsByName, agentName, registered, member);
|
|
2052
2452
|
while (active.size > 0) await Promise.allSettled([...active]);
|
|
2053
2453
|
if (backendPromise) {
|
|
2054
2454
|
const created = await backendPromise.catch(() => null);
|