@earendil-works/pi-durable 1.0.4 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +66 -0
- package/README.md +149 -11
- package/dist/codemode/codemode.d.ts +173 -0
- package/dist/codemode/codemode.d.ts.map +1 -0
- package/dist/codemode/codemode.js +886 -0
- package/dist/codemode/codemode.js.map +1 -0
- package/dist/codemode/describe.d.ts +38 -0
- package/dist/codemode/describe.d.ts.map +1 -0
- package/dist/codemode/describe.js +119 -0
- package/dist/codemode/describe.js.map +1 -0
- package/dist/codemode/index.d.ts +3 -0
- package/dist/codemode/index.d.ts.map +1 -0
- package/dist/codemode/index.js +3 -0
- package/dist/codemode/index.js.map +1 -0
- package/dist/codemode/models.d.ts +28 -0
- package/dist/codemode/models.d.ts.map +1 -0
- package/dist/codemode/models.js +231 -0
- package/dist/codemode/models.js.map +1 -0
- package/dist/codemode/output.d.ts +41 -0
- package/dist/codemode/output.d.ts.map +1 -0
- package/dist/codemode/output.js +167 -0
- package/dist/codemode/output.js.map +1 -0
- package/dist/env/index.d.ts +42 -0
- package/dist/env/index.d.ts.map +1 -1
- package/dist/env/index.js.map +1 -1
- package/dist/errors.d.ts +20 -3
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +34 -5
- package/dist/errors.js.map +1 -1
- package/dist/harness/agent.d.ts +5 -2
- package/dist/harness/agent.d.ts.map +1 -1
- package/dist/harness/agent.js +72 -25
- package/dist/harness/agent.js.map +1 -1
- package/dist/harness/compaction.d.ts +6 -1
- package/dist/harness/compaction.d.ts.map +1 -1
- package/dist/harness/compaction.js +11 -8
- package/dist/harness/compaction.js.map +1 -1
- package/dist/harness/context.d.ts +29 -1
- package/dist/harness/context.d.ts.map +1 -1
- package/dist/harness/context.js +125 -24
- package/dist/harness/context.js.map +1 -1
- package/dist/harness/events.d.ts +35 -15
- package/dist/harness/events.d.ts.map +1 -1
- package/dist/harness/events.js +77 -23
- package/dist/harness/events.js.map +1 -1
- package/dist/harness/generation.d.ts +2 -0
- package/dist/harness/generation.d.ts.map +1 -1
- package/dist/harness/generation.js +34 -11
- package/dist/harness/generation.js.map +1 -1
- package/dist/harness/harness.d.ts +7 -0
- package/dist/harness/harness.d.ts.map +1 -1
- package/dist/harness/harness.js +56 -28
- package/dist/harness/harness.js.map +1 -1
- package/dist/harness/inbox.d.ts +16 -4
- package/dist/harness/inbox.d.ts.map +1 -1
- package/dist/harness/inbox.js +26 -3
- package/dist/harness/inbox.js.map +1 -1
- package/dist/harness/live.d.ts +74 -17
- package/dist/harness/live.d.ts.map +1 -1
- package/dist/harness/live.js +66 -14
- package/dist/harness/live.js.map +1 -1
- package/dist/harness/scheduler.d.ts +18 -5
- package/dist/harness/scheduler.d.ts.map +1 -1
- package/dist/harness/scheduler.js +794 -168
- package/dist/harness/scheduler.js.map +1 -1
- package/dist/harness/submissions.d.ts +7 -4
- package/dist/harness/submissions.d.ts.map +1 -1
- package/dist/harness/submissions.js +32 -13
- package/dist/harness/submissions.js.map +1 -1
- package/dist/harness/task-graph.d.ts +1 -1
- package/dist/harness/task-graph.d.ts.map +1 -1
- package/dist/harness/task-graph.js +15 -10
- package/dist/harness/task-graph.js.map +1 -1
- package/dist/harness/tool.d.ts +45 -5
- package/dist/harness/tool.d.ts.map +1 -1
- package/dist/harness/tool.js +388 -63
- package/dist/harness/tool.js.map +1 -1
- package/dist/harness/types.d.ts +171 -15
- package/dist/harness/types.d.ts.map +1 -1
- package/dist/harness/types.js.map +1 -1
- package/dist/harness/util.d.ts +6 -1
- package/dist/harness/util.d.ts.map +1 -1
- package/dist/harness/util.js +5 -2
- package/dist/harness/util.js.map +1 -1
- package/dist/harness/view.d.ts +6 -2
- package/dist/harness/view.d.ts.map +1 -1
- package/dist/harness/view.js +22 -12
- package/dist/harness/view.js.map +1 -1
- package/dist/images/cloudflare.d.ts +4 -0
- package/dist/images/cloudflare.d.ts.map +1 -0
- package/dist/images/cloudflare.js +9 -0
- package/dist/images/cloudflare.js.map +1 -0
- package/dist/images/exif.d.ts +3 -0
- package/dist/images/exif.d.ts.map +1 -0
- package/dist/images/exif.js +71 -0
- package/dist/images/exif.js.map +1 -0
- package/dist/images/index.d.ts +18 -0
- package/dist/images/index.d.ts.map +1 -0
- package/dist/images/index.js +125 -0
- package/dist/images/index.js.map +1 -0
- package/dist/images/node-worker.d.ts +17 -0
- package/dist/images/node-worker.d.ts.map +1 -0
- package/dist/images/node-worker.js +16 -0
- package/dist/images/node-worker.js.map +1 -0
- package/dist/images/node.d.ts +21 -0
- package/dist/images/node.d.ts.map +1 -0
- package/dist/images/node.js +121 -0
- package/dist/images/node.js.map +1 -0
- package/dist/index.d.ts +6 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -3
- package/dist/index.js.map +1 -1
- package/dist/search.d.ts +13 -0
- package/dist/search.d.ts.map +1 -0
- package/dist/search.js +85 -0
- package/dist/search.js.map +1 -0
- package/dist/session/observation.d.ts +10 -3
- package/dist/session/observation.d.ts.map +1 -1
- package/dist/session/observation.js +24 -34
- package/dist/session/observation.js.map +1 -1
- package/dist/session/session.d.ts +71 -6
- package/dist/session/session.d.ts.map +1 -1
- package/dist/session/session.js +463 -50
- package/dist/session/session.js.map +1 -1
- package/dist/session/transaction.d.ts +2 -0
- package/dist/session/transaction.d.ts.map +1 -1
- package/dist/session/transaction.js +19 -1
- package/dist/session/transaction.js.map +1 -1
- package/dist/storage/jsonl/storage.d.ts +8 -0
- package/dist/storage/jsonl/storage.d.ts.map +1 -1
- package/dist/storage/jsonl/storage.js +2 -1
- package/dist/storage/jsonl/storage.js.map +1 -1
- package/dist/storage/memory.d.ts +2 -0
- package/dist/storage/memory.d.ts.map +1 -1
- package/dist/storage/memory.js +79 -27
- package/dist/storage/memory.js.map +1 -1
- package/dist/storage/scan.d.ts +18 -0
- package/dist/storage/scan.d.ts.map +1 -0
- package/dist/storage/scan.js +29 -0
- package/dist/storage/scan.js.map +1 -0
- package/dist/storage/sqlite/cloudflare.d.ts +46 -0
- package/dist/storage/sqlite/cloudflare.d.ts.map +1 -0
- package/dist/storage/sqlite/cloudflare.js +106 -0
- package/dist/storage/sqlite/cloudflare.js.map +1 -0
- package/dist/storage/sqlite/storage.d.ts +2 -0
- package/dist/storage/sqlite/storage.d.ts.map +1 -1
- package/dist/storage/sqlite/storage.js +77 -27
- package/dist/storage/sqlite/storage.js.map +1 -1
- package/dist/testing/env-conformance.d.ts.map +1 -1
- package/dist/testing/env-conformance.js +100 -0
- package/dist/testing/env-conformance.js.map +1 -1
- package/dist/testing/runner.d.ts +1 -1
- package/dist/testing/runner.d.ts.map +1 -1
- package/dist/testing/runner.js.map +1 -1
- package/dist/testing/storage-conformance.d.ts.map +1 -1
- package/dist/testing/storage-conformance.js +61 -2
- package/dist/testing/storage-conformance.js.map +1 -1
- package/dist/testing/types.d.ts +2 -0
- package/dist/testing/types.d.ts.map +1 -1
- package/dist/testing/types.js.map +1 -1
- package/dist/tool-search/index.d.ts +2 -0
- package/dist/tool-search/index.d.ts.map +1 -0
- package/dist/tool-search/index.js +2 -0
- package/dist/tool-search/index.js.map +1 -0
- package/dist/tool-search/tool-search.d.ts +23 -0
- package/dist/tool-search/tool-search.d.ts.map +1 -0
- package/dist/tool-search/tool-search.js +84 -0
- package/dist/tool-search/tool-search.js.map +1 -0
- package/dist/tools/bash.d.ts +10 -2
- package/dist/tools/bash.d.ts.map +1 -1
- package/dist/tools/bash.js +27 -9
- package/dist/tools/bash.js.map +1 -1
- package/dist/tools/edit.js +1 -1
- package/dist/tools/edit.js.map +1 -1
- package/dist/tools/image-processor.d.ts +47 -0
- package/dist/tools/image-processor.d.ts.map +1 -0
- package/dist/tools/image-processor.js +30 -0
- package/dist/tools/image-processor.js.map +1 -0
- package/dist/tools/image.d.ts +8 -0
- package/dist/tools/image.d.ts.map +1 -1
- package/dist/tools/image.js +47 -0
- package/dist/tools/image.js.map +1 -1
- package/dist/tools/index.d.ts +25 -2
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +13 -5
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/read.d.ts +10 -2
- package/dist/tools/read.d.ts.map +1 -1
- package/dist/tools/read.js +87 -28
- package/dist/tools/read.js.map +1 -1
- package/dist/tools/write.js +1 -1
- package/dist/tools/write.js.map +1 -1
- package/dist/types.d.ts +89 -12
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/package.json +50 -5
|
@@ -1,12 +1,23 @@
|
|
|
1
1
|
import { copyJson } from "@earendil-works/chord";
|
|
2
2
|
import { awaitWithContext, withAbortSignal } from "@earendil-works/chord/context";
|
|
3
|
+
import { errorMessage } from "../errors.js";
|
|
3
4
|
import { agentHooks } from "./agent.js";
|
|
4
|
-
import {
|
|
5
|
+
import { readContextFrom } from "./context.js";
|
|
5
6
|
import { closedError, scanAll, Waiters } from "./util.js";
|
|
6
7
|
const SCAN_PAGE_SIZE = 256;
|
|
7
8
|
/** Longest delay `setTimeout` supports; longer sleeps wait in several steps. */
|
|
8
9
|
const MAX_TIMER_DELAY = 2_147_483_647;
|
|
9
10
|
const LIVE_STATUSES = ["pending", "running", "waiting", "completing"];
|
|
11
|
+
function conversationNode(id) {
|
|
12
|
+
return `c${id}`;
|
|
13
|
+
}
|
|
14
|
+
function conversationOf(node) {
|
|
15
|
+
return Number(node.slice(1));
|
|
16
|
+
}
|
|
17
|
+
/** The node a task hangs from: its owner task, or its conversation. */
|
|
18
|
+
function parentNode(node) {
|
|
19
|
+
return node.owner ?? conversationNode(node.conversationId);
|
|
20
|
+
}
|
|
10
21
|
/**
|
|
11
22
|
* Durable task scheduler of one Harness.
|
|
12
23
|
*
|
|
@@ -14,8 +25,14 @@ const LIVE_STATUSES = ["pending", "running", "waiting", "completing"];
|
|
|
14
25
|
* commit listener updates it on the Session line, so code running on the line reads exactly the committed state from it.
|
|
15
26
|
*
|
|
16
27
|
* Tasks and conversations form one ownership tree (spec §5.5): a task's parent is its owner task, or its conversation;
|
|
17
|
-
* a conversation's parent is its owner task, if any. Walks up that tree decide
|
|
18
|
-
*
|
|
28
|
+
* a conversation's parent is its owner task, if any. Walks up that tree decide what reaches one task: a cancelling
|
|
29
|
+
* owner, a scope. `#below` lists the tree downward around live work, so what lies below one node is a walk down from
|
|
30
|
+
* it: a task's live ordinary owned work, which holds its outcome as `completing` and delays its abort handler, a
|
|
31
|
+
* scope's tasks, a cascade's reach. No pass visits every live task; the sets below hold each pass's candidates.
|
|
32
|
+
*
|
|
33
|
+
* The ownership indexes are bounded by live work: an owner edge and an ended task's ownership fields are kept only while
|
|
34
|
+
* something is listed below them in `#below`, so only the chains above live tasks are known, whole. Work that appears
|
|
35
|
+
* later below a dropped node finds it unknown and loads its chain from storage again.
|
|
19
36
|
*
|
|
20
37
|
* Invariant: every task transition is decided and written by one callback serialized on the Session line. That covers
|
|
21
38
|
* reservation, marks, runtime commits, finalization, and the synchronous step before each phase, which applies the
|
|
@@ -37,24 +54,57 @@ export class TaskScheduler {
|
|
|
37
54
|
#conversation;
|
|
38
55
|
#context;
|
|
39
56
|
#live = new Map();
|
|
57
|
+
/**
|
|
58
|
+
* Context range last read through a task runtime, per conversation: a later read, by any of its tasks, scans only
|
|
59
|
+
* newer entries. `idleSince` is the Harness time it was first seen idle. Derived and never persisted; dropped at the
|
|
60
|
+
* first idle check after `settings.contextRetentionMs` of idleness, and at close.
|
|
61
|
+
*/
|
|
62
|
+
#contexts = new Map();
|
|
63
|
+
/** Timer for the earliest expiry of an idle context, only where timers can be unreferenced (see `#scheduleExpiry`). */
|
|
64
|
+
#expiry;
|
|
40
65
|
#invocations = new Map();
|
|
41
66
|
#taskWaiters = new Waiters();
|
|
42
67
|
/** Idle waiters by conversation; `undefined` waits for the whole Harness. */
|
|
43
68
|
#idleWaiters = new Waiters();
|
|
44
69
|
/** Definition whose migration failed per task; retried only once the registry resolves another definition. */
|
|
45
70
|
#failedMigrations = new Map();
|
|
46
|
-
/** Owner task of each
|
|
71
|
+
/** Owner task of each known conversation, `null` when ownerless. */
|
|
47
72
|
#edges = new Map();
|
|
48
|
-
/**
|
|
49
|
-
#conversationOwners = new Set();
|
|
50
|
-
/** Ownership fields of terminal tasks that walks pass through. */
|
|
73
|
+
/** Ownership fields of the known terminal tasks, which walks pass through. */
|
|
51
74
|
#settled = new Map();
|
|
75
|
+
/**
|
|
76
|
+
* Ended tasks and conversations found unneeded, dropped by `#sweep()` at the end of the commit's publication unless
|
|
77
|
+
* needed again by then: a chain load installs nodes before the work below them is listed.
|
|
78
|
+
*/
|
|
79
|
+
#dropQueue = new Set();
|
|
80
|
+
/**
|
|
81
|
+
* The tree downward around live work: each node's child nodes that are live tasks or have nodes listed below them.
|
|
82
|
+
* A conversation's parent is its owner task; ownerless ones with work are in `#roots`.
|
|
83
|
+
*/
|
|
84
|
+
#below = new Map();
|
|
85
|
+
#roots = new Set();
|
|
86
|
+
/** Known tasks whose chain may not be loaded; every other known task's chain is, and stays, loaded. */
|
|
87
|
+
#unloaded = new Set();
|
|
88
|
+
/** Live tasks with cancellation intent, the sources of abort cascades (spec §5.4). */
|
|
89
|
+
#intent = new Set();
|
|
90
|
+
/** Live tasks that are not `completing`, the candidates of a reservation pass; ones found waiting leave it. */
|
|
91
|
+
#runnable = new Set();
|
|
92
|
+
/** `completing` tasks the next finalize pass checks: those that started holding, and holders of ended work. */
|
|
93
|
+
#finalizeChecks = new Set();
|
|
94
|
+
/** Live waiting tasks by each task in their `on`. */
|
|
95
|
+
#waiters = new Map();
|
|
52
96
|
/**
|
|
53
97
|
* `failFast` waiters the next reconcile checks for a failed task in `on`: at open, when they start waiting, and when
|
|
54
98
|
* one of their tasks fails.
|
|
55
99
|
*/
|
|
56
100
|
#failFastChecks = new Set();
|
|
101
|
+
/**
|
|
102
|
+
* Live tasks with `abandonOnRestart` found at open, from an earlier Harness. The first reservation pass abort-marks
|
|
103
|
+
* them with reason `restart`, so nothing of them or below them runs a phase again.
|
|
104
|
+
*/
|
|
105
|
+
#abandoned = new Set();
|
|
57
106
|
#reconcileScheduled = false;
|
|
107
|
+
#sweepScheduled = false;
|
|
58
108
|
#cascadePending = false;
|
|
59
109
|
#unsubscribeRegistry = () => { };
|
|
60
110
|
#enabled = false;
|
|
@@ -78,7 +128,7 @@ export class TaskScheduler {
|
|
|
78
128
|
}
|
|
79
129
|
/** Load live tasks and change surviving `running` tasks back to `pending`. Dispatches nothing. */
|
|
80
130
|
async open(context) {
|
|
81
|
-
this.#session.
|
|
131
|
+
this.#session.observeCommits((publication) => this.#observe(publication));
|
|
82
132
|
this.#session.subscribeClose(() => this.#seal());
|
|
83
133
|
this.#unsubscribeRegistry = this.#registry.subscribe(() => this.#kick());
|
|
84
134
|
await this.#session.commitWith(async (tx) => {
|
|
@@ -89,7 +139,9 @@ export class TaskScheduler {
|
|
|
89
139
|
}
|
|
90
140
|
for (const records of scans) {
|
|
91
141
|
for (const record of records) {
|
|
92
|
-
this.#
|
|
142
|
+
this.#track(undefined, record);
|
|
143
|
+
if (record.abandonOnRestart === true)
|
|
144
|
+
this.#abandoned.add(record.id);
|
|
93
145
|
if (record.state.status === "running") {
|
|
94
146
|
tx.setTask(withState(record, { status: "pending", checkpoint: record.state.checkpoint }));
|
|
95
147
|
}
|
|
@@ -108,16 +160,23 @@ export class TaskScheduler {
|
|
|
108
160
|
this.#enabled = true;
|
|
109
161
|
this.#kick();
|
|
110
162
|
}
|
|
111
|
-
/**
|
|
163
|
+
/**
|
|
164
|
+
* Signal every invocation and wait for them; writes nothing. Close calls this after every close listener has run, so
|
|
165
|
+
* waits, watches, and streams a task holds end with the Session's reason, not as cancelled by its signal.
|
|
166
|
+
*/
|
|
112
167
|
async join() {
|
|
168
|
+
for (const invocation of this.#invocations.values())
|
|
169
|
+
invocation.controller.abort();
|
|
113
170
|
await Promise.allSettled([...this.#invocations.values()].map((invocation) => invocation.done));
|
|
114
171
|
}
|
|
115
172
|
/**
|
|
116
173
|
* Commit the abort mark, or settle a task that no registered definition can take as `orphaned` when nothing it owns
|
|
117
174
|
* is live, then join the run invocation seen on the line; the commit listener signalled it. The abort invocation
|
|
118
|
-
* starts once the task's ordinary owned work is gone. A `completing` task is only marked.
|
|
175
|
+
* starts once the task's ordinary owned work is gone. A `completing` task is only marked. A request replaces a
|
|
176
|
+
* `restart` mark, so a task abandoned after a restart that waits for its definition is orphaned; with `keepRestart`,
|
|
177
|
+
* as for an owner's own cleanup, such a task keeps its mark and waits on.
|
|
119
178
|
*/
|
|
120
|
-
async abort(id, context) {
|
|
179
|
+
async abort(id, context, keepRestart = false) {
|
|
121
180
|
const marked = await this.#session.commitWith(async (tx) => {
|
|
122
181
|
const current = await tx.task(id);
|
|
123
182
|
if (current === undefined)
|
|
@@ -125,9 +184,12 @@ export class TaskScheduler {
|
|
|
125
184
|
if (current.state.status === "terminal")
|
|
126
185
|
return { result: "terminal" };
|
|
127
186
|
const invocation = this.#invocations.get(id);
|
|
187
|
+
// A restart-marked task has no run invocation; it only waits for its abort.
|
|
188
|
+
if (keepRestart && current.abortReason === "restart")
|
|
189
|
+
return { result: "marked" };
|
|
128
190
|
if (invocation === undefined && current.state.status !== "completing") {
|
|
129
191
|
await this.#loadScopes(false);
|
|
130
|
-
if (!this.#
|
|
192
|
+
if (!this.#hasOwnedLive(id)) {
|
|
131
193
|
const resolution = this.#resolve(current, this.#registry.snapshot());
|
|
132
194
|
if (resolution.kind === "blocked") {
|
|
133
195
|
await this.#terminate(tx, current, { status: "orphaned", reason: resolution.reason });
|
|
@@ -135,20 +197,22 @@ export class TaskScheduler {
|
|
|
135
197
|
}
|
|
136
198
|
}
|
|
137
199
|
}
|
|
138
|
-
if (!current.abortRequested)
|
|
139
|
-
tx.setTask(
|
|
200
|
+
if (!current.abortRequested || current.abortReason !== undefined)
|
|
201
|
+
tx.setTask(withAbortMark(current));
|
|
140
202
|
return { result: "marked", run: invocation?.mode === "run" ? invocation : undefined };
|
|
141
203
|
}, context);
|
|
142
|
-
// The commit listener signalled the run; join it.
|
|
143
|
-
|
|
144
|
-
|
|
204
|
+
// The commit listener signalled the run; join it. A run that ignores its signal can outlive a failed Session,
|
|
205
|
+
// which ends the wait instead.
|
|
206
|
+
if (marked.run !== undefined) {
|
|
207
|
+
await awaitWithContext(Promise.race([marked.run.done, this.#session.failed]), context);
|
|
208
|
+
}
|
|
145
209
|
return marked.result;
|
|
146
210
|
}
|
|
147
211
|
async waitForTask(id, context) {
|
|
148
212
|
// Check and register on the line so no terminal publication falls between them.
|
|
149
213
|
const found = await this.#session.readOnLine(async () => {
|
|
150
214
|
if (this.#closing)
|
|
151
|
-
throw closedError();
|
|
215
|
+
throw closedError(this.#session);
|
|
152
216
|
if (this.#live.has(id))
|
|
153
217
|
return { promise: this.#taskWaiters.add(id, context) };
|
|
154
218
|
const record = await this.#storage.task(id, context);
|
|
@@ -164,7 +228,7 @@ export class TaskScheduler {
|
|
|
164
228
|
*/
|
|
165
229
|
waitForIdle(conversationId, context) {
|
|
166
230
|
if (this.#closing)
|
|
167
|
-
return Promise.reject(closedError());
|
|
231
|
+
return Promise.reject(closedError(this.#session));
|
|
168
232
|
if (this.#idle(conversationId))
|
|
169
233
|
return Promise.resolve();
|
|
170
234
|
this.#scheduleReconcile();
|
|
@@ -180,14 +244,13 @@ export class TaskScheduler {
|
|
|
180
244
|
const queued = await this.#loadScopes(true);
|
|
181
245
|
const scope = { conversation: conversationId };
|
|
182
246
|
const reached = [];
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
if (this.#inScope(parentOf(record), scope, background) !== true)
|
|
247
|
+
const walk = this.#walkDown(conversationNode(conversationId), { crossBackground: background });
|
|
248
|
+
for (const { node, live } of walk) {
|
|
249
|
+
if (live === undefined || (node.background && !background))
|
|
187
250
|
continue;
|
|
188
|
-
reached.push(
|
|
189
|
-
if (!
|
|
190
|
-
tx.setTask(
|
|
251
|
+
reached.push(live.id);
|
|
252
|
+
if (!live.abortRequested || live.abortReason !== undefined)
|
|
253
|
+
tx.setTask(withAbortMark(live));
|
|
191
254
|
}
|
|
192
255
|
for (const id of queued) {
|
|
193
256
|
if (this.#inScope({ conversation: id }, scope, background) === true)
|
|
@@ -205,6 +268,12 @@ export class TaskScheduler {
|
|
|
205
268
|
const updated = [];
|
|
206
269
|
const failed = [];
|
|
207
270
|
let changed = false;
|
|
271
|
+
// Edges first, so the tasks a commit creates in a new conversation hang from it.
|
|
272
|
+
for (const change of publication.changes) {
|
|
273
|
+
if (change.type === "conversation" && !this.#edges.has(change.value.id)) {
|
|
274
|
+
this.#setEdge(change.value.id, change.value.owner?.taskId ?? null);
|
|
275
|
+
}
|
|
276
|
+
}
|
|
208
277
|
for (const change of publication.changes) {
|
|
209
278
|
if (change.type !== "task")
|
|
210
279
|
continue;
|
|
@@ -213,12 +282,10 @@ export class TaskScheduler {
|
|
|
213
282
|
const previous = this.#live.get(record.id);
|
|
214
283
|
if (failedOutcome(record) && (previous === undefined || !failedOutcome(previous)))
|
|
215
284
|
failed.push(record.id);
|
|
285
|
+
this.#track(previous, record);
|
|
216
286
|
if (record.state.status === "terminal") {
|
|
217
|
-
this.#live.delete(record.id);
|
|
218
287
|
this.#failedMigrations.delete(record.id);
|
|
219
288
|
this.#failFastChecks.delete(record.id);
|
|
220
|
-
if (this.#conversationOwners.has(record.id))
|
|
221
|
-
this.#settled.set(record.id, nodeOf(record));
|
|
222
289
|
this.#taskWaiters.resolve(record.id, record);
|
|
223
290
|
// Its owner may finalize now.
|
|
224
291
|
this.#scheduleReconcile();
|
|
@@ -231,6 +298,9 @@ export class TaskScheduler {
|
|
|
231
298
|
if (invocation?.mode === "run")
|
|
232
299
|
invocation.controller.abort();
|
|
233
300
|
}
|
|
301
|
+
// A request replacing a `restart` mark upgrades the marks of the work below.
|
|
302
|
+
if (previous?.abortReason !== undefined && record.abortReason === undefined)
|
|
303
|
+
this.#cascadePending = true;
|
|
234
304
|
const status = record.state.status;
|
|
235
305
|
if (status === "completing" && previous?.state.status !== "completing") {
|
|
236
306
|
if (cancellationIntent(record))
|
|
@@ -241,52 +311,116 @@ export class TaskScheduler {
|
|
|
241
311
|
this.#failFastChecks.add(record.id);
|
|
242
312
|
this.#scheduleReconcile();
|
|
243
313
|
}
|
|
244
|
-
|
|
245
|
-
|
|
314
|
+
// Its owners and place never change: what is above it is checked once, when it becomes live.
|
|
315
|
+
if (previous === undefined)
|
|
316
|
+
updated.push(record);
|
|
246
317
|
}
|
|
247
318
|
for (const id of failed) {
|
|
248
|
-
for (const
|
|
249
|
-
const state =
|
|
250
|
-
if (state
|
|
251
|
-
this.#failFastChecks.add(
|
|
319
|
+
for (const waiter of this.#waiters.get(id) ?? []) {
|
|
320
|
+
const state = this.#live.get(waiter)?.state;
|
|
321
|
+
if (state?.status === "waiting" && state.policy === "failFast") {
|
|
322
|
+
this.#failFastChecks.add(waiter);
|
|
252
323
|
this.#scheduleReconcile();
|
|
253
324
|
}
|
|
254
325
|
}
|
|
255
326
|
}
|
|
256
|
-
for (const change of publication.changes) {
|
|
257
|
-
if (change.type === "conversation" && !this.#edges.has(change.value.id)) {
|
|
258
|
-
this.#setEdge(change.value.id, change.value.owner?.taskId ?? null);
|
|
259
|
-
}
|
|
260
|
-
}
|
|
261
327
|
for (const change of publication.changes) {
|
|
262
328
|
// A queued input below a cancelled owner is withdrawn, even after its cascade.
|
|
263
329
|
if (change.type !== "submission" || change.value.status !== "queued" || change.value.type !== "input")
|
|
264
330
|
continue;
|
|
265
|
-
|
|
266
|
-
if (
|
|
331
|
+
// Only an owner with cancellation intent withdraws it; a later intent's mark runs the cascade then.
|
|
332
|
+
if (this.#intent.size === 0)
|
|
333
|
+
continue;
|
|
334
|
+
const conversation = change.value.conversationId;
|
|
335
|
+
if (!this.#known(conversationNode(conversation)) || this.#belowCancelled({ conversation })) {
|
|
267
336
|
this.#cascadePending = true;
|
|
337
|
+
}
|
|
268
338
|
}
|
|
269
339
|
for (const record of updated) {
|
|
270
340
|
// Work created below a cancelled owner, even after its cascade, is aborted too.
|
|
271
|
-
if (
|
|
341
|
+
if (this.#unloaded.has(record.id))
|
|
272
342
|
this.#scheduleReconcile();
|
|
273
343
|
else if (!record.background && !record.abortRequested && this.#belowCancelled(parentOf(record))) {
|
|
274
344
|
this.#cascadePending = true;
|
|
275
345
|
}
|
|
276
346
|
}
|
|
277
|
-
//
|
|
347
|
+
// A cascade that a reservation or step found pending runs with the next commit of any kind.
|
|
278
348
|
if (this.#cascadePending)
|
|
279
349
|
this.#scheduleReconcile();
|
|
350
|
+
this.#sweep();
|
|
280
351
|
if (!changed)
|
|
281
352
|
return;
|
|
282
|
-
this.#
|
|
353
|
+
this.#settleIdle();
|
|
283
354
|
this.#kick();
|
|
284
355
|
}
|
|
285
|
-
|
|
356
|
+
/**
|
|
357
|
+
* `settings.contextRetentionMs`, or 0 when the host's settings throw. Kept contexts are only a cache, so dropping them
|
|
358
|
+
* is safe, while a throw here would escape commit listeners, reconciliation, and the expiry timer.
|
|
359
|
+
*/
|
|
360
|
+
#contextRetentionMs() {
|
|
361
|
+
try {
|
|
362
|
+
return this.#settings().contextRetentionMs;
|
|
363
|
+
}
|
|
364
|
+
catch (error) {
|
|
365
|
+
this.#report(error);
|
|
366
|
+
return 0;
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
/** Resolve idle waiters, and drop each kept context whose conversation has been idle for the retention period. */
|
|
370
|
+
#settleIdle() {
|
|
286
371
|
for (const conversationId of this.#idleWaiters.keys()) {
|
|
287
372
|
if (this.#idle(conversationId))
|
|
288
373
|
this.#idleWaiters.resolve(conversationId);
|
|
289
374
|
}
|
|
375
|
+
const now = this.#now();
|
|
376
|
+
const retention = this.#contextRetentionMs();
|
|
377
|
+
for (const [conversationId, kept] of this.#contexts) {
|
|
378
|
+
if (kept.idleSince !== undefined && now - kept.idleSince >= retention) {
|
|
379
|
+
this.#contexts.delete(conversationId);
|
|
380
|
+
}
|
|
381
|
+
else if (!this.#idle(conversationId)) {
|
|
382
|
+
kept.idleSince = undefined;
|
|
383
|
+
}
|
|
384
|
+
else if (retention > 0) {
|
|
385
|
+
kept.idleSince ??= now;
|
|
386
|
+
}
|
|
387
|
+
else {
|
|
388
|
+
this.#contexts.delete(conversationId);
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
this.#scheduleExpiry();
|
|
392
|
+
}
|
|
393
|
+
/**
|
|
394
|
+
* Run `#settleIdle()` when the earliest idle context expires. The timer is unreferenced, so it never keeps the process
|
|
395
|
+
* alive. Where timers cannot be unreferenced, as in Cloudflare Workers, none is kept: a pending timer could keep a
|
|
396
|
+
* Durable Object from being evicted, and eviction frees the contexts. There, task changes alone check expiry.
|
|
397
|
+
*/
|
|
398
|
+
#scheduleExpiry() {
|
|
399
|
+
const retention = this.#contextRetentionMs();
|
|
400
|
+
let at;
|
|
401
|
+
for (const kept of this.#contexts.values()) {
|
|
402
|
+
if (kept.idleSince !== undefined && (at === undefined || kept.idleSince + retention < at)) {
|
|
403
|
+
at = kept.idleSince + retention;
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
if (this.#expiry !== undefined && this.#expiry.at === at)
|
|
407
|
+
return;
|
|
408
|
+
if (this.#expiry !== undefined)
|
|
409
|
+
clearTimeout(this.#expiry.timer);
|
|
410
|
+
this.#expiry = undefined;
|
|
411
|
+
if (at === undefined || this.#closing)
|
|
412
|
+
return;
|
|
413
|
+
const timer = setTimeout(() => {
|
|
414
|
+
this.#expiry = undefined;
|
|
415
|
+
this.#settleIdle();
|
|
416
|
+
}, Math.min(Math.max(0, at - this.#now()), MAX_TIMER_DELAY));
|
|
417
|
+
const unref = timer.unref;
|
|
418
|
+
if (typeof unref !== "function") {
|
|
419
|
+
clearTimeout(timer);
|
|
420
|
+
return;
|
|
421
|
+
}
|
|
422
|
+
unref.call(timer);
|
|
423
|
+
this.#expiry = { at, timer };
|
|
290
424
|
}
|
|
291
425
|
// ─── Ownership ───────────────────────────────────────────────────────────
|
|
292
426
|
#scheduleReconcile() {
|
|
@@ -312,17 +446,28 @@ export class TaskScheduler {
|
|
|
312
446
|
if (this.#closing)
|
|
313
447
|
return;
|
|
314
448
|
const queued = await this.#loadScopes(cascade);
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
449
|
+
// A cascade from an abandoned owner passes its `restart` reason on; any other intent marks, or upgrades a
|
|
450
|
+
// `restart` mark, as a request. Collected first, so a request wins over a `restart` mark of the same pass.
|
|
451
|
+
const marks = new Map();
|
|
452
|
+
const mark = (record, reason) => {
|
|
453
|
+
const staged = marks.get(record.id);
|
|
454
|
+
if (staged !== undefined && staged.reason === undefined)
|
|
455
|
+
return;
|
|
456
|
+
if (record.abortRequested && (record.abortReason === undefined || reason !== undefined))
|
|
318
457
|
return;
|
|
319
|
-
|
|
320
|
-
tx.setTask({ ...record, abortRequested: true });
|
|
458
|
+
marks.set(record.id, { record, reason });
|
|
321
459
|
};
|
|
322
|
-
//
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
460
|
+
// Every live foreground task below a live owner with cancellation intent, which is its nearest one: a walk down
|
|
461
|
+
// from each owner with intent does not enter another, whose own walk covers what is below it. Derived again on
|
|
462
|
+
// every pass, as loading edges can reveal a cancelled owner.
|
|
463
|
+
for (const id of this.#intent) {
|
|
464
|
+
const owner = this.#live.get(id);
|
|
465
|
+
// Only an owner abandoned after a restart, and nothing else, passes its reason on.
|
|
466
|
+
const restart = owner.abortReason === "restart" && !failedOutcome(owner);
|
|
467
|
+
for (const { node, live } of this.#walkDown(id, { enter: (below) => !this.#intent.has(below) })) {
|
|
468
|
+
if (live !== undefined && !node.background)
|
|
469
|
+
mark(live, restart ? "restart" : undefined);
|
|
470
|
+
}
|
|
326
471
|
}
|
|
327
472
|
for (const id of checks) {
|
|
328
473
|
const waiter = this.#live.get(id);
|
|
@@ -332,9 +477,11 @@ export class TaskScheduler {
|
|
|
332
477
|
for (const member of waiter.state.on) {
|
|
333
478
|
const record = this.#live.get(member);
|
|
334
479
|
if (record !== undefined && !failedOutcome(record))
|
|
335
|
-
mark(record);
|
|
480
|
+
mark(record, undefined);
|
|
336
481
|
}
|
|
337
482
|
}
|
|
483
|
+
for (const { record, reason } of marks.values())
|
|
484
|
+
tx.setTask(withAbortMark(record, reason));
|
|
338
485
|
for (const id of queued)
|
|
339
486
|
if (this.#belowCancelled({ conversation: id }))
|
|
340
487
|
await this.#withdrawInputs(tx, id);
|
|
@@ -342,14 +489,11 @@ export class TaskScheduler {
|
|
|
342
489
|
}, this.#context);
|
|
343
490
|
}
|
|
344
491
|
catch (error) {
|
|
345
|
-
//
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
this.#failFastChecks.add(id);
|
|
349
|
-
if (!this.#closing)
|
|
350
|
-
this.#report(error);
|
|
492
|
+
// No extension code runs in this commit: a throw is a storage failure, a host callback, or a bug. None is fixed by
|
|
493
|
+
// running the pass again, so it fails the Session, which reports it.
|
|
494
|
+
this.#failSession(error);
|
|
351
495
|
}
|
|
352
|
-
this.#
|
|
496
|
+
this.#settleIdle();
|
|
353
497
|
}
|
|
354
498
|
/** Whether any of `ids` holds or ended with an outcome other than `completed`. */
|
|
355
499
|
async #anyFailed(ids) {
|
|
@@ -366,53 +510,86 @@ export class TaskScheduler {
|
|
|
366
510
|
* Harness cleanup here.
|
|
367
511
|
*/
|
|
368
512
|
async #finalize(tx) {
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
513
|
+
// A worklist: the tasks that started holding, and the holders of work that ended. Finalizing one can free the
|
|
514
|
+
// nearest live task above it, which goes on the list. Tasks finalized here have no live work below them, so walks
|
|
515
|
+
// down skip them.
|
|
516
|
+
// In rounds: the holders the tasks of one round free are checked once, in the next, so a holder of many tasks that
|
|
517
|
+
// end together is walked once, not once per task.
|
|
518
|
+
let round = [...this.#finalizeChecks];
|
|
519
|
+
this.#finalizeChecks.clear();
|
|
520
|
+
const done = new Set();
|
|
521
|
+
const overlay = overlayOf(tx);
|
|
522
|
+
while (round.length > 0) {
|
|
523
|
+
const next = new Set();
|
|
524
|
+
for (const id of round) {
|
|
525
|
+
const record = overlay.tasks.get(id) ?? this.#live.get(id);
|
|
526
|
+
if (record?.state.status !== "completing" || done.has(id))
|
|
527
|
+
continue;
|
|
528
|
+
if (this.#hasOwnedLive(id, overlay, done))
|
|
529
|
+
continue;
|
|
376
530
|
const outcome = record.state.outcome;
|
|
377
|
-
|
|
531
|
+
const terminal = withState(record, { status: "terminal", outcome });
|
|
532
|
+
tx.setTask(terminal);
|
|
533
|
+
overlay.tasks.set(id, terminal);
|
|
534
|
+
done.add(id);
|
|
378
535
|
// REMINDER: only the scheduler writes `faulted` and `orphaned` (spec §5.4); their cleanup waits for this commit.
|
|
379
536
|
if (outcome.status === "faulted" || outcome.status === "orphaned") {
|
|
380
537
|
await this.#settleOutcome(tx, record, outcome);
|
|
381
538
|
}
|
|
539
|
+
const above = this.#nearestLiveAbove(record, overlay);
|
|
540
|
+
if (above?.state.status === "completing")
|
|
541
|
+
next.add(above.id);
|
|
382
542
|
}
|
|
543
|
+
round = [...next];
|
|
383
544
|
}
|
|
384
545
|
}
|
|
385
546
|
/**
|
|
386
|
-
* Load the owner chains of
|
|
387
|
-
* Session line; returns the latter. Reads committed Storage directly, so it may run inside a commit callback.
|
|
547
|
+
* Load the owner chains of the tasks in `#unloaded` and, with `queued`, of every conversation with queued inputs,
|
|
548
|
+
* on the Session line; returns the latter. Reads committed Storage directly, so it may run inside a commit callback.
|
|
388
549
|
*/
|
|
389
550
|
async #loadScopes(queued) {
|
|
390
|
-
for (const
|
|
391
|
-
|
|
392
|
-
|
|
551
|
+
for (const id of [...this.#unloaded]) {
|
|
552
|
+
// An earlier load in this loop may have passed it.
|
|
553
|
+
if (this.#unloaded.has(id))
|
|
554
|
+
await this.#loadChain({ task: id });
|
|
393
555
|
}
|
|
394
556
|
if (!queued)
|
|
395
557
|
return [];
|
|
396
558
|
const submissions = await scanAll((cursor) => this.#storage.scanSubmissions({ status: "queued" }, SCAN_PAGE_SIZE, cursor, this.#context));
|
|
397
|
-
const
|
|
559
|
+
const inputs = submissions.filter((submission) => submission.type === "input");
|
|
560
|
+
const conversations = [...new Set(inputs.map((submission) => submission.conversationId))];
|
|
398
561
|
for (const id of conversations)
|
|
399
562
|
await this.#loadChain({ conversation: id });
|
|
400
563
|
return conversations;
|
|
401
564
|
}
|
|
402
|
-
/**
|
|
565
|
+
/**
|
|
566
|
+
* Load the owner edges and task nodes from `start` up to its ownerless root. A known task outside `#unloaded` ends the
|
|
567
|
+
* walk, as its chain is loaded; the `#unloaded` tasks passed have a loaded chain afterwards. Loading a node can reveal
|
|
568
|
+
* a cancelled owner, so it makes the next reconcile derive cascade marks.
|
|
569
|
+
*/
|
|
403
570
|
async #loadChain(start, overlay) {
|
|
571
|
+
const passed = [];
|
|
404
572
|
let at = start;
|
|
405
573
|
while (at !== undefined) {
|
|
406
574
|
if ("task" in at) {
|
|
407
|
-
|
|
575
|
+
const id = at.task;
|
|
576
|
+
let node = this.#live.get(id) ?? this.#settled.get(id);
|
|
577
|
+
// Created in this commit: the load from its own record covers what is above it.
|
|
578
|
+
if (node === undefined && overlay?.tasks.has(id) === true)
|
|
579
|
+
break;
|
|
580
|
+
if (node !== undefined && !this.#unloaded.has(id))
|
|
581
|
+
break;
|
|
408
582
|
if (node === undefined) {
|
|
409
|
-
const record = await this.#storage.task(
|
|
583
|
+
const record = await this.#storage.task(id, this.#context);
|
|
410
584
|
if (record === undefined)
|
|
411
585
|
return;
|
|
412
586
|
node = nodeOf(record);
|
|
413
587
|
if (record.state.status === "terminal")
|
|
414
|
-
this.#
|
|
588
|
+
this.#settle(id, node);
|
|
589
|
+
// A loaded node can reveal a cancelling owner: the next reconcile derives marks again.
|
|
590
|
+
this.#scheduleReconcile();
|
|
415
591
|
}
|
|
592
|
+
passed.push(id);
|
|
416
593
|
at = parentOf(node);
|
|
417
594
|
}
|
|
418
595
|
else {
|
|
@@ -421,15 +598,202 @@ export class TaskScheduler {
|
|
|
421
598
|
const record = await this.#storage.conversation(at.conversation, this.#context);
|
|
422
599
|
edge = record?.owner?.taskId ?? null;
|
|
423
600
|
this.#setEdge(at.conversation, edge);
|
|
601
|
+
// A loaded node can reveal a cancelling owner: the next reconcile derives marks again.
|
|
602
|
+
this.#scheduleReconcile();
|
|
424
603
|
}
|
|
425
604
|
at = edge === null ? undefined : { task: edge };
|
|
426
605
|
}
|
|
427
606
|
}
|
|
607
|
+
for (const id of passed)
|
|
608
|
+
this.#unloaded.delete(id);
|
|
428
609
|
}
|
|
429
610
|
#setEdge(conversationId, owner) {
|
|
430
611
|
this.#edges.set(conversationId, owner);
|
|
431
|
-
|
|
432
|
-
|
|
612
|
+
// The conversation now hangs from its owner, or is a root; with nothing below it, it waits for the sweep.
|
|
613
|
+
this.#relist(conversationNode(conversationId));
|
|
614
|
+
this.#dropIfUnneeded(conversationNode(conversationId));
|
|
615
|
+
}
|
|
616
|
+
/** Know a terminal task a chain load read, so walks pass through it; its own chain is not loaded yet. */
|
|
617
|
+
#settle(id, node) {
|
|
618
|
+
this.#settled.set(id, node);
|
|
619
|
+
this.#unloaded.add(id);
|
|
620
|
+
this.#relist(id);
|
|
621
|
+
this.#dropIfUnneeded(id);
|
|
622
|
+
}
|
|
623
|
+
/** Queue an ended task or a conversation for the sweep when nothing is listed below it. */
|
|
624
|
+
#dropIfUnneeded(node) {
|
|
625
|
+
if (typeof node !== "string" && this.#live.has(node))
|
|
626
|
+
return;
|
|
627
|
+
if (this.#below.has(node))
|
|
628
|
+
return;
|
|
629
|
+
this.#dropQueue.add(node);
|
|
630
|
+
this.#scheduleSweep();
|
|
631
|
+
}
|
|
632
|
+
/**
|
|
633
|
+
* Sweep in a job of its own on the Session line, after the operation underway: a rejected commit, one with nothing to
|
|
634
|
+
* write, and a read publish nothing, so the sweep at the end of `#observe` would not run for what they loaded.
|
|
635
|
+
*/
|
|
636
|
+
#scheduleSweep() {
|
|
637
|
+
if (this.#sweepScheduled || this.#closing)
|
|
638
|
+
return;
|
|
639
|
+
this.#sweepScheduled = true;
|
|
640
|
+
this.#session
|
|
641
|
+
.readOnLine(async () => {
|
|
642
|
+
this.#sweepScheduled = false;
|
|
643
|
+
this.#sweep();
|
|
644
|
+
})
|
|
645
|
+
// Rejected only once the Session is closing or failed, when nothing is kept anyway.
|
|
646
|
+
.catch(() => { });
|
|
647
|
+
}
|
|
648
|
+
/**
|
|
649
|
+
* Drop the queued nodes that are still unneeded: not live, nothing listed below. Runs once a publication is fully
|
|
650
|
+
* tracked, and as a job of its own on the line, never inside an operation: work a commit creates below a chain its
|
|
651
|
+
* own callback loaded keeps that chain. A flat loop: unlisting already queued every node on the way up that lost its
|
|
652
|
+
* last listed node.
|
|
653
|
+
*/
|
|
654
|
+
#sweep() {
|
|
655
|
+
for (const node of this.#dropQueue) {
|
|
656
|
+
if (this.#below.has(node))
|
|
657
|
+
continue;
|
|
658
|
+
if (typeof node === "string") {
|
|
659
|
+
this.#edges.delete(conversationOf(node));
|
|
660
|
+
this.#roots.delete(conversationOf(node));
|
|
661
|
+
}
|
|
662
|
+
else if (!this.#live.has(node)) {
|
|
663
|
+
this.#settled.delete(node);
|
|
664
|
+
this.#unloaded.delete(node);
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
this.#dropQueue.clear();
|
|
668
|
+
}
|
|
669
|
+
/**
|
|
670
|
+
* Bring the scheduler's sets in line with a task's committed record. A task that ended may free the nearest live task
|
|
671
|
+
* above it: one holding its outcome finalizes, an abort-marked one runs its abort handler; its waiters may run.
|
|
672
|
+
*/
|
|
673
|
+
#track(previous, record) {
|
|
674
|
+
const id = record.id;
|
|
675
|
+
if (previous?.state.status === "waiting")
|
|
676
|
+
for (const on of previous.state.on)
|
|
677
|
+
deleteFrom(this.#waiters, on, id);
|
|
678
|
+
if (record.state.status === "terminal") {
|
|
679
|
+
const node = nodeOf(record);
|
|
680
|
+
this.#live.delete(id);
|
|
681
|
+
this.#intent.delete(id);
|
|
682
|
+
this.#runnable.delete(id);
|
|
683
|
+
this.#finalizeChecks.delete(id);
|
|
684
|
+
this.#settled.set(id, node);
|
|
685
|
+
// Before anything is forgotten: the walk up to the task it held passes the ended tasks between them. Its chain is
|
|
686
|
+
// loaded, as every commit that ends a task loads the chains of `#unloaded` first.
|
|
687
|
+
this.#release(node);
|
|
688
|
+
for (const waiter of this.#waiters.get(id) ?? [])
|
|
689
|
+
this.#runnable.add(waiter);
|
|
690
|
+
this.#relist(id);
|
|
691
|
+
this.#dropIfUnneeded(id);
|
|
692
|
+
return;
|
|
693
|
+
}
|
|
694
|
+
this.#live.set(id, record);
|
|
695
|
+
if (previous === undefined) {
|
|
696
|
+
if (!this.#known(parentNode(record)))
|
|
697
|
+
this.#unloaded.add(id);
|
|
698
|
+
this.#relist(id);
|
|
699
|
+
}
|
|
700
|
+
if (cancellationIntent(record))
|
|
701
|
+
this.#intent.add(id);
|
|
702
|
+
else
|
|
703
|
+
this.#intent.delete(id);
|
|
704
|
+
if (record.state.status === "completing") {
|
|
705
|
+
this.#runnable.delete(id);
|
|
706
|
+
this.#finalizeChecks.add(id);
|
|
707
|
+
}
|
|
708
|
+
else {
|
|
709
|
+
this.#runnable.add(id);
|
|
710
|
+
}
|
|
711
|
+
if (record.state.status === "waiting")
|
|
712
|
+
for (const on of record.state.on)
|
|
713
|
+
addTo(this.#waiters, on, id);
|
|
714
|
+
}
|
|
715
|
+
/** The nearest live task above ended ordinary work may be free now: one holding its outcome, or an aborting one. */
|
|
716
|
+
#release(node, overlay) {
|
|
717
|
+
const above = this.#nearestLiveAbove(node, overlay);
|
|
718
|
+
if (above?.state.status === "completing")
|
|
719
|
+
this.#finalizeChecks.add(above.id);
|
|
720
|
+
else if (above?.abortRequested === true)
|
|
721
|
+
this.#runnable.add(above.id);
|
|
722
|
+
return above;
|
|
723
|
+
}
|
|
724
|
+
/**
|
|
725
|
+
* List `start` under its parent node in `#below` while it is a live task or has nodes listed below it, and unlist it
|
|
726
|
+
* otherwise; then its parent, as far up as that changes. So `#below` holds exactly the tree around live work; a
|
|
727
|
+
* parent that loses its last listed node is queued for the sweep. A loop, as ownership chains can be deeper than the
|
|
728
|
+
* call stack.
|
|
729
|
+
*/
|
|
730
|
+
#relist(start) {
|
|
731
|
+
let node = start;
|
|
732
|
+
for (;;) {
|
|
733
|
+
const needed = (typeof node !== "string" && this.#live.has(node)) || this.#below.has(node);
|
|
734
|
+
let parent;
|
|
735
|
+
if (typeof node === "string") {
|
|
736
|
+
parent = this.#edges.get(conversationOf(node));
|
|
737
|
+
}
|
|
738
|
+
else {
|
|
739
|
+
const known = this.#live.get(node) ?? this.#settled.get(node);
|
|
740
|
+
if (known === undefined)
|
|
741
|
+
return;
|
|
742
|
+
parent = parentNode(known);
|
|
743
|
+
}
|
|
744
|
+
if (parent === undefined)
|
|
745
|
+
return;
|
|
746
|
+
if (parent === null) {
|
|
747
|
+
const root = conversationOf(node);
|
|
748
|
+
if (needed)
|
|
749
|
+
this.#roots.add(root);
|
|
750
|
+
else
|
|
751
|
+
this.#roots.delete(root);
|
|
752
|
+
return;
|
|
753
|
+
}
|
|
754
|
+
if ((this.#below.get(parent)?.has(node) === true) === needed)
|
|
755
|
+
return;
|
|
756
|
+
if (needed) {
|
|
757
|
+
const first = !this.#below.has(parent);
|
|
758
|
+
addTo(this.#below, parent, node);
|
|
759
|
+
if (!first)
|
|
760
|
+
return;
|
|
761
|
+
}
|
|
762
|
+
else {
|
|
763
|
+
deleteFrom(this.#below, parent, node);
|
|
764
|
+
if (this.#below.has(parent))
|
|
765
|
+
return;
|
|
766
|
+
this.#dropIfUnneeded(parent);
|
|
767
|
+
}
|
|
768
|
+
node = parent;
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
/**
|
|
772
|
+
* The nearest live task above ended ordinary work `node`, in the overlay's view: the one that work held, if any. Work
|
|
773
|
+
* at or below a background task holds nothing above it.
|
|
774
|
+
*/
|
|
775
|
+
#nearestLiveAbove(node, overlay) {
|
|
776
|
+
for (let current = node; !current.background;) {
|
|
777
|
+
const owner = current.owner ?? this.#edge(current.conversationId, overlay);
|
|
778
|
+
if (owner === undefined || owner === null)
|
|
779
|
+
return undefined;
|
|
780
|
+
const record = overlay?.tasks.get(owner) ?? this.#live.get(owner);
|
|
781
|
+
if (record !== undefined && record.state.status !== "terminal")
|
|
782
|
+
return record;
|
|
783
|
+
const above = record ?? this.#settled.get(owner);
|
|
784
|
+
if (above === undefined)
|
|
785
|
+
return undefined;
|
|
786
|
+
current = above;
|
|
787
|
+
}
|
|
788
|
+
return undefined;
|
|
789
|
+
}
|
|
790
|
+
/** Whether the chain above `node` is loaded, without walking: known tasks outside `#unloaded` have a loaded chain. */
|
|
791
|
+
#known(node) {
|
|
792
|
+
if (typeof node !== "string") {
|
|
793
|
+
return (this.#live.has(node) || this.#settled.has(node)) && !this.#unloaded.has(node);
|
|
794
|
+
}
|
|
795
|
+
const owner = this.#edges.get(conversationOf(node));
|
|
796
|
+
return owner === null || (owner !== undefined && this.#known(owner));
|
|
433
797
|
}
|
|
434
798
|
/** Owner task of a conversation, `null` when ownerless, `undefined` while not loaded. */
|
|
435
799
|
#edge(id, overlay) {
|
|
@@ -462,51 +826,96 @@ export class TaskScheduler {
|
|
|
462
826
|
}
|
|
463
827
|
}
|
|
464
828
|
}
|
|
465
|
-
/**
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
for (
|
|
475
|
-
const
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
829
|
+
/**
|
|
830
|
+
* The tasks below `start` that a walk down reaches: those `#below` lists and, under an overlay, the commit's new tasks,
|
|
831
|
+
* with its candidates in place of committed records. Background tasks are reached but not entered, unless
|
|
832
|
+
* `crossBackground`, and neither are the tasks `enter` rejects. A loop, as chains can be deeper than the call stack.
|
|
833
|
+
*/
|
|
834
|
+
*#walkDown(start, options = {}) {
|
|
835
|
+
const { overlay, crossBackground = false, enter } = options;
|
|
836
|
+
const links = overlay === undefined ? undefined : this.#links(overlay);
|
|
837
|
+
const pending = [start];
|
|
838
|
+
for (let at = pending.pop(); at !== undefined; at = pending.pop()) {
|
|
839
|
+
for (const children of [this.#below.get(at), links?.get(at)]) {
|
|
840
|
+
for (const child of children ?? []) {
|
|
841
|
+
if (typeof child === "string") {
|
|
842
|
+
pending.push(child);
|
|
843
|
+
continue;
|
|
844
|
+
}
|
|
845
|
+
const record = overlay?.tasks.get(child) ?? this.#live.get(child);
|
|
846
|
+
const node = record ?? this.#settled.get(child);
|
|
847
|
+
if (node === undefined)
|
|
848
|
+
continue;
|
|
849
|
+
const live = record !== undefined && record.state.status !== "terminal" ? record : undefined;
|
|
850
|
+
yield { id: child, node, live };
|
|
851
|
+
if ((crossBackground || !node.background) && (enter === undefined || enter(child)))
|
|
852
|
+
pending.push(child);
|
|
853
|
+
}
|
|
854
|
+
}
|
|
484
855
|
}
|
|
485
856
|
}
|
|
486
857
|
/**
|
|
487
|
-
*
|
|
488
|
-
*
|
|
858
|
+
* The links walks down need for the commit's new tasks: each one under its parent node, then each node above it that
|
|
859
|
+
* `#below` does not list yet, until one it lists, a live task (always listed), or the chain's end.
|
|
489
860
|
*/
|
|
490
|
-
#
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
861
|
+
#links(overlay) {
|
|
862
|
+
if (overlay.links !== undefined)
|
|
863
|
+
return overlay.links;
|
|
864
|
+
const links = new Map();
|
|
865
|
+
for (const record of overlay.tasks.values()) {
|
|
866
|
+
if (this.#live.has(record.id) || this.#settled.has(record.id))
|
|
494
867
|
continue;
|
|
495
|
-
|
|
496
|
-
|
|
868
|
+
let child = record.id;
|
|
869
|
+
let parent = parentNode(record);
|
|
870
|
+
while (parent !== undefined && parent !== null && links.get(parent)?.has(child) !== true) {
|
|
871
|
+
addTo(links, parent, child);
|
|
872
|
+
if (typeof parent !== "string" && (this.#live.has(parent) || overlay.tasks.has(parent)))
|
|
497
873
|
break;
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
874
|
+
let above;
|
|
875
|
+
if (typeof parent === "string") {
|
|
876
|
+
above = this.#edge(conversationOf(parent), overlay);
|
|
877
|
+
}
|
|
878
|
+
else {
|
|
879
|
+
const fields = this.#settled.get(parent);
|
|
880
|
+
above = fields === undefined ? undefined : parentNode(fields);
|
|
881
|
+
}
|
|
882
|
+
if (above !== undefined && above !== null && this.#below.get(above)?.has(parent) === true)
|
|
506
883
|
break;
|
|
884
|
+
child = parent;
|
|
885
|
+
parent = above;
|
|
507
886
|
}
|
|
508
887
|
}
|
|
509
|
-
|
|
888
|
+
overlay.links = links;
|
|
889
|
+
return links;
|
|
890
|
+
}
|
|
891
|
+
/**
|
|
892
|
+
* The live ordinary owned work of `owner` (spec §5.5): the live non-background tasks below it that a walk down reaches
|
|
893
|
+
* without entering a background task; with `first`, only the first. Candidates in `overlay` replace committed records;
|
|
894
|
+
* the walk does not enter tasks in `done`, which have none. Owner chains must be loaded.
|
|
895
|
+
*/
|
|
896
|
+
#ownedLive(owner, options = {}) {
|
|
897
|
+
const { overlay, first = false, done } = options;
|
|
898
|
+
const found = [];
|
|
899
|
+
const walk = this.#walkDown(owner, {
|
|
900
|
+
...(overlay === undefined ? {} : { overlay }),
|
|
901
|
+
...(done === undefined ? {} : { enter: (id) => !done.has(id) }),
|
|
902
|
+
});
|
|
903
|
+
for (const { id, node, live } of walk) {
|
|
904
|
+
if (live === undefined || node.background)
|
|
905
|
+
continue;
|
|
906
|
+
found.push(id);
|
|
907
|
+
if (first)
|
|
908
|
+
break;
|
|
909
|
+
}
|
|
910
|
+
return found;
|
|
911
|
+
}
|
|
912
|
+
#hasOwnedLive(owner, overlay, done) {
|
|
913
|
+
const options = {
|
|
914
|
+
first: true,
|
|
915
|
+
...(overlay === undefined ? {} : { overlay }),
|
|
916
|
+
...(done === undefined ? {} : { done }),
|
|
917
|
+
};
|
|
918
|
+
return this.#ownedLive(owner, options).length > 0;
|
|
510
919
|
}
|
|
511
920
|
/**
|
|
512
921
|
* Whether ordinary traversal from `scope` reaches `start`: walking up reaches the scope's conversation, or an
|
|
@@ -532,28 +941,47 @@ export class TaskScheduler {
|
|
|
532
941
|
* background owner without it. Terminal owners never cascade (spec §5.4).
|
|
533
942
|
*/
|
|
534
943
|
#belowCancelled(start) {
|
|
944
|
+
return this.#cancellingOwner(start) !== undefined;
|
|
945
|
+
}
|
|
946
|
+
/** The nearest live owner above `start` whose cancellation intent reaches it, if any; see `#belowCancelled`. */
|
|
947
|
+
#cancellingOwner(start) {
|
|
948
|
+
// Nothing cancels, the common case: no walk.
|
|
949
|
+
if (this.#intent.size === 0)
|
|
950
|
+
return undefined;
|
|
535
951
|
for (const step of this.#above(start)) {
|
|
536
952
|
if ("unknown" in step)
|
|
537
|
-
return
|
|
953
|
+
return undefined;
|
|
538
954
|
if (!("task" in step))
|
|
539
955
|
continue;
|
|
540
956
|
const live = this.#live.get(step.task);
|
|
541
957
|
if (live !== undefined && cancellationIntent(live))
|
|
542
|
-
return
|
|
958
|
+
return live;
|
|
543
959
|
if (step.node.background)
|
|
544
|
-
return
|
|
960
|
+
return undefined;
|
|
545
961
|
}
|
|
546
|
-
return
|
|
962
|
+
return undefined;
|
|
963
|
+
}
|
|
964
|
+
/** Fail the Session, which reports it, for a throw in one of the scheduler's own commits; ignored once closing. */
|
|
965
|
+
#failSession(error) {
|
|
966
|
+
if (!this.#closing)
|
|
967
|
+
this.#session.fail(error);
|
|
547
968
|
}
|
|
548
|
-
/** Close listener: runs synchronously once admission is sealed, before `join()
|
|
969
|
+
/** Close listener: runs synchronously once admission is sealed, before `join()`, or when the Session fails. */
|
|
549
970
|
#seal() {
|
|
550
971
|
this.#closing = true;
|
|
551
|
-
|
|
552
|
-
|
|
972
|
+
try {
|
|
973
|
+
this.#unsubscribeRegistry();
|
|
974
|
+
}
|
|
975
|
+
catch (error) {
|
|
976
|
+
this.#report(error);
|
|
977
|
+
}
|
|
978
|
+
const error = closedError(this.#session);
|
|
553
979
|
this.#taskWaiters.rejectAll(error);
|
|
554
980
|
this.#idleWaiters.rejectAll(error);
|
|
555
|
-
|
|
556
|
-
|
|
981
|
+
this.#contexts.clear();
|
|
982
|
+
if (this.#expiry !== undefined)
|
|
983
|
+
clearTimeout(this.#expiry.timer);
|
|
984
|
+
this.#expiry = undefined;
|
|
557
985
|
}
|
|
558
986
|
#kick() {
|
|
559
987
|
this.#dirty = true;
|
|
@@ -572,38 +1000,64 @@ export class TaskScheduler {
|
|
|
572
1000
|
}
|
|
573
1001
|
}
|
|
574
1002
|
catch (error) {
|
|
575
|
-
|
|
576
|
-
|
|
1003
|
+
// As in `#reconcile()`: the reservation commit runs no extension code.
|
|
1004
|
+
this.#failSession(error);
|
|
577
1005
|
}
|
|
578
1006
|
finally {
|
|
579
1007
|
this.#draining = false;
|
|
580
|
-
// A wakeup that arrived
|
|
1008
|
+
// A wakeup that arrived after the loop's last check still needs its pass.
|
|
581
1009
|
if (this.#dirty)
|
|
582
1010
|
this.#kick();
|
|
583
1011
|
}
|
|
584
1012
|
}
|
|
585
|
-
/**
|
|
1013
|
+
/**
|
|
1014
|
+
* Reserve every eligible task in one commit; orphan abort-marked tasks no definition can take, unless abandoned after
|
|
1015
|
+
* a restart. The first pass after open only abort-marks the abandoned tasks, so later passes see the marks.
|
|
1016
|
+
*/
|
|
586
1017
|
async #reserve() {
|
|
587
1018
|
const reservations = [];
|
|
1019
|
+
let abandoned = [];
|
|
588
1020
|
try {
|
|
589
1021
|
await this.#session.commitWith(async (tx) => {
|
|
590
1022
|
if (!this.#enabled || this.#closing)
|
|
591
1023
|
return;
|
|
1024
|
+
for (const id of this.#abandoned) {
|
|
1025
|
+
const record = this.#live.get(id);
|
|
1026
|
+
if (record !== undefined && !record.abortRequested)
|
|
1027
|
+
tx.setTask(withAbortMark(record, "restart"));
|
|
1028
|
+
}
|
|
1029
|
+
if (this.#abandoned.size > 0) {
|
|
1030
|
+
abandoned = [...this.#abandoned];
|
|
1031
|
+
return;
|
|
1032
|
+
}
|
|
592
1033
|
await this.#loadScopes(false);
|
|
593
|
-
const owned = this.#ownedLive();
|
|
594
1034
|
// Taken once per pass, and only when some task is a candidate.
|
|
595
1035
|
let snapshot;
|
|
596
|
-
for (const
|
|
597
|
-
|
|
1036
|
+
for (const id of [...this.#runnable]) {
|
|
1037
|
+
const record = this.#live.get(id);
|
|
1038
|
+
if (record === undefined || record.state.status === "completing" || this.#invocations.has(id)) {
|
|
1039
|
+
this.#runnable.delete(id);
|
|
598
1040
|
continue;
|
|
599
|
-
|
|
1041
|
+
}
|
|
1042
|
+
// It returns once what it waits for ends, its record changes, or the work below it ends.
|
|
1043
|
+
if (this.#waitingOn(record, true).length > 0) {
|
|
1044
|
+
this.#runnable.delete(id);
|
|
600
1045
|
continue;
|
|
1046
|
+
}
|
|
601
1047
|
const runnable = record;
|
|
602
1048
|
const mode = record.abortRequested ? "abort" : "run";
|
|
1049
|
+
// Work below an owner with cancellation intent waits for its cascade mark instead of running a phase; schedule
|
|
1050
|
+
// the cascade, which may not have run yet.
|
|
1051
|
+
if (mode === "run" && !record.background && this.#belowCancelled(parentOf(record))) {
|
|
1052
|
+
this.#cascadePending = true;
|
|
1053
|
+
this.#scheduleReconcile();
|
|
1054
|
+
continue;
|
|
1055
|
+
}
|
|
603
1056
|
snapshot ??= this.#registry.snapshot();
|
|
604
1057
|
const resolution = this.#resolve(runnable, snapshot);
|
|
605
1058
|
if (resolution.kind === "blocked") {
|
|
606
|
-
|
|
1059
|
+
// An abandoned task waits for its definition, so its abort handler can clean up.
|
|
1060
|
+
if (mode === "abort" && record.abortReason === undefined) {
|
|
607
1061
|
await this.#terminate(tx, record, { status: "orphaned", reason: resolution.reason });
|
|
608
1062
|
}
|
|
609
1063
|
continue;
|
|
@@ -614,7 +1068,9 @@ export class TaskScheduler {
|
|
|
614
1068
|
checkpoint: resolution.record.state.checkpoint,
|
|
615
1069
|
}));
|
|
616
1070
|
}
|
|
617
|
-
// Registered on the line, so marks and later reservations see it and close joins it
|
|
1071
|
+
// Registered on the line, so marks and later reservations see it and close joins it; `#end` makes it a
|
|
1072
|
+
// candidate again.
|
|
1073
|
+
this.#runnable.delete(id);
|
|
618
1074
|
const invocation = this.#createInvocation(record, mode);
|
|
619
1075
|
reservations.push({ invocation, task: resolution.task, snapshot });
|
|
620
1076
|
}
|
|
@@ -623,22 +1079,31 @@ export class TaskScheduler {
|
|
|
623
1079
|
catch (error) {
|
|
624
1080
|
for (const { invocation } of reservations) {
|
|
625
1081
|
this.#invocations.delete(invocation.taskId);
|
|
1082
|
+
this.#runnable.add(invocation.taskId);
|
|
626
1083
|
invocation.finish();
|
|
627
1084
|
}
|
|
628
1085
|
throw error;
|
|
629
1086
|
}
|
|
1087
|
+
// Marked, or already marked or gone: done with them. Reserve again, now seeing the marks.
|
|
1088
|
+
for (const id of abandoned)
|
|
1089
|
+
this.#abandoned.delete(id);
|
|
1090
|
+
if (abandoned.length > 0)
|
|
1091
|
+
this.#dirty = true;
|
|
630
1092
|
return reservations;
|
|
631
1093
|
}
|
|
632
1094
|
/**
|
|
633
1095
|
* Live tasks a task waits for before its next invocation: its live ordinary owned work when abort-marked, since
|
|
634
1096
|
* abort runs bottom-up, otherwise the live part of the `on` of a wait.
|
|
635
1097
|
*/
|
|
636
|
-
#waitingOn(record,
|
|
1098
|
+
#waitingOn(record, first = false) {
|
|
637
1099
|
if (record.abortRequested)
|
|
638
|
-
return
|
|
1100
|
+
return this.#ownedLive(record.id, { first });
|
|
639
1101
|
if (record.state.status !== "waiting")
|
|
640
1102
|
return [];
|
|
641
|
-
|
|
1103
|
+
if (!first)
|
|
1104
|
+
return record.state.on.filter((id) => this.#live.has(id));
|
|
1105
|
+
const live = record.state.on.find((id) => this.#live.has(id));
|
|
1106
|
+
return live === undefined ? [] : [live];
|
|
642
1107
|
}
|
|
643
1108
|
/** Resolve the record's definition by kind, migrating an older stored version. */
|
|
644
1109
|
#resolve(record, snapshot) {
|
|
@@ -675,6 +1140,21 @@ export class TaskScheduler {
|
|
|
675
1140
|
return { reason: "migration_failed", error: failed.error };
|
|
676
1141
|
return { task, migrates: true };
|
|
677
1142
|
}
|
|
1143
|
+
/** Sizes of the ownership indexes, for tests that bound the scheduler's memory. */
|
|
1144
|
+
get indexSizes() {
|
|
1145
|
+
let below = 0;
|
|
1146
|
+
for (const children of this.#below.values())
|
|
1147
|
+
below += children.size;
|
|
1148
|
+
return {
|
|
1149
|
+
live: this.#live.size,
|
|
1150
|
+
settled: this.#settled.size,
|
|
1151
|
+
edges: this.#edges.size,
|
|
1152
|
+
below,
|
|
1153
|
+
roots: this.#roots.size,
|
|
1154
|
+
unloaded: this.#unloaded.size,
|
|
1155
|
+
dropQueue: this.#dropQueue.size,
|
|
1156
|
+
};
|
|
1157
|
+
}
|
|
678
1158
|
/**
|
|
679
1159
|
* Scheduling state and every live task with its derived state, read on the Session line. Runs no task code: a
|
|
680
1160
|
* pending migration shows as `ready` with `migrates`, and only a migration the scheduler already tried, or one that
|
|
@@ -682,20 +1162,18 @@ export class TaskScheduler {
|
|
|
682
1162
|
*/
|
|
683
1163
|
async inspect(snapshot) {
|
|
684
1164
|
await this.#loadScopes(false);
|
|
685
|
-
const owned = this.#ownedLive();
|
|
686
1165
|
const tasks = [];
|
|
687
|
-
for (const record of this.#live.values())
|
|
688
|
-
tasks.push({ record, state: this.#inspectTask(record, snapshot
|
|
689
|
-
}
|
|
1166
|
+
for (const record of this.#live.values())
|
|
1167
|
+
tasks.push({ record, state: this.#inspectTask(record, snapshot) });
|
|
690
1168
|
const scheduling = this.#closing ? "closing" : this.#enabled ? "running" : "paused";
|
|
691
1169
|
return { scheduling, tasks };
|
|
692
1170
|
}
|
|
693
|
-
#inspectTask(record, snapshot
|
|
1171
|
+
#inspectTask(record, snapshot) {
|
|
694
1172
|
if (this.#invocations.has(record.id))
|
|
695
1173
|
return { kind: "running" };
|
|
696
1174
|
if (record.state.status === "completing")
|
|
697
1175
|
return { kind: "completing" };
|
|
698
|
-
const on = this.#waitingOn(record
|
|
1176
|
+
const on = this.#waitingOn(record);
|
|
699
1177
|
if (on.length > 0)
|
|
700
1178
|
return { kind: "waiting", on };
|
|
701
1179
|
const fit = this.#fit(record, snapshot.task(record.kind));
|
|
@@ -774,6 +1252,12 @@ export class TaskScheduler {
|
|
|
774
1252
|
// 3. abort mark: end; a fresh abort invocation starts once the task's ordinary owned work is gone.
|
|
775
1253
|
if (current.abortRequested)
|
|
776
1254
|
return false;
|
|
1255
|
+
// An owner's cancellation intent ends it too, before its cascade marks it; reservation then holds it back.
|
|
1256
|
+
if (!current.background && this.#belowCancelled(parentOf(current))) {
|
|
1257
|
+
this.#cascadePending = true;
|
|
1258
|
+
this.#scheduleReconcile();
|
|
1259
|
+
return false;
|
|
1260
|
+
}
|
|
777
1261
|
if (previous === undefined)
|
|
778
1262
|
return true;
|
|
779
1263
|
// 4. uncaught error.
|
|
@@ -834,16 +1318,17 @@ export class TaskScheduler {
|
|
|
834
1318
|
return current;
|
|
835
1319
|
this.#end(invocation);
|
|
836
1320
|
if (decision !== false) {
|
|
837
|
-
const message =
|
|
1321
|
+
const message = errorMessage(decision.fault);
|
|
838
1322
|
await this.#terminate(tx, current, { status: "faulted", error: { message } });
|
|
839
1323
|
}
|
|
840
1324
|
return undefined;
|
|
841
1325
|
}, this.#context);
|
|
842
1326
|
}
|
|
843
1327
|
catch (error) {
|
|
1328
|
+
// The step writes the scheduler's own decision, a fault or a terminal record; a throw there would otherwise
|
|
1329
|
+
// leave the task running, to be reserved and run again.
|
|
844
1330
|
this.#end(invocation);
|
|
845
|
-
|
|
846
|
-
this.#report(error);
|
|
1331
|
+
this.#failSession(error);
|
|
847
1332
|
return undefined;
|
|
848
1333
|
}
|
|
849
1334
|
}
|
|
@@ -853,7 +1338,7 @@ export class TaskScheduler {
|
|
|
853
1338
|
*/
|
|
854
1339
|
async #terminate(tx, record, outcome) {
|
|
855
1340
|
await this.#loadScopes(false);
|
|
856
|
-
if (this.#
|
|
1341
|
+
if (this.#hasOwnedLive(record.id, overlayOf(tx))) {
|
|
857
1342
|
tx.setTask(withState(record, { status: "completing", outcome }));
|
|
858
1343
|
return;
|
|
859
1344
|
}
|
|
@@ -873,7 +1358,7 @@ export class TaskScheduler {
|
|
|
873
1358
|
await this.#loadScopes(false);
|
|
874
1359
|
for (const record of overlay.tasks.values())
|
|
875
1360
|
await this.#loadChain(parentOf(record), overlay);
|
|
876
|
-
if (this.#
|
|
1361
|
+
if (this.#hasOwnedLive(current.id, overlay)) {
|
|
877
1362
|
tx.setTask(withState(current, { status: "completing", outcome: next.outcome }));
|
|
878
1363
|
return;
|
|
879
1364
|
}
|
|
@@ -881,37 +1366,102 @@ export class TaskScheduler {
|
|
|
881
1366
|
tx.setTask(withState(current, next));
|
|
882
1367
|
}
|
|
883
1368
|
/**
|
|
884
|
-
* A wait names existing tasks
|
|
885
|
-
*
|
|
1369
|
+
* A wait names existing tasks, and `failFast` only tasks the waiter owns. It may not close a cycle in the wait-for
|
|
1370
|
+
* graph, whose edges run from each task to what it waits on and to its live ordinary owned work (spec §5.5); waiting
|
|
1371
|
+
* on itself or on an owner holding it are the shortest such cycles. An abort handler cannot wait.
|
|
1372
|
+
*
|
|
1373
|
+
* Invariant: the wait-for graph is acyclic. Only a wait adds an edge between existing tasks, as a new task has only
|
|
1374
|
+
* incoming edges, so checking each wait maintains it.
|
|
886
1375
|
*/
|
|
887
1376
|
async #validateWait(tx, invocation, current, on, policy) {
|
|
888
1377
|
if (invocation.mode === "abort")
|
|
889
1378
|
throw new Error(`Abort handler of task ${current.id} cannot wait`);
|
|
890
1379
|
const overlay = overlayOf(tx);
|
|
891
|
-
|
|
892
|
-
const owners = new Set();
|
|
893
|
-
for (const step of this.#above(parentOf(current)))
|
|
894
|
-
if ("task" in step)
|
|
895
|
-
owners.add(step.task);
|
|
1380
|
+
const targets = [];
|
|
896
1381
|
for (const id of on) {
|
|
897
|
-
if (id === current.id || owners.has(id)) {
|
|
898
|
-
throw new Error(`Task ${current.id} cannot wait on itself or its owner ${id}`);
|
|
899
|
-
}
|
|
900
1382
|
const member = overlay.tasks.get(id) ?? this.#live.get(id) ?? (await this.#storage.task(id, this.#context));
|
|
901
1383
|
if (member === undefined)
|
|
902
1384
|
throw new Error(`Task ${id} does not exist`);
|
|
903
1385
|
if (policy === "failFast" && member.owner !== current.id) {
|
|
904
1386
|
throw new Error(`Task ${current.id} can wait failFast only on tasks it owns; ${id} is not one`);
|
|
905
1387
|
}
|
|
1388
|
+
// A live child already holds the waiter (task-owned tasks are never background): waiting on it adds no edge. A
|
|
1389
|
+
// terminal task has no edges.
|
|
1390
|
+
if (member.state.status !== "terminal" && member.owner !== current.id)
|
|
1391
|
+
targets.push(id);
|
|
1392
|
+
}
|
|
1393
|
+
if (targets.length === 0)
|
|
1394
|
+
return;
|
|
1395
|
+
const cycle = await this.#waitCycle(current.id, targets, overlay);
|
|
1396
|
+
if (cycle !== undefined) {
|
|
1397
|
+
throw new Error(`Task ${current.id} cannot wait on ${cycle[0]}: ${[current.id, ...cycle].join(" -> ")} is a wait cycle`);
|
|
1398
|
+
}
|
|
1399
|
+
}
|
|
1400
|
+
/**
|
|
1401
|
+
* The path from a member of `on` back to `task` in the wait-for graph, if the wait would close a cycle (spec §5.1).
|
|
1402
|
+
* An edge X -> Y means X cannot settle before Y: Y is in the `on` of waiting X that is not abort-marked, or Y is live
|
|
1403
|
+
* ordinary owned work of X,
|
|
1404
|
+
* which holds X as `completing`. Each task is visited once and each subtree walked once, so a check is bounded by the
|
|
1405
|
+
* live work it reaches.
|
|
1406
|
+
*/
|
|
1407
|
+
async #waitCycle(task, on, overlay) {
|
|
1408
|
+
// Owned work is found by walks down, which need the chains of every live task loaded.
|
|
1409
|
+
await this.#loadScopes(false);
|
|
1410
|
+
const from = new Map();
|
|
1411
|
+
const walked = new Set();
|
|
1412
|
+
const pending = [];
|
|
1413
|
+
const reach = (id, via) => {
|
|
1414
|
+
if (from.has(id))
|
|
1415
|
+
return;
|
|
1416
|
+
from.set(id, via);
|
|
1417
|
+
pending.push(id);
|
|
1418
|
+
};
|
|
1419
|
+
const path = (end) => {
|
|
1420
|
+
const steps = [];
|
|
1421
|
+
for (let at = end; at !== undefined; at = from.get(at))
|
|
1422
|
+
steps.unshift(at);
|
|
1423
|
+
return steps;
|
|
1424
|
+
};
|
|
1425
|
+
for (const id of on)
|
|
1426
|
+
reach(id, undefined);
|
|
1427
|
+
for (let id = pending.pop(); id !== undefined; id = pending.pop()) {
|
|
1428
|
+
if (id === task)
|
|
1429
|
+
return path(id);
|
|
1430
|
+
const record = overlay.tasks.get(id) ?? this.#live.get(id);
|
|
1431
|
+
if (record === undefined || record.state.status === "terminal")
|
|
1432
|
+
continue;
|
|
1433
|
+
// An abort-marked task waits only for its owned work, as abort runs bottom-up (`#waitingOn`).
|
|
1434
|
+
if (record.state.status === "waiting" && !record.abortRequested) {
|
|
1435
|
+
for (const member of record.state.on)
|
|
1436
|
+
reach(member, id);
|
|
1437
|
+
}
|
|
1438
|
+
if (walked.has(id))
|
|
1439
|
+
continue;
|
|
1440
|
+
walked.add(id);
|
|
1441
|
+
// Below a walked task the walk does not go again: its own walk covered what lies below.
|
|
1442
|
+
const found = [];
|
|
1443
|
+
const walk = this.#walkDown(id, { overlay, enter: (below) => !walked.has(below) });
|
|
1444
|
+
for (const { id: below, node, live } of walk) {
|
|
1445
|
+
if (live === undefined || node.background)
|
|
1446
|
+
continue;
|
|
1447
|
+
found.push(below);
|
|
1448
|
+
reach(below, id);
|
|
1449
|
+
}
|
|
1450
|
+
for (const below of found)
|
|
1451
|
+
walked.add(below);
|
|
906
1452
|
}
|
|
1453
|
+
return undefined;
|
|
907
1454
|
}
|
|
908
1455
|
/** End an invocation: its runtime operations reject from now on, its signal aborts, its watches stop, and its task is free. */
|
|
909
1456
|
#end(invocation) {
|
|
910
1457
|
if (invocation.ended)
|
|
911
1458
|
return;
|
|
912
1459
|
invocation.ended = true;
|
|
913
|
-
if (this.#invocations.get(invocation.taskId) === invocation)
|
|
1460
|
+
if (this.#invocations.get(invocation.taskId) === invocation) {
|
|
914
1461
|
this.#invocations.delete(invocation.taskId);
|
|
1462
|
+
if (this.#live.has(invocation.taskId))
|
|
1463
|
+
this.#runnable.add(invocation.taskId);
|
|
1464
|
+
}
|
|
915
1465
|
for (const watch of invocation.watches)
|
|
916
1466
|
void watch.stop();
|
|
917
1467
|
// Pending waits bound to the invocation, such as a tool's waitForTask(), reject with it.
|
|
@@ -920,10 +1470,20 @@ export class TaskScheduler {
|
|
|
920
1470
|
/** No live non-background task in the scope; a task whose owner edges are not loaded yet counts as inside. */
|
|
921
1471
|
#idle(conversationId) {
|
|
922
1472
|
const scope = conversationId === undefined ? { roots: true } : { conversation: conversationId };
|
|
923
|
-
|
|
924
|
-
|
|
1473
|
+
// Tasks whose chain is not loaded are not listed below their scope yet; one counts as inside unless its walk up
|
|
1474
|
+
// decides otherwise.
|
|
1475
|
+
for (const id of this.#unloaded) {
|
|
1476
|
+
const record = this.#live.get(id);
|
|
1477
|
+
if (record !== undefined && !record.background && this.#inScope(parentOf(record), scope) !== false)
|
|
925
1478
|
return false;
|
|
926
1479
|
}
|
|
1480
|
+
const starts = conversationId === undefined ? this.#roots : [conversationId];
|
|
1481
|
+
for (const start of starts) {
|
|
1482
|
+
for (const { node, live } of this.#walkDown(conversationNode(start))) {
|
|
1483
|
+
if (live !== undefined && !node.background)
|
|
1484
|
+
return false;
|
|
1485
|
+
}
|
|
1486
|
+
}
|
|
927
1487
|
return true;
|
|
928
1488
|
}
|
|
929
1489
|
// ─── Invocation runtime ──────────────────────────────────────────────────
|
|
@@ -995,6 +1555,34 @@ export class TaskScheduler {
|
|
|
995
1555
|
snapshotAsOf: ((...args) => this.#read(invocation, () => sessionMethod(this.#session, "snapshotAsOf")(...args))),
|
|
996
1556
|
getTask: ((id, context) => this.#read(invocation, () => this.#session.readOnLine(() => this.#storage.task(id, context)))),
|
|
997
1557
|
waitForTask: ((id, context) => this.#read(invocation, () => this.waitForTask(id, withAbortSignal(invocation.controller.signal, context)))),
|
|
1558
|
+
ownedTasks: (context) => this.#read(invocation, () => this.#session.readOnLine(async () => {
|
|
1559
|
+
// The step that ends the invocation may have run on the line before this.
|
|
1560
|
+
if (invocation.ended)
|
|
1561
|
+
throw endedError(invocation);
|
|
1562
|
+
context.abortSignal?.throwIfAborted();
|
|
1563
|
+
// Every live task is listed under its parent node, its owner task when it has one.
|
|
1564
|
+
const owned = [];
|
|
1565
|
+
for (const node of this.#below.get(invocation.taskId) ?? []) {
|
|
1566
|
+
const record = typeof node === "string" ? undefined : this.#live.get(node);
|
|
1567
|
+
if (record?.owner === invocation.taskId)
|
|
1568
|
+
owned.push(record);
|
|
1569
|
+
}
|
|
1570
|
+
// Copies, as from storage: the scheduler reads its own.
|
|
1571
|
+
return owned
|
|
1572
|
+
.sort((a, b) => a.id - b.id)
|
|
1573
|
+
.map((record) => copyJson(record));
|
|
1574
|
+
})),
|
|
1575
|
+
abortOwned: (id, context) => this.#read(invocation, async () => {
|
|
1576
|
+
const bound = withAbortSignal(invocation.controller.signal, context);
|
|
1577
|
+
const record = await this.#session.readOnLine(() => this.#storage.task(id, bound));
|
|
1578
|
+
if (record?.owner !== invocation.taskId) {
|
|
1579
|
+
throw new Error(`Task ${id} is not owned by task ${invocation.taskId}`);
|
|
1580
|
+
}
|
|
1581
|
+
if (record.state.status === "terminal")
|
|
1582
|
+
return;
|
|
1583
|
+
await this.abort(id, bound, true);
|
|
1584
|
+
await this.waitForTask(id, bound);
|
|
1585
|
+
}),
|
|
998
1586
|
outcomes: ((ids, context) => this.#read(invocation, () => this.#session.readOnLine(async () => {
|
|
999
1587
|
const outcomes = [];
|
|
1000
1588
|
for (const id of ids) {
|
|
@@ -1022,7 +1610,25 @@ export class TaskScheduler {
|
|
|
1022
1610
|
return token === undefined || entry?.kind === token.kind ? entry : undefined;
|
|
1023
1611
|
});
|
|
1024
1612
|
}),
|
|
1025
|
-
context: (conversationId, context,
|
|
1613
|
+
context: (conversationId, context, options) => this.#read(invocation, async () => {
|
|
1614
|
+
const { view, range } = await readContextFrom(this.#session, this.#storage, conversationId, context, options?.at, this.#contexts.get(conversationId)?.range);
|
|
1615
|
+
// Keep it unless the invocation ended or a concurrent read already kept a newer range.
|
|
1616
|
+
const kept = this.#contexts.get(conversationId);
|
|
1617
|
+
if (range !== undefined &&
|
|
1618
|
+
!this.#closing &&
|
|
1619
|
+
!invocation.ended &&
|
|
1620
|
+
(kept === undefined || kept.range.bounds.tail <= range.bounds.tail)) {
|
|
1621
|
+
// A read of another, idle conversation starts or continues its retention period.
|
|
1622
|
+
if (!this.#idle(conversationId)) {
|
|
1623
|
+
this.#contexts.set(conversationId, { range, idleSince: undefined });
|
|
1624
|
+
}
|
|
1625
|
+
else if (this.#contextRetentionMs() > 0) {
|
|
1626
|
+
this.#contexts.set(conversationId, { range, idleSince: kept?.idleSince ?? this.#now() });
|
|
1627
|
+
this.#scheduleExpiry();
|
|
1628
|
+
}
|
|
1629
|
+
}
|
|
1630
|
+
return view;
|
|
1631
|
+
}),
|
|
1026
1632
|
now: () => {
|
|
1027
1633
|
if (invocation.ended)
|
|
1028
1634
|
throw endedError(invocation);
|
|
@@ -1049,7 +1655,7 @@ export class TaskScheduler {
|
|
|
1049
1655
|
if (invocation.ended)
|
|
1050
1656
|
throw endedError(invocation);
|
|
1051
1657
|
if (this.#closing)
|
|
1052
|
-
throw closedError();
|
|
1658
|
+
throw closedError(this.#session);
|
|
1053
1659
|
const found = this.#live.get(invocation.taskId);
|
|
1054
1660
|
if (found === undefined)
|
|
1055
1661
|
throw new Error(`Task ${invocation.taskId} is terminal`);
|
|
@@ -1093,6 +1699,11 @@ export class TaskScheduler {
|
|
|
1093
1699
|
return watch;
|
|
1094
1700
|
}
|
|
1095
1701
|
}
|
|
1702
|
+
/** The record with an abort mark: `restart` for an abandonment; without a reason, a request, which replaces `restart`. */
|
|
1703
|
+
function withAbortMark(record, reason) {
|
|
1704
|
+
const { abortReason: _reason, ...rest } = record;
|
|
1705
|
+
return { ...rest, abortRequested: true, ...(reason === undefined ? {} : { abortReason: reason }) };
|
|
1706
|
+
}
|
|
1096
1707
|
/** A live owner's durable cancellation intent: its abort mark, or a held outcome other than `completed`. */
|
|
1097
1708
|
function cancellationIntent(record) {
|
|
1098
1709
|
return record.state.status !== "terminal" && (record.abortRequested || failedOutcome(record));
|
|
@@ -1117,6 +1728,21 @@ function overlayOf(tx) {
|
|
|
1117
1728
|
const edges = new Map(tx.stagedConversations().map((record) => [record.id, record.owner?.taskId ?? null]));
|
|
1118
1729
|
return { tasks, edges };
|
|
1119
1730
|
}
|
|
1731
|
+
function addTo(map, key, value) {
|
|
1732
|
+
const values = map.get(key);
|
|
1733
|
+
if (values === undefined)
|
|
1734
|
+
map.set(key, new Set([value]));
|
|
1735
|
+
else
|
|
1736
|
+
values.add(value);
|
|
1737
|
+
}
|
|
1738
|
+
function deleteFrom(map, key, value) {
|
|
1739
|
+
const values = map.get(key);
|
|
1740
|
+
if (values === undefined)
|
|
1741
|
+
return;
|
|
1742
|
+
values.delete(value);
|
|
1743
|
+
if (values.size === 0)
|
|
1744
|
+
map.delete(key);
|
|
1745
|
+
}
|
|
1120
1746
|
/** Own memo entry only; memo names such as `toString` must not resolve to inherited properties. */
|
|
1121
1747
|
function memoOf(record, name) {
|
|
1122
1748
|
const memos = record?.memos;
|