@earendil-works/pi-durable 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +55 -0
- package/README.md +164 -11
- package/dist/codemode/codemode.d.ts +173 -0
- package/dist/codemode/codemode.d.ts.map +1 -0
- package/dist/codemode/codemode.js +886 -0
- package/dist/codemode/codemode.js.map +1 -0
- package/dist/codemode/describe.d.ts +38 -0
- package/dist/codemode/describe.d.ts.map +1 -0
- package/dist/codemode/describe.js +119 -0
- package/dist/codemode/describe.js.map +1 -0
- package/dist/codemode/index.d.ts +3 -0
- package/dist/codemode/index.d.ts.map +1 -0
- package/dist/codemode/index.js +3 -0
- package/dist/codemode/index.js.map +1 -0
- package/dist/codemode/models.d.ts +28 -0
- package/dist/codemode/models.d.ts.map +1 -0
- package/dist/codemode/models.js +231 -0
- package/dist/codemode/models.js.map +1 -0
- package/dist/codemode/output.d.ts +41 -0
- package/dist/codemode/output.d.ts.map +1 -0
- package/dist/codemode/output.js +167 -0
- package/dist/codemode/output.js.map +1 -0
- package/dist/env/index.d.ts +43 -0
- package/dist/env/index.d.ts.map +1 -1
- package/dist/env/index.js.map +1 -1
- package/dist/errors.d.ts +20 -3
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +34 -5
- package/dist/errors.js.map +1 -1
- package/dist/harness/agent.d.ts +5 -2
- package/dist/harness/agent.d.ts.map +1 -1
- package/dist/harness/agent.js +69 -25
- package/dist/harness/agent.js.map +1 -1
- package/dist/harness/compaction.d.ts +6 -1
- package/dist/harness/compaction.d.ts.map +1 -1
- package/dist/harness/compaction.js +10 -7
- package/dist/harness/compaction.js.map +1 -1
- package/dist/harness/context.d.ts +5 -0
- package/dist/harness/context.d.ts.map +1 -1
- package/dist/harness/context.js +46 -12
- package/dist/harness/context.js.map +1 -1
- package/dist/harness/events.d.ts +35 -15
- package/dist/harness/events.d.ts.map +1 -1
- package/dist/harness/events.js +77 -23
- package/dist/harness/events.js.map +1 -1
- package/dist/harness/generation.d.ts +2 -0
- package/dist/harness/generation.d.ts.map +1 -1
- package/dist/harness/generation.js +31 -8
- package/dist/harness/generation.js.map +1 -1
- package/dist/harness/harness.d.ts +7 -0
- package/dist/harness/harness.d.ts.map +1 -1
- package/dist/harness/harness.js +53 -26
- package/dist/harness/harness.js.map +1 -1
- package/dist/harness/inbox.d.ts +16 -4
- package/dist/harness/inbox.d.ts.map +1 -1
- package/dist/harness/inbox.js +26 -3
- package/dist/harness/inbox.js.map +1 -1
- package/dist/harness/live.d.ts +74 -17
- package/dist/harness/live.d.ts.map +1 -1
- package/dist/harness/live.js +66 -14
- package/dist/harness/live.js.map +1 -1
- package/dist/harness/scheduler.d.ts +18 -5
- package/dist/harness/scheduler.d.ts.map +1 -1
- package/dist/harness/scheduler.js +696 -163
- package/dist/harness/scheduler.js.map +1 -1
- package/dist/harness/submissions.d.ts +7 -4
- package/dist/harness/submissions.d.ts.map +1 -1
- package/dist/harness/submissions.js +32 -13
- package/dist/harness/submissions.js.map +1 -1
- package/dist/harness/task-graph.d.ts +1 -1
- package/dist/harness/task-graph.d.ts.map +1 -1
- package/dist/harness/task-graph.js +15 -10
- package/dist/harness/task-graph.js.map +1 -1
- package/dist/harness/tool.d.ts +45 -5
- package/dist/harness/tool.d.ts.map +1 -1
- package/dist/harness/tool.js +378 -63
- package/dist/harness/tool.js.map +1 -1
- package/dist/harness/types.d.ts +153 -13
- package/dist/harness/types.d.ts.map +1 -1
- package/dist/harness/types.js.map +1 -1
- package/dist/harness/util.d.ts +6 -1
- package/dist/harness/util.d.ts.map +1 -1
- package/dist/harness/util.js +5 -2
- package/dist/harness/util.js.map +1 -1
- package/dist/harness/view.d.ts +6 -2
- package/dist/harness/view.d.ts.map +1 -1
- package/dist/harness/view.js +22 -12
- package/dist/harness/view.js.map +1 -1
- package/dist/images/cloudflare.d.ts +4 -0
- package/dist/images/cloudflare.d.ts.map +1 -0
- package/dist/images/cloudflare.js +9 -0
- package/dist/images/cloudflare.js.map +1 -0
- package/dist/images/exif.d.ts +3 -0
- package/dist/images/exif.d.ts.map +1 -0
- package/dist/images/exif.js +71 -0
- package/dist/images/exif.js.map +1 -0
- package/dist/images/index.d.ts +18 -0
- package/dist/images/index.d.ts.map +1 -0
- package/dist/images/index.js +125 -0
- package/dist/images/index.js.map +1 -0
- package/dist/images/node-worker.d.ts +17 -0
- package/dist/images/node-worker.d.ts.map +1 -0
- package/dist/images/node-worker.js +16 -0
- package/dist/images/node-worker.js.map +1 -0
- package/dist/images/node.d.ts +21 -0
- package/dist/images/node.d.ts.map +1 -0
- package/dist/images/node.js +121 -0
- package/dist/images/node.js.map +1 -0
- package/dist/index.d.ts +6 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -3
- package/dist/index.js.map +1 -1
- package/dist/search.d.ts +13 -0
- package/dist/search.d.ts.map +1 -0
- package/dist/search.js +85 -0
- package/dist/search.js.map +1 -0
- package/dist/session/observation.d.ts +10 -3
- package/dist/session/observation.d.ts.map +1 -1
- package/dist/session/observation.js +24 -34
- package/dist/session/observation.js.map +1 -1
- package/dist/session/session.d.ts +68 -5
- package/dist/session/session.d.ts.map +1 -1
- package/dist/session/session.js +461 -49
- package/dist/session/session.js.map +1 -1
- package/dist/session/transaction.d.ts.map +1 -1
- package/dist/session/transaction.js +1 -0
- package/dist/session/transaction.js.map +1 -1
- package/dist/storage/jsonl/storage.d.ts +4 -0
- package/dist/storage/jsonl/storage.d.ts.map +1 -1
- package/dist/storage/jsonl/storage.js +2 -1
- package/dist/storage/jsonl/storage.js.map +1 -1
- package/dist/storage/memory.d.ts.map +1 -1
- package/dist/storage/memory.js +8 -10
- package/dist/storage/memory.js.map +1 -1
- package/dist/storage/scan.d.ts.map +1 -1
- package/dist/storage/scan.js +5 -4
- package/dist/storage/scan.js.map +1 -1
- package/dist/storage/sqlite/cloudflare.d.ts +17 -5
- package/dist/storage/sqlite/cloudflare.d.ts.map +1 -1
- package/dist/storage/sqlite/cloudflare.js +57 -21
- package/dist/storage/sqlite/cloudflare.js.map +1 -1
- package/dist/storage/sqlite/index.d.ts +1 -1
- package/dist/storage/sqlite/index.d.ts.map +1 -1
- package/dist/storage/sqlite/index.js.map +1 -1
- package/dist/storage/sqlite/migrations.d.ts +5 -1
- package/dist/storage/sqlite/migrations.d.ts.map +1 -1
- package/dist/storage/sqlite/migrations.js +94 -84
- package/dist/storage/sqlite/migrations.js.map +1 -1
- package/dist/storage/sqlite/node.d.ts +6 -4
- package/dist/storage/sqlite/node.d.ts.map +1 -1
- package/dist/storage/sqlite/node.js +6 -2
- package/dist/storage/sqlite/node.js.map +1 -1
- package/dist/storage/sqlite/schema.d.ts +47 -0
- package/dist/storage/sqlite/schema.d.ts.map +1 -0
- package/dist/storage/sqlite/schema.js +57 -0
- package/dist/storage/sqlite/schema.js.map +1 -0
- package/dist/storage/sqlite/storage.d.ts +9 -1
- package/dist/storage/sqlite/storage.d.ts.map +1 -1
- package/dist/storage/sqlite/storage.js +56 -55
- package/dist/storage/sqlite/storage.js.map +1 -1
- package/dist/testing/env-conformance.d.ts.map +1 -1
- package/dist/testing/env-conformance.js +100 -0
- package/dist/testing/env-conformance.js.map +1 -1
- package/dist/testing/runner.d.ts +1 -1
- package/dist/testing/runner.d.ts.map +1 -1
- package/dist/testing/runner.js.map +1 -1
- package/dist/testing/storage-conformance.d.ts.map +1 -1
- package/dist/testing/storage-conformance.js +3 -2
- package/dist/testing/storage-conformance.js.map +1 -1
- package/dist/testing/types.d.ts +2 -0
- package/dist/testing/types.d.ts.map +1 -1
- package/dist/testing/types.js.map +1 -1
- package/dist/tool-search/index.d.ts +2 -0
- package/dist/tool-search/index.d.ts.map +1 -0
- package/dist/tool-search/index.js +2 -0
- package/dist/tool-search/index.js.map +1 -0
- package/dist/tool-search/tool-search.d.ts +23 -0
- package/dist/tool-search/tool-search.d.ts.map +1 -0
- package/dist/tool-search/tool-search.js +84 -0
- package/dist/tool-search/tool-search.js.map +1 -0
- package/dist/tools/bash.d.ts +10 -2
- package/dist/tools/bash.d.ts.map +1 -1
- package/dist/tools/bash.js +27 -9
- package/dist/tools/bash.js.map +1 -1
- package/dist/tools/edit.js +1 -1
- package/dist/tools/edit.js.map +1 -1
- package/dist/tools/image-processor.d.ts +50 -0
- package/dist/tools/image-processor.d.ts.map +1 -0
- package/dist/tools/image-processor.js +20 -0
- package/dist/tools/image-processor.js.map +1 -0
- package/dist/tools/image.d.ts +8 -0
- package/dist/tools/image.d.ts.map +1 -1
- package/dist/tools/image.js +47 -0
- package/dist/tools/image.js.map +1 -1
- package/dist/tools/index.d.ts +25 -2
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +13 -5
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/read.d.ts +10 -2
- package/dist/tools/read.d.ts.map +1 -1
- package/dist/tools/read.js +88 -29
- package/dist/tools/read.js.map +1 -1
- package/dist/tools/write.js +1 -1
- package/dist/tools/write.js.map +1 -1
- package/dist/truncate.d.ts +3 -2
- package/dist/truncate.d.ts.map +1 -1
- package/dist/truncate.js +4 -34
- package/dist/truncate.js.map +1 -1
- package/dist/types.d.ts +59 -5
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/package.json +43 -4
- package/dist/harness/output.d.ts +0 -69
- package/dist/harness/output.d.ts.map +0 -1
- package/dist/harness/output.js +0 -318
- package/dist/harness/output.js.map +0 -1
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { copyJson } from "@earendil-works/chord";
|
|
2
2
|
import { awaitWithContext, withAbortSignal } from "@earendil-works/chord/context";
|
|
3
|
+
import { errorMessage } from "../errors.js";
|
|
3
4
|
import { agentHooks } from "./agent.js";
|
|
4
5
|
import { readContextFrom } from "./context.js";
|
|
5
6
|
import { closedError, scanAll, Waiters } from "./util.js";
|
|
@@ -7,6 +8,16 @@ const SCAN_PAGE_SIZE = 256;
|
|
|
7
8
|
/** Longest delay `setTimeout` supports; longer sleeps wait in several steps. */
|
|
8
9
|
const MAX_TIMER_DELAY = 2_147_483_647;
|
|
9
10
|
const LIVE_STATUSES = ["pending", "running", "waiting", "completing"];
|
|
11
|
+
function conversationNode(id) {
|
|
12
|
+
return `c${id}`;
|
|
13
|
+
}
|
|
14
|
+
function conversationOf(node) {
|
|
15
|
+
return Number(node.slice(1));
|
|
16
|
+
}
|
|
17
|
+
/** The node a task hangs from: its owner task, or its conversation. */
|
|
18
|
+
function parentNode(node) {
|
|
19
|
+
return node.owner ?? conversationNode(node.conversationId);
|
|
20
|
+
}
|
|
10
21
|
/**
|
|
11
22
|
* Durable task scheduler of one Harness.
|
|
12
23
|
*
|
|
@@ -14,8 +25,14 @@ const LIVE_STATUSES = ["pending", "running", "waiting", "completing"];
|
|
|
14
25
|
* commit listener updates it on the Session line, so code running on the line reads exactly the committed state from it.
|
|
15
26
|
*
|
|
16
27
|
* Tasks and conversations form one ownership tree (spec §5.5): a task's parent is its owner task, or its conversation;
|
|
17
|
-
* a conversation's parent is its owner task, if any. Walks up that tree decide
|
|
18
|
-
*
|
|
28
|
+
* a conversation's parent is its owner task, if any. Walks up that tree decide what reaches one task: a cancelling
|
|
29
|
+
* owner, a scope. `#below` lists the tree downward around live work, so what lies below one node is a walk down from
|
|
30
|
+
* it: a task's live ordinary owned work, which holds its outcome as `completing` and delays its abort handler, a
|
|
31
|
+
* scope's tasks, a cascade's reach. No pass visits every live task; the sets below hold each pass's candidates.
|
|
32
|
+
*
|
|
33
|
+
* The ownership indexes are bounded by live work: an owner edge and an ended task's ownership fields are kept only while
|
|
34
|
+
* something is listed below them in `#below`, so only the chains above live tasks are known, whole. Work that appears
|
|
35
|
+
* later below a dropped node finds it unknown and loads its chain from storage again.
|
|
19
36
|
*
|
|
20
37
|
* Invariant: every task transition is decided and written by one callback serialized on the Session line. That covers
|
|
21
38
|
* reservation, marks, runtime commits, finalization, and the synchronous step before each phase, which applies the
|
|
@@ -51,18 +68,43 @@ export class TaskScheduler {
|
|
|
51
68
|
#idleWaiters = new Waiters();
|
|
52
69
|
/** Definition whose migration failed per task; retried only once the registry resolves another definition. */
|
|
53
70
|
#failedMigrations = new Map();
|
|
54
|
-
/** Owner task of each
|
|
71
|
+
/** Owner task of each known conversation, `null` when ownerless. */
|
|
55
72
|
#edges = new Map();
|
|
56
|
-
/**
|
|
57
|
-
#conversationOwners = new Set();
|
|
58
|
-
/** Ownership fields of terminal tasks that walks pass through. */
|
|
73
|
+
/** Ownership fields of the known terminal tasks, which walks pass through. */
|
|
59
74
|
#settled = new Map();
|
|
75
|
+
/**
|
|
76
|
+
* Ended tasks and conversations found unneeded, dropped by `#sweep()` at the end of the commit's publication unless
|
|
77
|
+
* needed again by then: a chain load installs nodes before the work below them is listed.
|
|
78
|
+
*/
|
|
79
|
+
#dropQueue = new Set();
|
|
80
|
+
/**
|
|
81
|
+
* The tree downward around live work: each node's child nodes that are live tasks or have nodes listed below them.
|
|
82
|
+
* A conversation's parent is its owner task; ownerless ones with work are in `#roots`.
|
|
83
|
+
*/
|
|
84
|
+
#below = new Map();
|
|
85
|
+
#roots = new Set();
|
|
86
|
+
/** Known tasks whose chain may not be loaded; every other known task's chain is, and stays, loaded. */
|
|
87
|
+
#unloaded = new Set();
|
|
88
|
+
/** Live tasks with cancellation intent, the sources of abort cascades (spec §5.4). */
|
|
89
|
+
#intent = new Set();
|
|
90
|
+
/** Live tasks that are not `completing`, the candidates of a reservation pass; ones found waiting leave it. */
|
|
91
|
+
#runnable = new Set();
|
|
92
|
+
/** `completing` tasks the next finalize pass checks: those that started holding, and holders of ended work. */
|
|
93
|
+
#finalizeChecks = new Set();
|
|
94
|
+
/** Live waiting tasks by each task in their `on`. */
|
|
95
|
+
#waiters = new Map();
|
|
60
96
|
/**
|
|
61
97
|
* `failFast` waiters the next reconcile checks for a failed task in `on`: at open, when they start waiting, and when
|
|
62
98
|
* one of their tasks fails.
|
|
63
99
|
*/
|
|
64
100
|
#failFastChecks = new Set();
|
|
101
|
+
/**
|
|
102
|
+
* Live tasks with `abandonOnRestart` found at open, from an earlier Harness. The first reservation pass abort-marks
|
|
103
|
+
* them with reason `restart`, so nothing of them or below them runs a phase again.
|
|
104
|
+
*/
|
|
105
|
+
#abandoned = new Set();
|
|
65
106
|
#reconcileScheduled = false;
|
|
107
|
+
#sweepScheduled = false;
|
|
66
108
|
#cascadePending = false;
|
|
67
109
|
#unsubscribeRegistry = () => { };
|
|
68
110
|
#enabled = false;
|
|
@@ -86,7 +128,7 @@ export class TaskScheduler {
|
|
|
86
128
|
}
|
|
87
129
|
/** Load live tasks and change surviving `running` tasks back to `pending`. Dispatches nothing. */
|
|
88
130
|
async open(context) {
|
|
89
|
-
this.#session.
|
|
131
|
+
this.#session.observeCommits((publication) => this.#observe(publication));
|
|
90
132
|
this.#session.subscribeClose(() => this.#seal());
|
|
91
133
|
this.#unsubscribeRegistry = this.#registry.subscribe(() => this.#kick());
|
|
92
134
|
await this.#session.commitWith(async (tx) => {
|
|
@@ -97,7 +139,9 @@ export class TaskScheduler {
|
|
|
97
139
|
}
|
|
98
140
|
for (const records of scans) {
|
|
99
141
|
for (const record of records) {
|
|
100
|
-
this.#
|
|
142
|
+
this.#track(undefined, record);
|
|
143
|
+
if (record.abandonOnRestart === true)
|
|
144
|
+
this.#abandoned.add(record.id);
|
|
101
145
|
if (record.state.status === "running") {
|
|
102
146
|
tx.setTask(withState(record, { status: "pending", checkpoint: record.state.checkpoint }));
|
|
103
147
|
}
|
|
@@ -116,16 +160,23 @@ export class TaskScheduler {
|
|
|
116
160
|
this.#enabled = true;
|
|
117
161
|
this.#kick();
|
|
118
162
|
}
|
|
119
|
-
/**
|
|
163
|
+
/**
|
|
164
|
+
* Signal every invocation and wait for them; writes nothing. Close calls this after every close listener has run, so
|
|
165
|
+
* waits, watches, and streams a task holds end with the Session's reason, not as cancelled by its signal.
|
|
166
|
+
*/
|
|
120
167
|
async join() {
|
|
168
|
+
for (const invocation of this.#invocations.values())
|
|
169
|
+
invocation.controller.abort();
|
|
121
170
|
await Promise.allSettled([...this.#invocations.values()].map((invocation) => invocation.done));
|
|
122
171
|
}
|
|
123
172
|
/**
|
|
124
173
|
* Commit the abort mark, or settle a task that no registered definition can take as `orphaned` when nothing it owns
|
|
125
174
|
* is live, then join the run invocation seen on the line; the commit listener signalled it. The abort invocation
|
|
126
|
-
* starts once the task's ordinary owned work is gone. A `completing` task is only marked.
|
|
175
|
+
* starts once the task's ordinary owned work is gone. A `completing` task is only marked. A request replaces a
|
|
176
|
+
* `restart` mark, so a task abandoned after a restart that waits for its definition is orphaned; with `keepRestart`,
|
|
177
|
+
* as for an owner's own cleanup, such a task keeps its mark and waits on.
|
|
127
178
|
*/
|
|
128
|
-
async abort(id, context) {
|
|
179
|
+
async abort(id, context, keepRestart = false) {
|
|
129
180
|
const marked = await this.#session.commitWith(async (tx) => {
|
|
130
181
|
const current = await tx.task(id);
|
|
131
182
|
if (current === undefined)
|
|
@@ -133,9 +184,12 @@ export class TaskScheduler {
|
|
|
133
184
|
if (current.state.status === "terminal")
|
|
134
185
|
return { result: "terminal" };
|
|
135
186
|
const invocation = this.#invocations.get(id);
|
|
187
|
+
// A restart-marked task has no run invocation; it only waits for its abort.
|
|
188
|
+
if (keepRestart && current.abortReason === "restart")
|
|
189
|
+
return { result: "marked" };
|
|
136
190
|
if (invocation === undefined && current.state.status !== "completing") {
|
|
137
191
|
await this.#loadScopes(false);
|
|
138
|
-
if (!this.#
|
|
192
|
+
if (!this.#hasOwnedLive(id)) {
|
|
139
193
|
const resolution = this.#resolve(current, this.#registry.snapshot());
|
|
140
194
|
if (resolution.kind === "blocked") {
|
|
141
195
|
await this.#terminate(tx, current, { status: "orphaned", reason: resolution.reason });
|
|
@@ -143,20 +197,22 @@ export class TaskScheduler {
|
|
|
143
197
|
}
|
|
144
198
|
}
|
|
145
199
|
}
|
|
146
|
-
if (!current.abortRequested)
|
|
147
|
-
tx.setTask(
|
|
200
|
+
if (!current.abortRequested || current.abortReason !== undefined)
|
|
201
|
+
tx.setTask(withAbortMark(current));
|
|
148
202
|
return { result: "marked", run: invocation?.mode === "run" ? invocation : undefined };
|
|
149
203
|
}, context);
|
|
150
|
-
// The commit listener signalled the run; join it.
|
|
151
|
-
|
|
152
|
-
|
|
204
|
+
// The commit listener signalled the run; join it. A run that ignores its signal can outlive a failed Session,
|
|
205
|
+
// which ends the wait instead.
|
|
206
|
+
if (marked.run !== undefined) {
|
|
207
|
+
await awaitWithContext(Promise.race([marked.run.done, this.#session.failed]), context);
|
|
208
|
+
}
|
|
153
209
|
return marked.result;
|
|
154
210
|
}
|
|
155
211
|
async waitForTask(id, context) {
|
|
156
212
|
// Check and register on the line so no terminal publication falls between them.
|
|
157
213
|
const found = await this.#session.readOnLine(async () => {
|
|
158
214
|
if (this.#closing)
|
|
159
|
-
throw closedError();
|
|
215
|
+
throw closedError(this.#session);
|
|
160
216
|
if (this.#live.has(id))
|
|
161
217
|
return { promise: this.#taskWaiters.add(id, context) };
|
|
162
218
|
const record = await this.#storage.task(id, context);
|
|
@@ -172,7 +228,7 @@ export class TaskScheduler {
|
|
|
172
228
|
*/
|
|
173
229
|
waitForIdle(conversationId, context) {
|
|
174
230
|
if (this.#closing)
|
|
175
|
-
return Promise.reject(closedError());
|
|
231
|
+
return Promise.reject(closedError(this.#session));
|
|
176
232
|
if (this.#idle(conversationId))
|
|
177
233
|
return Promise.resolve();
|
|
178
234
|
this.#scheduleReconcile();
|
|
@@ -188,14 +244,13 @@ export class TaskScheduler {
|
|
|
188
244
|
const queued = await this.#loadScopes(true);
|
|
189
245
|
const scope = { conversation: conversationId };
|
|
190
246
|
const reached = [];
|
|
191
|
-
|
|
192
|
-
|
|
247
|
+
const walk = this.#walkDown(conversationNode(conversationId), { crossBackground: background });
|
|
248
|
+
for (const { node, live } of walk) {
|
|
249
|
+
if (live === undefined || (node.background && !background))
|
|
193
250
|
continue;
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
if (!record.abortRequested)
|
|
198
|
-
tx.setTask({ ...record, abortRequested: true });
|
|
251
|
+
reached.push(live.id);
|
|
252
|
+
if (!live.abortRequested || live.abortReason !== undefined)
|
|
253
|
+
tx.setTask(withAbortMark(live));
|
|
199
254
|
}
|
|
200
255
|
for (const id of queued) {
|
|
201
256
|
if (this.#inScope({ conversation: id }, scope, background) === true)
|
|
@@ -213,6 +268,12 @@ export class TaskScheduler {
|
|
|
213
268
|
const updated = [];
|
|
214
269
|
const failed = [];
|
|
215
270
|
let changed = false;
|
|
271
|
+
// Edges first, so the tasks a commit creates in a new conversation hang from it.
|
|
272
|
+
for (const change of publication.changes) {
|
|
273
|
+
if (change.type === "conversation" && !this.#edges.has(change.value.id)) {
|
|
274
|
+
this.#setEdge(change.value.id, change.value.owner?.taskId ?? null);
|
|
275
|
+
}
|
|
276
|
+
}
|
|
216
277
|
for (const change of publication.changes) {
|
|
217
278
|
if (change.type !== "task")
|
|
218
279
|
continue;
|
|
@@ -221,12 +282,10 @@ export class TaskScheduler {
|
|
|
221
282
|
const previous = this.#live.get(record.id);
|
|
222
283
|
if (failedOutcome(record) && (previous === undefined || !failedOutcome(previous)))
|
|
223
284
|
failed.push(record.id);
|
|
285
|
+
this.#track(previous, record);
|
|
224
286
|
if (record.state.status === "terminal") {
|
|
225
|
-
this.#live.delete(record.id);
|
|
226
287
|
this.#failedMigrations.delete(record.id);
|
|
227
288
|
this.#failFastChecks.delete(record.id);
|
|
228
|
-
if (this.#conversationOwners.has(record.id))
|
|
229
|
-
this.#settled.set(record.id, nodeOf(record));
|
|
230
289
|
this.#taskWaiters.resolve(record.id, record);
|
|
231
290
|
// Its owner may finalize now.
|
|
232
291
|
this.#scheduleReconcile();
|
|
@@ -239,6 +298,9 @@ export class TaskScheduler {
|
|
|
239
298
|
if (invocation?.mode === "run")
|
|
240
299
|
invocation.controller.abort();
|
|
241
300
|
}
|
|
301
|
+
// A request replacing a `restart` mark upgrades the marks of the work below.
|
|
302
|
+
if (previous?.abortReason !== undefined && record.abortReason === undefined)
|
|
303
|
+
this.#cascadePending = true;
|
|
242
304
|
const status = record.state.status;
|
|
243
305
|
if (status === "completing" && previous?.state.status !== "completing") {
|
|
244
306
|
if (cancellationIntent(record))
|
|
@@ -249,42 +311,43 @@ export class TaskScheduler {
|
|
|
249
311
|
this.#failFastChecks.add(record.id);
|
|
250
312
|
this.#scheduleReconcile();
|
|
251
313
|
}
|
|
252
|
-
|
|
253
|
-
|
|
314
|
+
// Its owners and place never change: what is above it is checked once, when it becomes live.
|
|
315
|
+
if (previous === undefined)
|
|
316
|
+
updated.push(record);
|
|
254
317
|
}
|
|
255
318
|
for (const id of failed) {
|
|
256
|
-
for (const
|
|
257
|
-
const state =
|
|
258
|
-
if (state
|
|
259
|
-
this.#failFastChecks.add(
|
|
319
|
+
for (const waiter of this.#waiters.get(id) ?? []) {
|
|
320
|
+
const state = this.#live.get(waiter)?.state;
|
|
321
|
+
if (state?.status === "waiting" && state.policy === "failFast") {
|
|
322
|
+
this.#failFastChecks.add(waiter);
|
|
260
323
|
this.#scheduleReconcile();
|
|
261
324
|
}
|
|
262
325
|
}
|
|
263
326
|
}
|
|
264
|
-
for (const change of publication.changes) {
|
|
265
|
-
if (change.type === "conversation" && !this.#edges.has(change.value.id)) {
|
|
266
|
-
this.#setEdge(change.value.id, change.value.owner?.taskId ?? null);
|
|
267
|
-
}
|
|
268
|
-
}
|
|
269
327
|
for (const change of publication.changes) {
|
|
270
328
|
// A queued input below a cancelled owner is withdrawn, even after its cascade.
|
|
271
329
|
if (change.type !== "submission" || change.value.status !== "queued" || change.value.type !== "input")
|
|
272
330
|
continue;
|
|
273
|
-
|
|
274
|
-
if (
|
|
331
|
+
// Only an owner with cancellation intent withdraws it; a later intent's mark runs the cascade then.
|
|
332
|
+
if (this.#intent.size === 0)
|
|
333
|
+
continue;
|
|
334
|
+
const conversation = change.value.conversationId;
|
|
335
|
+
if (!this.#known(conversationNode(conversation)) || this.#belowCancelled({ conversation })) {
|
|
275
336
|
this.#cascadePending = true;
|
|
337
|
+
}
|
|
276
338
|
}
|
|
277
339
|
for (const record of updated) {
|
|
278
340
|
// Work created below a cancelled owner, even after its cascade, is aborted too.
|
|
279
|
-
if (
|
|
341
|
+
if (this.#unloaded.has(record.id))
|
|
280
342
|
this.#scheduleReconcile();
|
|
281
343
|
else if (!record.background && !record.abortRequested && this.#belowCancelled(parentOf(record))) {
|
|
282
344
|
this.#cascadePending = true;
|
|
283
345
|
}
|
|
284
346
|
}
|
|
285
|
-
//
|
|
347
|
+
// A cascade that a reservation or step found pending runs with the next commit of any kind.
|
|
286
348
|
if (this.#cascadePending)
|
|
287
349
|
this.#scheduleReconcile();
|
|
350
|
+
this.#sweep();
|
|
288
351
|
if (!changed)
|
|
289
352
|
return;
|
|
290
353
|
this.#settleIdle();
|
|
@@ -383,17 +446,28 @@ export class TaskScheduler {
|
|
|
383
446
|
if (this.#closing)
|
|
384
447
|
return;
|
|
385
448
|
const queued = await this.#loadScopes(cascade);
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
449
|
+
// A cascade from an abandoned owner passes its `restart` reason on; any other intent marks, or upgrades a
|
|
450
|
+
// `restart` mark, as a request. Collected first, so a request wins over a `restart` mark of the same pass.
|
|
451
|
+
const marks = new Map();
|
|
452
|
+
const mark = (record, reason) => {
|
|
453
|
+
const staged = marks.get(record.id);
|
|
454
|
+
if (staged !== undefined && staged.reason === undefined)
|
|
455
|
+
return;
|
|
456
|
+
if (record.abortRequested && (record.abortReason === undefined || reason !== undefined))
|
|
389
457
|
return;
|
|
390
|
-
|
|
391
|
-
tx.setTask({ ...record, abortRequested: true });
|
|
458
|
+
marks.set(record.id, { record, reason });
|
|
392
459
|
};
|
|
393
|
-
//
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
460
|
+
// Every live foreground task below a live owner with cancellation intent, which is its nearest one: a walk down
|
|
461
|
+
// from each owner with intent does not enter another, whose own walk covers what is below it. Derived again on
|
|
462
|
+
// every pass, as loading edges can reveal a cancelled owner.
|
|
463
|
+
for (const id of this.#intent) {
|
|
464
|
+
const owner = this.#live.get(id);
|
|
465
|
+
// Only an owner abandoned after a restart, and nothing else, passes its reason on.
|
|
466
|
+
const restart = owner.abortReason === "restart" && !failedOutcome(owner);
|
|
467
|
+
for (const { node, live } of this.#walkDown(id, { enter: (below) => !this.#intent.has(below) })) {
|
|
468
|
+
if (live !== undefined && !node.background)
|
|
469
|
+
mark(live, restart ? "restart" : undefined);
|
|
470
|
+
}
|
|
397
471
|
}
|
|
398
472
|
for (const id of checks) {
|
|
399
473
|
const waiter = this.#live.get(id);
|
|
@@ -403,9 +477,11 @@ export class TaskScheduler {
|
|
|
403
477
|
for (const member of waiter.state.on) {
|
|
404
478
|
const record = this.#live.get(member);
|
|
405
479
|
if (record !== undefined && !failedOutcome(record))
|
|
406
|
-
mark(record);
|
|
480
|
+
mark(record, undefined);
|
|
407
481
|
}
|
|
408
482
|
}
|
|
483
|
+
for (const { record, reason } of marks.values())
|
|
484
|
+
tx.setTask(withAbortMark(record, reason));
|
|
409
485
|
for (const id of queued)
|
|
410
486
|
if (this.#belowCancelled({ conversation: id }))
|
|
411
487
|
await this.#withdrawInputs(tx, id);
|
|
@@ -413,12 +489,9 @@ export class TaskScheduler {
|
|
|
413
489
|
}, this.#context);
|
|
414
490
|
}
|
|
415
491
|
catch (error) {
|
|
416
|
-
//
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
this.#failFastChecks.add(id);
|
|
420
|
-
if (!this.#closing)
|
|
421
|
-
this.#report(error);
|
|
492
|
+
// No extension code runs in this commit: a throw is a storage failure, a host callback, or a bug. None is fixed by
|
|
493
|
+
// running the pass again, so it fails the Session, which reports it.
|
|
494
|
+
this.#failSession(error);
|
|
422
495
|
}
|
|
423
496
|
this.#settleIdle();
|
|
424
497
|
}
|
|
@@ -437,53 +510,86 @@ export class TaskScheduler {
|
|
|
437
510
|
* Harness cleanup here.
|
|
438
511
|
*/
|
|
439
512
|
async #finalize(tx) {
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
513
|
+
// A worklist: the tasks that started holding, and the holders of work that ended. Finalizing one can free the
|
|
514
|
+
// nearest live task above it, which goes on the list. Tasks finalized here have no live work below them, so walks
|
|
515
|
+
// down skip them.
|
|
516
|
+
// In rounds: the holders the tasks of one round free are checked once, in the next, so a holder of many tasks that
|
|
517
|
+
// end together is walked once, not once per task.
|
|
518
|
+
let round = [...this.#finalizeChecks];
|
|
519
|
+
this.#finalizeChecks.clear();
|
|
520
|
+
const done = new Set();
|
|
521
|
+
const overlay = overlayOf(tx);
|
|
522
|
+
while (round.length > 0) {
|
|
523
|
+
const next = new Set();
|
|
524
|
+
for (const id of round) {
|
|
525
|
+
const record = overlay.tasks.get(id) ?? this.#live.get(id);
|
|
526
|
+
if (record?.state.status !== "completing" || done.has(id))
|
|
527
|
+
continue;
|
|
528
|
+
if (this.#hasOwnedLive(id, overlay, done))
|
|
529
|
+
continue;
|
|
447
530
|
const outcome = record.state.outcome;
|
|
448
|
-
|
|
531
|
+
const terminal = withState(record, { status: "terminal", outcome });
|
|
532
|
+
tx.setTask(terminal);
|
|
533
|
+
overlay.tasks.set(id, terminal);
|
|
534
|
+
done.add(id);
|
|
449
535
|
// REMINDER: only the scheduler writes `faulted` and `orphaned` (spec §5.4); their cleanup waits for this commit.
|
|
450
536
|
if (outcome.status === "faulted" || outcome.status === "orphaned") {
|
|
451
537
|
await this.#settleOutcome(tx, record, outcome);
|
|
452
538
|
}
|
|
539
|
+
const above = this.#nearestLiveAbove(record, overlay);
|
|
540
|
+
if (above?.state.status === "completing")
|
|
541
|
+
next.add(above.id);
|
|
453
542
|
}
|
|
543
|
+
round = [...next];
|
|
454
544
|
}
|
|
455
545
|
}
|
|
456
546
|
/**
|
|
457
|
-
* Load the owner chains of
|
|
458
|
-
* Session line; returns the latter. Reads committed Storage directly, so it may run inside a commit callback.
|
|
547
|
+
* Load the owner chains of the tasks in `#unloaded` and, with `queued`, of every conversation with queued inputs,
|
|
548
|
+
* on the Session line; returns the latter. Reads committed Storage directly, so it may run inside a commit callback.
|
|
459
549
|
*/
|
|
460
550
|
async #loadScopes(queued) {
|
|
461
|
-
for (const
|
|
462
|
-
|
|
463
|
-
|
|
551
|
+
for (const id of [...this.#unloaded]) {
|
|
552
|
+
// An earlier load in this loop may have passed it.
|
|
553
|
+
if (this.#unloaded.has(id))
|
|
554
|
+
await this.#loadChain({ task: id });
|
|
464
555
|
}
|
|
465
556
|
if (!queued)
|
|
466
557
|
return [];
|
|
467
558
|
const submissions = await scanAll((cursor) => this.#storage.scanSubmissions({ status: "queued" }, SCAN_PAGE_SIZE, cursor, this.#context));
|
|
468
|
-
const
|
|
559
|
+
const inputs = submissions.filter((submission) => submission.type === "input");
|
|
560
|
+
const conversations = [...new Set(inputs.map((submission) => submission.conversationId))];
|
|
469
561
|
for (const id of conversations)
|
|
470
562
|
await this.#loadChain({ conversation: id });
|
|
471
563
|
return conversations;
|
|
472
564
|
}
|
|
473
|
-
/**
|
|
565
|
+
/**
|
|
566
|
+
* Load the owner edges and task nodes from `start` up to its ownerless root. A known task outside `#unloaded` ends the
|
|
567
|
+
* walk, as its chain is loaded; the `#unloaded` tasks passed have a loaded chain afterwards. Loading a node can reveal
|
|
568
|
+
* a cancelled owner, so it makes the next reconcile derive cascade marks.
|
|
569
|
+
*/
|
|
474
570
|
async #loadChain(start, overlay) {
|
|
571
|
+
const passed = [];
|
|
475
572
|
let at = start;
|
|
476
573
|
while (at !== undefined) {
|
|
477
574
|
if ("task" in at) {
|
|
478
|
-
|
|
575
|
+
const id = at.task;
|
|
576
|
+
let node = this.#live.get(id) ?? this.#settled.get(id);
|
|
577
|
+
// Created in this commit: the load from its own record covers what is above it.
|
|
578
|
+
if (node === undefined && overlay?.tasks.has(id) === true)
|
|
579
|
+
break;
|
|
580
|
+
if (node !== undefined && !this.#unloaded.has(id))
|
|
581
|
+
break;
|
|
479
582
|
if (node === undefined) {
|
|
480
|
-
const record = await this.#storage.task(
|
|
583
|
+
const record = await this.#storage.task(id, this.#context);
|
|
481
584
|
if (record === undefined)
|
|
482
585
|
return;
|
|
483
586
|
node = nodeOf(record);
|
|
484
587
|
if (record.state.status === "terminal")
|
|
485
|
-
this.#
|
|
588
|
+
this.#settle(id, node);
|
|
589
|
+
// A loaded node can reveal a cancelling owner: the next reconcile derives marks again.
|
|
590
|
+
this.#scheduleReconcile();
|
|
486
591
|
}
|
|
592
|
+
passed.push(id);
|
|
487
593
|
at = parentOf(node);
|
|
488
594
|
}
|
|
489
595
|
else {
|
|
@@ -492,15 +598,202 @@ export class TaskScheduler {
|
|
|
492
598
|
const record = await this.#storage.conversation(at.conversation, this.#context);
|
|
493
599
|
edge = record?.owner?.taskId ?? null;
|
|
494
600
|
this.#setEdge(at.conversation, edge);
|
|
601
|
+
// A loaded node can reveal a cancelling owner: the next reconcile derives marks again.
|
|
602
|
+
this.#scheduleReconcile();
|
|
495
603
|
}
|
|
496
604
|
at = edge === null ? undefined : { task: edge };
|
|
497
605
|
}
|
|
498
606
|
}
|
|
607
|
+
for (const id of passed)
|
|
608
|
+
this.#unloaded.delete(id);
|
|
499
609
|
}
|
|
500
610
|
#setEdge(conversationId, owner) {
|
|
501
611
|
this.#edges.set(conversationId, owner);
|
|
502
|
-
|
|
503
|
-
|
|
612
|
+
// The conversation now hangs from its owner, or is a root; with nothing below it, it waits for the sweep.
|
|
613
|
+
this.#relist(conversationNode(conversationId));
|
|
614
|
+
this.#dropIfUnneeded(conversationNode(conversationId));
|
|
615
|
+
}
|
|
616
|
+
/** Know a terminal task a chain load read, so walks pass through it; its own chain is not loaded yet. */
|
|
617
|
+
#settle(id, node) {
|
|
618
|
+
this.#settled.set(id, node);
|
|
619
|
+
this.#unloaded.add(id);
|
|
620
|
+
this.#relist(id);
|
|
621
|
+
this.#dropIfUnneeded(id);
|
|
622
|
+
}
|
|
623
|
+
/** Queue an ended task or a conversation for the sweep when nothing is listed below it. */
|
|
624
|
+
#dropIfUnneeded(node) {
|
|
625
|
+
if (typeof node !== "string" && this.#live.has(node))
|
|
626
|
+
return;
|
|
627
|
+
if (this.#below.has(node))
|
|
628
|
+
return;
|
|
629
|
+
this.#dropQueue.add(node);
|
|
630
|
+
this.#scheduleSweep();
|
|
631
|
+
}
|
|
632
|
+
/**
|
|
633
|
+
* Sweep in a job of its own on the Session line, after the operation underway: a rejected commit, one with nothing to
|
|
634
|
+
* write, and a read publish nothing, so the sweep at the end of `#observe` would not run for what they loaded.
|
|
635
|
+
*/
|
|
636
|
+
#scheduleSweep() {
|
|
637
|
+
if (this.#sweepScheduled || this.#closing)
|
|
638
|
+
return;
|
|
639
|
+
this.#sweepScheduled = true;
|
|
640
|
+
this.#session
|
|
641
|
+
.readOnLine(async () => {
|
|
642
|
+
this.#sweepScheduled = false;
|
|
643
|
+
this.#sweep();
|
|
644
|
+
})
|
|
645
|
+
// Rejected only once the Session is closing or failed, when nothing is kept anyway.
|
|
646
|
+
.catch(() => { });
|
|
647
|
+
}
|
|
648
|
+
/**
|
|
649
|
+
* Drop the queued nodes that are still unneeded: not live, nothing listed below. Runs once a publication is fully
|
|
650
|
+
* tracked, and as a job of its own on the line, never inside an operation: work a commit creates below a chain its
|
|
651
|
+
* own callback loaded keeps that chain. A flat loop: unlisting already queued every node on the way up that lost its
|
|
652
|
+
* last listed node.
|
|
653
|
+
*/
|
|
654
|
+
#sweep() {
|
|
655
|
+
for (const node of this.#dropQueue) {
|
|
656
|
+
if (this.#below.has(node))
|
|
657
|
+
continue;
|
|
658
|
+
if (typeof node === "string") {
|
|
659
|
+
this.#edges.delete(conversationOf(node));
|
|
660
|
+
this.#roots.delete(conversationOf(node));
|
|
661
|
+
}
|
|
662
|
+
else if (!this.#live.has(node)) {
|
|
663
|
+
this.#settled.delete(node);
|
|
664
|
+
this.#unloaded.delete(node);
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
this.#dropQueue.clear();
|
|
668
|
+
}
|
|
669
|
+
/**
|
|
670
|
+
* Bring the scheduler's sets in line with a task's committed record. A task that ended may free the nearest live task
|
|
671
|
+
* above it: one holding its outcome finalizes, an abort-marked one runs its abort handler; its waiters may run.
|
|
672
|
+
*/
|
|
673
|
+
#track(previous, record) {
|
|
674
|
+
const id = record.id;
|
|
675
|
+
if (previous?.state.status === "waiting")
|
|
676
|
+
for (const on of previous.state.on)
|
|
677
|
+
deleteFrom(this.#waiters, on, id);
|
|
678
|
+
if (record.state.status === "terminal") {
|
|
679
|
+
const node = nodeOf(record);
|
|
680
|
+
this.#live.delete(id);
|
|
681
|
+
this.#intent.delete(id);
|
|
682
|
+
this.#runnable.delete(id);
|
|
683
|
+
this.#finalizeChecks.delete(id);
|
|
684
|
+
this.#settled.set(id, node);
|
|
685
|
+
// Before anything is forgotten: the walk up to the task it held passes the ended tasks between them. Its chain is
|
|
686
|
+
// loaded, as every commit that ends a task loads the chains of `#unloaded` first.
|
|
687
|
+
this.#release(node);
|
|
688
|
+
for (const waiter of this.#waiters.get(id) ?? [])
|
|
689
|
+
this.#runnable.add(waiter);
|
|
690
|
+
this.#relist(id);
|
|
691
|
+
this.#dropIfUnneeded(id);
|
|
692
|
+
return;
|
|
693
|
+
}
|
|
694
|
+
this.#live.set(id, record);
|
|
695
|
+
if (previous === undefined) {
|
|
696
|
+
if (!this.#known(parentNode(record)))
|
|
697
|
+
this.#unloaded.add(id);
|
|
698
|
+
this.#relist(id);
|
|
699
|
+
}
|
|
700
|
+
if (cancellationIntent(record))
|
|
701
|
+
this.#intent.add(id);
|
|
702
|
+
else
|
|
703
|
+
this.#intent.delete(id);
|
|
704
|
+
if (record.state.status === "completing") {
|
|
705
|
+
this.#runnable.delete(id);
|
|
706
|
+
this.#finalizeChecks.add(id);
|
|
707
|
+
}
|
|
708
|
+
else {
|
|
709
|
+
this.#runnable.add(id);
|
|
710
|
+
}
|
|
711
|
+
if (record.state.status === "waiting")
|
|
712
|
+
for (const on of record.state.on)
|
|
713
|
+
addTo(this.#waiters, on, id);
|
|
714
|
+
}
|
|
715
|
+
/** The nearest live task above ended ordinary work may be free now: one holding its outcome, or an aborting one. */
|
|
716
|
+
#release(node, overlay) {
|
|
717
|
+
const above = this.#nearestLiveAbove(node, overlay);
|
|
718
|
+
if (above?.state.status === "completing")
|
|
719
|
+
this.#finalizeChecks.add(above.id);
|
|
720
|
+
else if (above?.abortRequested === true)
|
|
721
|
+
this.#runnable.add(above.id);
|
|
722
|
+
return above;
|
|
723
|
+
}
|
|
724
|
+
/**
|
|
725
|
+
* List `start` under its parent node in `#below` while it is a live task or has nodes listed below it, and unlist it
|
|
726
|
+
* otherwise; then its parent, as far up as that changes. So `#below` holds exactly the tree around live work; a
|
|
727
|
+
* parent that loses its last listed node is queued for the sweep. A loop, as ownership chains can be deeper than the
|
|
728
|
+
* call stack.
|
|
729
|
+
*/
|
|
730
|
+
#relist(start) {
|
|
731
|
+
let node = start;
|
|
732
|
+
for (;;) {
|
|
733
|
+
const needed = (typeof node !== "string" && this.#live.has(node)) || this.#below.has(node);
|
|
734
|
+
let parent;
|
|
735
|
+
if (typeof node === "string") {
|
|
736
|
+
parent = this.#edges.get(conversationOf(node));
|
|
737
|
+
}
|
|
738
|
+
else {
|
|
739
|
+
const known = this.#live.get(node) ?? this.#settled.get(node);
|
|
740
|
+
if (known === undefined)
|
|
741
|
+
return;
|
|
742
|
+
parent = parentNode(known);
|
|
743
|
+
}
|
|
744
|
+
if (parent === undefined)
|
|
745
|
+
return;
|
|
746
|
+
if (parent === null) {
|
|
747
|
+
const root = conversationOf(node);
|
|
748
|
+
if (needed)
|
|
749
|
+
this.#roots.add(root);
|
|
750
|
+
else
|
|
751
|
+
this.#roots.delete(root);
|
|
752
|
+
return;
|
|
753
|
+
}
|
|
754
|
+
if ((this.#below.get(parent)?.has(node) === true) === needed)
|
|
755
|
+
return;
|
|
756
|
+
if (needed) {
|
|
757
|
+
const first = !this.#below.has(parent);
|
|
758
|
+
addTo(this.#below, parent, node);
|
|
759
|
+
if (!first)
|
|
760
|
+
return;
|
|
761
|
+
}
|
|
762
|
+
else {
|
|
763
|
+
deleteFrom(this.#below, parent, node);
|
|
764
|
+
if (this.#below.has(parent))
|
|
765
|
+
return;
|
|
766
|
+
this.#dropIfUnneeded(parent);
|
|
767
|
+
}
|
|
768
|
+
node = parent;
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
/**
|
|
772
|
+
* The nearest live task above ended ordinary work `node`, in the overlay's view: the one that work held, if any. Work
|
|
773
|
+
* at or below a background task holds nothing above it.
|
|
774
|
+
*/
|
|
775
|
+
#nearestLiveAbove(node, overlay) {
|
|
776
|
+
for (let current = node; !current.background;) {
|
|
777
|
+
const owner = current.owner ?? this.#edge(current.conversationId, overlay);
|
|
778
|
+
if (owner === undefined || owner === null)
|
|
779
|
+
return undefined;
|
|
780
|
+
const record = overlay?.tasks.get(owner) ?? this.#live.get(owner);
|
|
781
|
+
if (record !== undefined && record.state.status !== "terminal")
|
|
782
|
+
return record;
|
|
783
|
+
const above = record ?? this.#settled.get(owner);
|
|
784
|
+
if (above === undefined)
|
|
785
|
+
return undefined;
|
|
786
|
+
current = above;
|
|
787
|
+
}
|
|
788
|
+
return undefined;
|
|
789
|
+
}
|
|
790
|
+
/** Whether the chain above `node` is loaded, without walking: known tasks outside `#unloaded` have a loaded chain. */
|
|
791
|
+
#known(node) {
|
|
792
|
+
if (typeof node !== "string") {
|
|
793
|
+
return (this.#live.has(node) || this.#settled.has(node)) && !this.#unloaded.has(node);
|
|
794
|
+
}
|
|
795
|
+
const owner = this.#edges.get(conversationOf(node));
|
|
796
|
+
return owner === null || (owner !== undefined && this.#known(owner));
|
|
504
797
|
}
|
|
505
798
|
/** Owner task of a conversation, `null` when ownerless, `undefined` while not loaded. */
|
|
506
799
|
#edge(id, overlay) {
|
|
@@ -533,51 +826,96 @@ export class TaskScheduler {
|
|
|
533
826
|
}
|
|
534
827
|
}
|
|
535
828
|
}
|
|
536
|
-
/**
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
for (
|
|
546
|
-
const
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
829
|
+
/**
|
|
830
|
+
* The tasks below `start` that a walk down reaches: those `#below` lists and, under an overlay, the commit's new tasks,
|
|
831
|
+
* with its candidates in place of committed records. Background tasks are reached but not entered, unless
|
|
832
|
+
* `crossBackground`, and neither are the tasks `enter` rejects. A loop, as chains can be deeper than the call stack.
|
|
833
|
+
*/
|
|
834
|
+
*#walkDown(start, options = {}) {
|
|
835
|
+
const { overlay, crossBackground = false, enter } = options;
|
|
836
|
+
const links = overlay === undefined ? undefined : this.#links(overlay);
|
|
837
|
+
const pending = [start];
|
|
838
|
+
for (let at = pending.pop(); at !== undefined; at = pending.pop()) {
|
|
839
|
+
for (const children of [this.#below.get(at), links?.get(at)]) {
|
|
840
|
+
for (const child of children ?? []) {
|
|
841
|
+
if (typeof child === "string") {
|
|
842
|
+
pending.push(child);
|
|
843
|
+
continue;
|
|
844
|
+
}
|
|
845
|
+
const record = overlay?.tasks.get(child) ?? this.#live.get(child);
|
|
846
|
+
const node = record ?? this.#settled.get(child);
|
|
847
|
+
if (node === undefined)
|
|
848
|
+
continue;
|
|
849
|
+
const live = record !== undefined && record.state.status !== "terminal" ? record : undefined;
|
|
850
|
+
yield { id: child, node, live };
|
|
851
|
+
if ((crossBackground || !node.background) && (enter === undefined || enter(child)))
|
|
852
|
+
pending.push(child);
|
|
853
|
+
}
|
|
854
|
+
}
|
|
555
855
|
}
|
|
556
856
|
}
|
|
557
857
|
/**
|
|
558
|
-
*
|
|
559
|
-
*
|
|
858
|
+
* The links walks down need for the commit's new tasks: each one under its parent node, then each node above it that
|
|
859
|
+
* `#below` does not list yet, until one it lists, a live task (always listed), or the chain's end.
|
|
560
860
|
*/
|
|
561
|
-
#
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
861
|
+
#links(overlay) {
|
|
862
|
+
if (overlay.links !== undefined)
|
|
863
|
+
return overlay.links;
|
|
864
|
+
const links = new Map();
|
|
865
|
+
for (const record of overlay.tasks.values()) {
|
|
866
|
+
if (this.#live.has(record.id) || this.#settled.has(record.id))
|
|
565
867
|
continue;
|
|
566
|
-
|
|
567
|
-
|
|
868
|
+
let child = record.id;
|
|
869
|
+
let parent = parentNode(record);
|
|
870
|
+
while (parent !== undefined && parent !== null && links.get(parent)?.has(child) !== true) {
|
|
871
|
+
addTo(links, parent, child);
|
|
872
|
+
if (typeof parent !== "string" && (this.#live.has(parent) || overlay.tasks.has(parent)))
|
|
568
873
|
break;
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
874
|
+
let above;
|
|
875
|
+
if (typeof parent === "string") {
|
|
876
|
+
above = this.#edge(conversationOf(parent), overlay);
|
|
877
|
+
}
|
|
878
|
+
else {
|
|
879
|
+
const fields = this.#settled.get(parent);
|
|
880
|
+
above = fields === undefined ? undefined : parentNode(fields);
|
|
881
|
+
}
|
|
882
|
+
if (above !== undefined && above !== null && this.#below.get(above)?.has(parent) === true)
|
|
577
883
|
break;
|
|
884
|
+
child = parent;
|
|
885
|
+
parent = above;
|
|
578
886
|
}
|
|
579
887
|
}
|
|
580
|
-
|
|
888
|
+
overlay.links = links;
|
|
889
|
+
return links;
|
|
890
|
+
}
|
|
891
|
+
/**
|
|
892
|
+
* The live ordinary owned work of `owner` (spec §5.5): the live non-background tasks below it that a walk down reaches
|
|
893
|
+
* without entering a background task; with `first`, only the first. Candidates in `overlay` replace committed records;
|
|
894
|
+
* the walk does not enter tasks in `done`, which have none. Owner chains must be loaded.
|
|
895
|
+
*/
|
|
896
|
+
#ownedLive(owner, options = {}) {
|
|
897
|
+
const { overlay, first = false, done } = options;
|
|
898
|
+
const found = [];
|
|
899
|
+
const walk = this.#walkDown(owner, {
|
|
900
|
+
...(overlay === undefined ? {} : { overlay }),
|
|
901
|
+
...(done === undefined ? {} : { enter: (id) => !done.has(id) }),
|
|
902
|
+
});
|
|
903
|
+
for (const { id, node, live } of walk) {
|
|
904
|
+
if (live === undefined || node.background)
|
|
905
|
+
continue;
|
|
906
|
+
found.push(id);
|
|
907
|
+
if (first)
|
|
908
|
+
break;
|
|
909
|
+
}
|
|
910
|
+
return found;
|
|
911
|
+
}
|
|
912
|
+
#hasOwnedLive(owner, overlay, done) {
|
|
913
|
+
const options = {
|
|
914
|
+
first: true,
|
|
915
|
+
...(overlay === undefined ? {} : { overlay }),
|
|
916
|
+
...(done === undefined ? {} : { done }),
|
|
917
|
+
};
|
|
918
|
+
return this.#ownedLive(owner, options).length > 0;
|
|
581
919
|
}
|
|
582
920
|
/**
|
|
583
921
|
* Whether ordinary traversal from `scope` reaches `start`: walking up reaches the scope's conversation, or an
|
|
@@ -603,32 +941,47 @@ export class TaskScheduler {
|
|
|
603
941
|
* background owner without it. Terminal owners never cascade (spec §5.4).
|
|
604
942
|
*/
|
|
605
943
|
#belowCancelled(start) {
|
|
944
|
+
return this.#cancellingOwner(start) !== undefined;
|
|
945
|
+
}
|
|
946
|
+
/** The nearest live owner above `start` whose cancellation intent reaches it, if any; see `#belowCancelled`. */
|
|
947
|
+
#cancellingOwner(start) {
|
|
948
|
+
// Nothing cancels, the common case: no walk.
|
|
949
|
+
if (this.#intent.size === 0)
|
|
950
|
+
return undefined;
|
|
606
951
|
for (const step of this.#above(start)) {
|
|
607
952
|
if ("unknown" in step)
|
|
608
|
-
return
|
|
953
|
+
return undefined;
|
|
609
954
|
if (!("task" in step))
|
|
610
955
|
continue;
|
|
611
956
|
const live = this.#live.get(step.task);
|
|
612
957
|
if (live !== undefined && cancellationIntent(live))
|
|
613
|
-
return
|
|
958
|
+
return live;
|
|
614
959
|
if (step.node.background)
|
|
615
|
-
return
|
|
960
|
+
return undefined;
|
|
616
961
|
}
|
|
617
|
-
return
|
|
962
|
+
return undefined;
|
|
963
|
+
}
|
|
964
|
+
/** Fail the Session, which reports it, for a throw in one of the scheduler's own commits; ignored once closing. */
|
|
965
|
+
#failSession(error) {
|
|
966
|
+
if (!this.#closing)
|
|
967
|
+
this.#session.fail(error);
|
|
618
968
|
}
|
|
619
|
-
/** Close listener: runs synchronously once admission is sealed, before `join()
|
|
969
|
+
/** Close listener: runs synchronously once admission is sealed, before `join()`, or when the Session fails. */
|
|
620
970
|
#seal() {
|
|
621
971
|
this.#closing = true;
|
|
622
|
-
|
|
623
|
-
|
|
972
|
+
try {
|
|
973
|
+
this.#unsubscribeRegistry();
|
|
974
|
+
}
|
|
975
|
+
catch (error) {
|
|
976
|
+
this.#report(error);
|
|
977
|
+
}
|
|
978
|
+
const error = closedError(this.#session);
|
|
624
979
|
this.#taskWaiters.rejectAll(error);
|
|
625
980
|
this.#idleWaiters.rejectAll(error);
|
|
626
981
|
this.#contexts.clear();
|
|
627
982
|
if (this.#expiry !== undefined)
|
|
628
983
|
clearTimeout(this.#expiry.timer);
|
|
629
984
|
this.#expiry = undefined;
|
|
630
|
-
for (const invocation of this.#invocations.values())
|
|
631
|
-
invocation.controller.abort();
|
|
632
985
|
}
|
|
633
986
|
#kick() {
|
|
634
987
|
this.#dirty = true;
|
|
@@ -647,38 +1000,64 @@ export class TaskScheduler {
|
|
|
647
1000
|
}
|
|
648
1001
|
}
|
|
649
1002
|
catch (error) {
|
|
650
|
-
|
|
651
|
-
|
|
1003
|
+
// As in `#reconcile()`: the reservation commit runs no extension code.
|
|
1004
|
+
this.#failSession(error);
|
|
652
1005
|
}
|
|
653
1006
|
finally {
|
|
654
1007
|
this.#draining = false;
|
|
655
|
-
// A wakeup that arrived
|
|
1008
|
+
// A wakeup that arrived after the loop's last check still needs its pass.
|
|
656
1009
|
if (this.#dirty)
|
|
657
1010
|
this.#kick();
|
|
658
1011
|
}
|
|
659
1012
|
}
|
|
660
|
-
/**
|
|
1013
|
+
/**
|
|
1014
|
+
* Reserve every eligible task in one commit; orphan abort-marked tasks no definition can take, unless abandoned after
|
|
1015
|
+
* a restart. The first pass after open only abort-marks the abandoned tasks, so later passes see the marks.
|
|
1016
|
+
*/
|
|
661
1017
|
async #reserve() {
|
|
662
1018
|
const reservations = [];
|
|
1019
|
+
let abandoned = [];
|
|
663
1020
|
try {
|
|
664
1021
|
await this.#session.commitWith(async (tx) => {
|
|
665
1022
|
if (!this.#enabled || this.#closing)
|
|
666
1023
|
return;
|
|
1024
|
+
for (const id of this.#abandoned) {
|
|
1025
|
+
const record = this.#live.get(id);
|
|
1026
|
+
if (record !== undefined && !record.abortRequested)
|
|
1027
|
+
tx.setTask(withAbortMark(record, "restart"));
|
|
1028
|
+
}
|
|
1029
|
+
if (this.#abandoned.size > 0) {
|
|
1030
|
+
abandoned = [...this.#abandoned];
|
|
1031
|
+
return;
|
|
1032
|
+
}
|
|
667
1033
|
await this.#loadScopes(false);
|
|
668
|
-
const owned = this.#ownedLive();
|
|
669
1034
|
// Taken once per pass, and only when some task is a candidate.
|
|
670
1035
|
let snapshot;
|
|
671
|
-
for (const
|
|
672
|
-
|
|
1036
|
+
for (const id of [...this.#runnable]) {
|
|
1037
|
+
const record = this.#live.get(id);
|
|
1038
|
+
if (record === undefined || record.state.status === "completing" || this.#invocations.has(id)) {
|
|
1039
|
+
this.#runnable.delete(id);
|
|
673
1040
|
continue;
|
|
674
|
-
|
|
1041
|
+
}
|
|
1042
|
+
// It returns once what it waits for ends, its record changes, or the work below it ends.
|
|
1043
|
+
if (this.#waitingOn(record, true).length > 0) {
|
|
1044
|
+
this.#runnable.delete(id);
|
|
675
1045
|
continue;
|
|
1046
|
+
}
|
|
676
1047
|
const runnable = record;
|
|
677
1048
|
const mode = record.abortRequested ? "abort" : "run";
|
|
1049
|
+
// Work below an owner with cancellation intent waits for its cascade mark instead of running a phase; schedule
|
|
1050
|
+
// the cascade, which may not have run yet.
|
|
1051
|
+
if (mode === "run" && !record.background && this.#belowCancelled(parentOf(record))) {
|
|
1052
|
+
this.#cascadePending = true;
|
|
1053
|
+
this.#scheduleReconcile();
|
|
1054
|
+
continue;
|
|
1055
|
+
}
|
|
678
1056
|
snapshot ??= this.#registry.snapshot();
|
|
679
1057
|
const resolution = this.#resolve(runnable, snapshot);
|
|
680
1058
|
if (resolution.kind === "blocked") {
|
|
681
|
-
|
|
1059
|
+
// An abandoned task waits for its definition, so its abort handler can clean up.
|
|
1060
|
+
if (mode === "abort" && record.abortReason === undefined) {
|
|
682
1061
|
await this.#terminate(tx, record, { status: "orphaned", reason: resolution.reason });
|
|
683
1062
|
}
|
|
684
1063
|
continue;
|
|
@@ -689,7 +1068,9 @@ export class TaskScheduler {
|
|
|
689
1068
|
checkpoint: resolution.record.state.checkpoint,
|
|
690
1069
|
}));
|
|
691
1070
|
}
|
|
692
|
-
// Registered on the line, so marks and later reservations see it and close joins it
|
|
1071
|
+
// Registered on the line, so marks and later reservations see it and close joins it; `#end` makes it a
|
|
1072
|
+
// candidate again.
|
|
1073
|
+
this.#runnable.delete(id);
|
|
693
1074
|
const invocation = this.#createInvocation(record, mode);
|
|
694
1075
|
reservations.push({ invocation, task: resolution.task, snapshot });
|
|
695
1076
|
}
|
|
@@ -698,22 +1079,31 @@ export class TaskScheduler {
|
|
|
698
1079
|
catch (error) {
|
|
699
1080
|
for (const { invocation } of reservations) {
|
|
700
1081
|
this.#invocations.delete(invocation.taskId);
|
|
1082
|
+
this.#runnable.add(invocation.taskId);
|
|
701
1083
|
invocation.finish();
|
|
702
1084
|
}
|
|
703
1085
|
throw error;
|
|
704
1086
|
}
|
|
1087
|
+
// Marked, or already marked or gone: done with them. Reserve again, now seeing the marks.
|
|
1088
|
+
for (const id of abandoned)
|
|
1089
|
+
this.#abandoned.delete(id);
|
|
1090
|
+
if (abandoned.length > 0)
|
|
1091
|
+
this.#dirty = true;
|
|
705
1092
|
return reservations;
|
|
706
1093
|
}
|
|
707
1094
|
/**
|
|
708
1095
|
* Live tasks a task waits for before its next invocation: its live ordinary owned work when abort-marked, since
|
|
709
1096
|
* abort runs bottom-up, otherwise the live part of the `on` of a wait.
|
|
710
1097
|
*/
|
|
711
|
-
#waitingOn(record,
|
|
1098
|
+
#waitingOn(record, first = false) {
|
|
712
1099
|
if (record.abortRequested)
|
|
713
|
-
return
|
|
1100
|
+
return this.#ownedLive(record.id, { first });
|
|
714
1101
|
if (record.state.status !== "waiting")
|
|
715
1102
|
return [];
|
|
716
|
-
|
|
1103
|
+
if (!first)
|
|
1104
|
+
return record.state.on.filter((id) => this.#live.has(id));
|
|
1105
|
+
const live = record.state.on.find((id) => this.#live.has(id));
|
|
1106
|
+
return live === undefined ? [] : [live];
|
|
717
1107
|
}
|
|
718
1108
|
/** Resolve the record's definition by kind, migrating an older stored version. */
|
|
719
1109
|
#resolve(record, snapshot) {
|
|
@@ -750,6 +1140,21 @@ export class TaskScheduler {
|
|
|
750
1140
|
return { reason: "migration_failed", error: failed.error };
|
|
751
1141
|
return { task, migrates: true };
|
|
752
1142
|
}
|
|
1143
|
+
/** Sizes of the ownership indexes, for tests that bound the scheduler's memory. */
|
|
1144
|
+
get indexSizes() {
|
|
1145
|
+
let below = 0;
|
|
1146
|
+
for (const children of this.#below.values())
|
|
1147
|
+
below += children.size;
|
|
1148
|
+
return {
|
|
1149
|
+
live: this.#live.size,
|
|
1150
|
+
settled: this.#settled.size,
|
|
1151
|
+
edges: this.#edges.size,
|
|
1152
|
+
below,
|
|
1153
|
+
roots: this.#roots.size,
|
|
1154
|
+
unloaded: this.#unloaded.size,
|
|
1155
|
+
dropQueue: this.#dropQueue.size,
|
|
1156
|
+
};
|
|
1157
|
+
}
|
|
753
1158
|
/**
|
|
754
1159
|
* Scheduling state and every live task with its derived state, read on the Session line. Runs no task code: a
|
|
755
1160
|
* pending migration shows as `ready` with `migrates`, and only a migration the scheduler already tried, or one that
|
|
@@ -757,20 +1162,18 @@ export class TaskScheduler {
|
|
|
757
1162
|
*/
|
|
758
1163
|
async inspect(snapshot) {
|
|
759
1164
|
await this.#loadScopes(false);
|
|
760
|
-
const owned = this.#ownedLive();
|
|
761
1165
|
const tasks = [];
|
|
762
|
-
for (const record of this.#live.values())
|
|
763
|
-
tasks.push({ record, state: this.#inspectTask(record, snapshot
|
|
764
|
-
}
|
|
1166
|
+
for (const record of this.#live.values())
|
|
1167
|
+
tasks.push({ record, state: this.#inspectTask(record, snapshot) });
|
|
765
1168
|
const scheduling = this.#closing ? "closing" : this.#enabled ? "running" : "paused";
|
|
766
1169
|
return { scheduling, tasks };
|
|
767
1170
|
}
|
|
768
|
-
#inspectTask(record, snapshot
|
|
1171
|
+
#inspectTask(record, snapshot) {
|
|
769
1172
|
if (this.#invocations.has(record.id))
|
|
770
1173
|
return { kind: "running" };
|
|
771
1174
|
if (record.state.status === "completing")
|
|
772
1175
|
return { kind: "completing" };
|
|
773
|
-
const on = this.#waitingOn(record
|
|
1176
|
+
const on = this.#waitingOn(record);
|
|
774
1177
|
if (on.length > 0)
|
|
775
1178
|
return { kind: "waiting", on };
|
|
776
1179
|
const fit = this.#fit(record, snapshot.task(record.kind));
|
|
@@ -849,6 +1252,12 @@ export class TaskScheduler {
|
|
|
849
1252
|
// 3. abort mark: end; a fresh abort invocation starts once the task's ordinary owned work is gone.
|
|
850
1253
|
if (current.abortRequested)
|
|
851
1254
|
return false;
|
|
1255
|
+
// An owner's cancellation intent ends it too, before its cascade marks it; reservation then holds it back.
|
|
1256
|
+
if (!current.background && this.#belowCancelled(parentOf(current))) {
|
|
1257
|
+
this.#cascadePending = true;
|
|
1258
|
+
this.#scheduleReconcile();
|
|
1259
|
+
return false;
|
|
1260
|
+
}
|
|
852
1261
|
if (previous === undefined)
|
|
853
1262
|
return true;
|
|
854
1263
|
// 4. uncaught error.
|
|
@@ -909,16 +1318,17 @@ export class TaskScheduler {
|
|
|
909
1318
|
return current;
|
|
910
1319
|
this.#end(invocation);
|
|
911
1320
|
if (decision !== false) {
|
|
912
|
-
const message =
|
|
1321
|
+
const message = errorMessage(decision.fault);
|
|
913
1322
|
await this.#terminate(tx, current, { status: "faulted", error: { message } });
|
|
914
1323
|
}
|
|
915
1324
|
return undefined;
|
|
916
1325
|
}, this.#context);
|
|
917
1326
|
}
|
|
918
1327
|
catch (error) {
|
|
1328
|
+
// The step writes the scheduler's own decision, a fault or a terminal record; a throw there would otherwise
|
|
1329
|
+
// leave the task running, to be reserved and run again.
|
|
919
1330
|
this.#end(invocation);
|
|
920
|
-
|
|
921
|
-
this.#report(error);
|
|
1331
|
+
this.#failSession(error);
|
|
922
1332
|
return undefined;
|
|
923
1333
|
}
|
|
924
1334
|
}
|
|
@@ -928,7 +1338,7 @@ export class TaskScheduler {
|
|
|
928
1338
|
*/
|
|
929
1339
|
async #terminate(tx, record, outcome) {
|
|
930
1340
|
await this.#loadScopes(false);
|
|
931
|
-
if (this.#
|
|
1341
|
+
if (this.#hasOwnedLive(record.id, overlayOf(tx))) {
|
|
932
1342
|
tx.setTask(withState(record, { status: "completing", outcome }));
|
|
933
1343
|
return;
|
|
934
1344
|
}
|
|
@@ -948,7 +1358,7 @@ export class TaskScheduler {
|
|
|
948
1358
|
await this.#loadScopes(false);
|
|
949
1359
|
for (const record of overlay.tasks.values())
|
|
950
1360
|
await this.#loadChain(parentOf(record), overlay);
|
|
951
|
-
if (this.#
|
|
1361
|
+
if (this.#hasOwnedLive(current.id, overlay)) {
|
|
952
1362
|
tx.setTask(withState(current, { status: "completing", outcome: next.outcome }));
|
|
953
1363
|
return;
|
|
954
1364
|
}
|
|
@@ -956,37 +1366,102 @@ export class TaskScheduler {
|
|
|
956
1366
|
tx.setTask(withState(current, next));
|
|
957
1367
|
}
|
|
958
1368
|
/**
|
|
959
|
-
* A wait names existing tasks
|
|
960
|
-
*
|
|
1369
|
+
* A wait names existing tasks, and `failFast` only tasks the waiter owns. It may not close a cycle in the wait-for
|
|
1370
|
+
* graph, whose edges run from each task to what it waits on and to its live ordinary owned work (spec §5.5); waiting
|
|
1371
|
+
* on itself or on an owner holding it are the shortest such cycles. An abort handler cannot wait.
|
|
1372
|
+
*
|
|
1373
|
+
* Invariant: the wait-for graph is acyclic. Only a wait adds an edge between existing tasks, as a new task has only
|
|
1374
|
+
* incoming edges, so checking each wait maintains it.
|
|
961
1375
|
*/
|
|
962
1376
|
async #validateWait(tx, invocation, current, on, policy) {
|
|
963
1377
|
if (invocation.mode === "abort")
|
|
964
1378
|
throw new Error(`Abort handler of task ${current.id} cannot wait`);
|
|
965
1379
|
const overlay = overlayOf(tx);
|
|
966
|
-
|
|
967
|
-
const owners = new Set();
|
|
968
|
-
for (const step of this.#above(parentOf(current)))
|
|
969
|
-
if ("task" in step)
|
|
970
|
-
owners.add(step.task);
|
|
1380
|
+
const targets = [];
|
|
971
1381
|
for (const id of on) {
|
|
972
|
-
if (id === current.id || owners.has(id)) {
|
|
973
|
-
throw new Error(`Task ${current.id} cannot wait on itself or its owner ${id}`);
|
|
974
|
-
}
|
|
975
1382
|
const member = overlay.tasks.get(id) ?? this.#live.get(id) ?? (await this.#storage.task(id, this.#context));
|
|
976
1383
|
if (member === undefined)
|
|
977
1384
|
throw new Error(`Task ${id} does not exist`);
|
|
978
1385
|
if (policy === "failFast" && member.owner !== current.id) {
|
|
979
1386
|
throw new Error(`Task ${current.id} can wait failFast only on tasks it owns; ${id} is not one`);
|
|
980
1387
|
}
|
|
1388
|
+
// A live child already holds the waiter (task-owned tasks are never background): waiting on it adds no edge. A
|
|
1389
|
+
// terminal task has no edges.
|
|
1390
|
+
if (member.state.status !== "terminal" && member.owner !== current.id)
|
|
1391
|
+
targets.push(id);
|
|
1392
|
+
}
|
|
1393
|
+
if (targets.length === 0)
|
|
1394
|
+
return;
|
|
1395
|
+
const cycle = await this.#waitCycle(current.id, targets, overlay);
|
|
1396
|
+
if (cycle !== undefined) {
|
|
1397
|
+
throw new Error(`Task ${current.id} cannot wait on ${cycle[0]}: ${[current.id, ...cycle].join(" -> ")} is a wait cycle`);
|
|
1398
|
+
}
|
|
1399
|
+
}
|
|
1400
|
+
/**
|
|
1401
|
+
* The path from a member of `on` back to `task` in the wait-for graph, if the wait would close a cycle (spec §5.1).
|
|
1402
|
+
* An edge X -> Y means X cannot settle before Y: Y is in the `on` of waiting X that is not abort-marked, or Y is live
|
|
1403
|
+
* ordinary owned work of X,
|
|
1404
|
+
* which holds X as `completing`. Each task is visited once and each subtree walked once, so a check is bounded by the
|
|
1405
|
+
* live work it reaches.
|
|
1406
|
+
*/
|
|
1407
|
+
async #waitCycle(task, on, overlay) {
|
|
1408
|
+
// Owned work is found by walks down, which need the chains of every live task loaded.
|
|
1409
|
+
await this.#loadScopes(false);
|
|
1410
|
+
const from = new Map();
|
|
1411
|
+
const walked = new Set();
|
|
1412
|
+
const pending = [];
|
|
1413
|
+
const reach = (id, via) => {
|
|
1414
|
+
if (from.has(id))
|
|
1415
|
+
return;
|
|
1416
|
+
from.set(id, via);
|
|
1417
|
+
pending.push(id);
|
|
1418
|
+
};
|
|
1419
|
+
const path = (end) => {
|
|
1420
|
+
const steps = [];
|
|
1421
|
+
for (let at = end; at !== undefined; at = from.get(at))
|
|
1422
|
+
steps.unshift(at);
|
|
1423
|
+
return steps;
|
|
1424
|
+
};
|
|
1425
|
+
for (const id of on)
|
|
1426
|
+
reach(id, undefined);
|
|
1427
|
+
for (let id = pending.pop(); id !== undefined; id = pending.pop()) {
|
|
1428
|
+
if (id === task)
|
|
1429
|
+
return path(id);
|
|
1430
|
+
const record = overlay.tasks.get(id) ?? this.#live.get(id);
|
|
1431
|
+
if (record === undefined || record.state.status === "terminal")
|
|
1432
|
+
continue;
|
|
1433
|
+
// An abort-marked task waits only for its owned work, as abort runs bottom-up (`#waitingOn`).
|
|
1434
|
+
if (record.state.status === "waiting" && !record.abortRequested) {
|
|
1435
|
+
for (const member of record.state.on)
|
|
1436
|
+
reach(member, id);
|
|
1437
|
+
}
|
|
1438
|
+
if (walked.has(id))
|
|
1439
|
+
continue;
|
|
1440
|
+
walked.add(id);
|
|
1441
|
+
// Below a walked task the walk does not go again: its own walk covered what lies below.
|
|
1442
|
+
const found = [];
|
|
1443
|
+
const walk = this.#walkDown(id, { overlay, enter: (below) => !walked.has(below) });
|
|
1444
|
+
for (const { id: below, node, live } of walk) {
|
|
1445
|
+
if (live === undefined || node.background)
|
|
1446
|
+
continue;
|
|
1447
|
+
found.push(below);
|
|
1448
|
+
reach(below, id);
|
|
1449
|
+
}
|
|
1450
|
+
for (const below of found)
|
|
1451
|
+
walked.add(below);
|
|
981
1452
|
}
|
|
1453
|
+
return undefined;
|
|
982
1454
|
}
|
|
983
1455
|
/** End an invocation: its runtime operations reject from now on, its signal aborts, its watches stop, and its task is free. */
|
|
984
1456
|
#end(invocation) {
|
|
985
1457
|
if (invocation.ended)
|
|
986
1458
|
return;
|
|
987
1459
|
invocation.ended = true;
|
|
988
|
-
if (this.#invocations.get(invocation.taskId) === invocation)
|
|
1460
|
+
if (this.#invocations.get(invocation.taskId) === invocation) {
|
|
989
1461
|
this.#invocations.delete(invocation.taskId);
|
|
1462
|
+
if (this.#live.has(invocation.taskId))
|
|
1463
|
+
this.#runnable.add(invocation.taskId);
|
|
1464
|
+
}
|
|
990
1465
|
for (const watch of invocation.watches)
|
|
991
1466
|
void watch.stop();
|
|
992
1467
|
// Pending waits bound to the invocation, such as a tool's waitForTask(), reject with it.
|
|
@@ -995,10 +1470,20 @@ export class TaskScheduler {
|
|
|
995
1470
|
/** No live non-background task in the scope; a task whose owner edges are not loaded yet counts as inside. */
|
|
996
1471
|
#idle(conversationId) {
|
|
997
1472
|
const scope = conversationId === undefined ? { roots: true } : { conversation: conversationId };
|
|
998
|
-
|
|
999
|
-
|
|
1473
|
+
// Tasks whose chain is not loaded are not listed below their scope yet; one counts as inside unless its walk up
|
|
1474
|
+
// decides otherwise.
|
|
1475
|
+
for (const id of this.#unloaded) {
|
|
1476
|
+
const record = this.#live.get(id);
|
|
1477
|
+
if (record !== undefined && !record.background && this.#inScope(parentOf(record), scope) !== false)
|
|
1000
1478
|
return false;
|
|
1001
1479
|
}
|
|
1480
|
+
const starts = conversationId === undefined ? this.#roots : [conversationId];
|
|
1481
|
+
for (const start of starts) {
|
|
1482
|
+
for (const { node, live } of this.#walkDown(conversationNode(start))) {
|
|
1483
|
+
if (live !== undefined && !node.background)
|
|
1484
|
+
return false;
|
|
1485
|
+
}
|
|
1486
|
+
}
|
|
1002
1487
|
return true;
|
|
1003
1488
|
}
|
|
1004
1489
|
// ─── Invocation runtime ──────────────────────────────────────────────────
|
|
@@ -1070,6 +1555,34 @@ export class TaskScheduler {
|
|
|
1070
1555
|
snapshotAsOf: ((...args) => this.#read(invocation, () => sessionMethod(this.#session, "snapshotAsOf")(...args))),
|
|
1071
1556
|
getTask: ((id, context) => this.#read(invocation, () => this.#session.readOnLine(() => this.#storage.task(id, context)))),
|
|
1072
1557
|
waitForTask: ((id, context) => this.#read(invocation, () => this.waitForTask(id, withAbortSignal(invocation.controller.signal, context)))),
|
|
1558
|
+
ownedTasks: (context) => this.#read(invocation, () => this.#session.readOnLine(async () => {
|
|
1559
|
+
// The step that ends the invocation may have run on the line before this.
|
|
1560
|
+
if (invocation.ended)
|
|
1561
|
+
throw endedError(invocation);
|
|
1562
|
+
context.abortSignal?.throwIfAborted();
|
|
1563
|
+
// Every live task is listed under its parent node, its owner task when it has one.
|
|
1564
|
+
const owned = [];
|
|
1565
|
+
for (const node of this.#below.get(invocation.taskId) ?? []) {
|
|
1566
|
+
const record = typeof node === "string" ? undefined : this.#live.get(node);
|
|
1567
|
+
if (record?.owner === invocation.taskId)
|
|
1568
|
+
owned.push(record);
|
|
1569
|
+
}
|
|
1570
|
+
// Copies, as from storage: the scheduler reads its own.
|
|
1571
|
+
return owned
|
|
1572
|
+
.sort((a, b) => a.id - b.id)
|
|
1573
|
+
.map((record) => copyJson(record));
|
|
1574
|
+
})),
|
|
1575
|
+
abortOwned: (id, context) => this.#read(invocation, async () => {
|
|
1576
|
+
const bound = withAbortSignal(invocation.controller.signal, context);
|
|
1577
|
+
const record = await this.#session.readOnLine(() => this.#storage.task(id, bound));
|
|
1578
|
+
if (record?.owner !== invocation.taskId) {
|
|
1579
|
+
throw new Error(`Task ${id} is not owned by task ${invocation.taskId}`);
|
|
1580
|
+
}
|
|
1581
|
+
if (record.state.status === "terminal")
|
|
1582
|
+
return;
|
|
1583
|
+
await this.abort(id, bound, true);
|
|
1584
|
+
await this.waitForTask(id, bound);
|
|
1585
|
+
}),
|
|
1073
1586
|
outcomes: ((ids, context) => this.#read(invocation, () => this.#session.readOnLine(async () => {
|
|
1074
1587
|
const outcomes = [];
|
|
1075
1588
|
for (const id of ids) {
|
|
@@ -1142,7 +1655,7 @@ export class TaskScheduler {
|
|
|
1142
1655
|
if (invocation.ended)
|
|
1143
1656
|
throw endedError(invocation);
|
|
1144
1657
|
if (this.#closing)
|
|
1145
|
-
throw closedError();
|
|
1658
|
+
throw closedError(this.#session);
|
|
1146
1659
|
const found = this.#live.get(invocation.taskId);
|
|
1147
1660
|
if (found === undefined)
|
|
1148
1661
|
throw new Error(`Task ${invocation.taskId} is terminal`);
|
|
@@ -1186,6 +1699,11 @@ export class TaskScheduler {
|
|
|
1186
1699
|
return watch;
|
|
1187
1700
|
}
|
|
1188
1701
|
}
|
|
1702
|
+
/** The record with an abort mark: `restart` for an abandonment; without a reason, a request, which replaces `restart`. */
|
|
1703
|
+
function withAbortMark(record, reason) {
|
|
1704
|
+
const { abortReason: _reason, ...rest } = record;
|
|
1705
|
+
return { ...rest, abortRequested: true, ...(reason === undefined ? {} : { abortReason: reason }) };
|
|
1706
|
+
}
|
|
1189
1707
|
/** A live owner's durable cancellation intent: its abort mark, or a held outcome other than `completed`. */
|
|
1190
1708
|
function cancellationIntent(record) {
|
|
1191
1709
|
return record.state.status !== "terminal" && (record.abortRequested || failedOutcome(record));
|
|
@@ -1210,6 +1728,21 @@ function overlayOf(tx) {
|
|
|
1210
1728
|
const edges = new Map(tx.stagedConversations().map((record) => [record.id, record.owner?.taskId ?? null]));
|
|
1211
1729
|
return { tasks, edges };
|
|
1212
1730
|
}
|
|
1731
|
+
function addTo(map, key, value) {
|
|
1732
|
+
const values = map.get(key);
|
|
1733
|
+
if (values === undefined)
|
|
1734
|
+
map.set(key, new Set([value]));
|
|
1735
|
+
else
|
|
1736
|
+
values.add(value);
|
|
1737
|
+
}
|
|
1738
|
+
function deleteFrom(map, key, value) {
|
|
1739
|
+
const values = map.get(key);
|
|
1740
|
+
if (values === undefined)
|
|
1741
|
+
return;
|
|
1742
|
+
values.delete(value);
|
|
1743
|
+
if (values.size === 0)
|
|
1744
|
+
map.delete(key);
|
|
1745
|
+
}
|
|
1213
1746
|
/** Own memo entry only; memo names such as `toString` must not resolve to inherited properties. */
|
|
1214
1747
|
function memoOf(record, name) {
|
|
1215
1748
|
const memos = record?.memos;
|