@earendil-works/pi-durable 1.1.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/CHANGELOG.md +40 -0
  2. package/README.md +147 -11
  3. package/dist/codemode/codemode.d.ts +173 -0
  4. package/dist/codemode/codemode.d.ts.map +1 -0
  5. package/dist/codemode/codemode.js +886 -0
  6. package/dist/codemode/codemode.js.map +1 -0
  7. package/dist/codemode/describe.d.ts +38 -0
  8. package/dist/codemode/describe.d.ts.map +1 -0
  9. package/dist/codemode/describe.js +119 -0
  10. package/dist/codemode/describe.js.map +1 -0
  11. package/dist/codemode/index.d.ts +3 -0
  12. package/dist/codemode/index.d.ts.map +1 -0
  13. package/dist/codemode/index.js +3 -0
  14. package/dist/codemode/index.js.map +1 -0
  15. package/dist/codemode/models.d.ts +28 -0
  16. package/dist/codemode/models.d.ts.map +1 -0
  17. package/dist/codemode/models.js +231 -0
  18. package/dist/codemode/models.js.map +1 -0
  19. package/dist/codemode/output.d.ts +41 -0
  20. package/dist/codemode/output.d.ts.map +1 -0
  21. package/dist/codemode/output.js +167 -0
  22. package/dist/codemode/output.js.map +1 -0
  23. package/dist/env/index.d.ts +42 -0
  24. package/dist/env/index.d.ts.map +1 -1
  25. package/dist/env/index.js.map +1 -1
  26. package/dist/errors.d.ts +20 -3
  27. package/dist/errors.d.ts.map +1 -1
  28. package/dist/errors.js +34 -5
  29. package/dist/errors.js.map +1 -1
  30. package/dist/harness/agent.d.ts +5 -2
  31. package/dist/harness/agent.d.ts.map +1 -1
  32. package/dist/harness/agent.js +69 -25
  33. package/dist/harness/agent.js.map +1 -1
  34. package/dist/harness/compaction.d.ts +6 -1
  35. package/dist/harness/compaction.d.ts.map +1 -1
  36. package/dist/harness/compaction.js +10 -7
  37. package/dist/harness/compaction.js.map +1 -1
  38. package/dist/harness/events.d.ts +35 -15
  39. package/dist/harness/events.d.ts.map +1 -1
  40. package/dist/harness/events.js +77 -23
  41. package/dist/harness/events.js.map +1 -1
  42. package/dist/harness/generation.d.ts +2 -0
  43. package/dist/harness/generation.d.ts.map +1 -1
  44. package/dist/harness/generation.js +31 -8
  45. package/dist/harness/generation.js.map +1 -1
  46. package/dist/harness/harness.d.ts +7 -0
  47. package/dist/harness/harness.d.ts.map +1 -1
  48. package/dist/harness/harness.js +53 -26
  49. package/dist/harness/harness.js.map +1 -1
  50. package/dist/harness/inbox.d.ts +16 -4
  51. package/dist/harness/inbox.d.ts.map +1 -1
  52. package/dist/harness/inbox.js +26 -3
  53. package/dist/harness/inbox.js.map +1 -1
  54. package/dist/harness/live.d.ts +74 -17
  55. package/dist/harness/live.d.ts.map +1 -1
  56. package/dist/harness/live.js +66 -14
  57. package/dist/harness/live.js.map +1 -1
  58. package/dist/harness/scheduler.d.ts +18 -5
  59. package/dist/harness/scheduler.d.ts.map +1 -1
  60. package/dist/harness/scheduler.js +696 -163
  61. package/dist/harness/scheduler.js.map +1 -1
  62. package/dist/harness/submissions.d.ts +7 -4
  63. package/dist/harness/submissions.d.ts.map +1 -1
  64. package/dist/harness/submissions.js +32 -13
  65. package/dist/harness/submissions.js.map +1 -1
  66. package/dist/harness/task-graph.d.ts +1 -1
  67. package/dist/harness/task-graph.d.ts.map +1 -1
  68. package/dist/harness/task-graph.js +15 -10
  69. package/dist/harness/task-graph.js.map +1 -1
  70. package/dist/harness/tool.d.ts +45 -5
  71. package/dist/harness/tool.d.ts.map +1 -1
  72. package/dist/harness/tool.js +377 -62
  73. package/dist/harness/tool.js.map +1 -1
  74. package/dist/harness/types.d.ts +153 -13
  75. package/dist/harness/types.d.ts.map +1 -1
  76. package/dist/harness/types.js.map +1 -1
  77. package/dist/harness/util.d.ts +6 -1
  78. package/dist/harness/util.d.ts.map +1 -1
  79. package/dist/harness/util.js +5 -2
  80. package/dist/harness/util.js.map +1 -1
  81. package/dist/harness/view.d.ts +6 -2
  82. package/dist/harness/view.d.ts.map +1 -1
  83. package/dist/harness/view.js +22 -12
  84. package/dist/harness/view.js.map +1 -1
  85. package/dist/images/cloudflare.d.ts +4 -0
  86. package/dist/images/cloudflare.d.ts.map +1 -0
  87. package/dist/images/cloudflare.js +9 -0
  88. package/dist/images/cloudflare.js.map +1 -0
  89. package/dist/images/exif.d.ts +3 -0
  90. package/dist/images/exif.d.ts.map +1 -0
  91. package/dist/images/exif.js +71 -0
  92. package/dist/images/exif.js.map +1 -0
  93. package/dist/images/index.d.ts +18 -0
  94. package/dist/images/index.d.ts.map +1 -0
  95. package/dist/images/index.js +125 -0
  96. package/dist/images/index.js.map +1 -0
  97. package/dist/images/node-worker.d.ts +17 -0
  98. package/dist/images/node-worker.d.ts.map +1 -0
  99. package/dist/images/node-worker.js +16 -0
  100. package/dist/images/node-worker.js.map +1 -0
  101. package/dist/images/node.d.ts +21 -0
  102. package/dist/images/node.d.ts.map +1 -0
  103. package/dist/images/node.js +121 -0
  104. package/dist/images/node.js.map +1 -0
  105. package/dist/index.d.ts +6 -6
  106. package/dist/index.d.ts.map +1 -1
  107. package/dist/index.js +3 -3
  108. package/dist/index.js.map +1 -1
  109. package/dist/search.d.ts +13 -0
  110. package/dist/search.d.ts.map +1 -0
  111. package/dist/search.js +85 -0
  112. package/dist/search.js.map +1 -0
  113. package/dist/session/observation.d.ts +10 -3
  114. package/dist/session/observation.d.ts.map +1 -1
  115. package/dist/session/observation.js +24 -34
  116. package/dist/session/observation.js.map +1 -1
  117. package/dist/session/session.d.ts +68 -5
  118. package/dist/session/session.d.ts.map +1 -1
  119. package/dist/session/session.js +461 -49
  120. package/dist/session/session.js.map +1 -1
  121. package/dist/session/transaction.d.ts.map +1 -1
  122. package/dist/session/transaction.js +1 -0
  123. package/dist/session/transaction.js.map +1 -1
  124. package/dist/storage/jsonl/storage.d.ts +4 -0
  125. package/dist/storage/jsonl/storage.d.ts.map +1 -1
  126. package/dist/storage/jsonl/storage.js +2 -1
  127. package/dist/storage/jsonl/storage.js.map +1 -1
  128. package/dist/storage/memory.d.ts.map +1 -1
  129. package/dist/storage/memory.js +8 -10
  130. package/dist/storage/memory.js.map +1 -1
  131. package/dist/storage/scan.d.ts.map +1 -1
  132. package/dist/storage/scan.js +5 -4
  133. package/dist/storage/scan.js.map +1 -1
  134. package/dist/storage/sqlite/storage.d.ts.map +1 -1
  135. package/dist/storage/sqlite/storage.js +10 -12
  136. package/dist/storage/sqlite/storage.js.map +1 -1
  137. package/dist/testing/env-conformance.d.ts.map +1 -1
  138. package/dist/testing/env-conformance.js +100 -0
  139. package/dist/testing/env-conformance.js.map +1 -1
  140. package/dist/testing/runner.d.ts +1 -1
  141. package/dist/testing/runner.d.ts.map +1 -1
  142. package/dist/testing/runner.js.map +1 -1
  143. package/dist/testing/storage-conformance.d.ts.map +1 -1
  144. package/dist/testing/storage-conformance.js +3 -2
  145. package/dist/testing/storage-conformance.js.map +1 -1
  146. package/dist/testing/types.d.ts +2 -0
  147. package/dist/testing/types.d.ts.map +1 -1
  148. package/dist/testing/types.js.map +1 -1
  149. package/dist/tool-search/index.d.ts +2 -0
  150. package/dist/tool-search/index.d.ts.map +1 -0
  151. package/dist/tool-search/index.js +2 -0
  152. package/dist/tool-search/index.js.map +1 -0
  153. package/dist/tool-search/tool-search.d.ts +23 -0
  154. package/dist/tool-search/tool-search.d.ts.map +1 -0
  155. package/dist/tool-search/tool-search.js +84 -0
  156. package/dist/tool-search/tool-search.js.map +1 -0
  157. package/dist/tools/bash.d.ts +10 -2
  158. package/dist/tools/bash.d.ts.map +1 -1
  159. package/dist/tools/bash.js +27 -9
  160. package/dist/tools/bash.js.map +1 -1
  161. package/dist/tools/edit.js +1 -1
  162. package/dist/tools/edit.js.map +1 -1
  163. package/dist/tools/image-processor.d.ts +47 -0
  164. package/dist/tools/image-processor.d.ts.map +1 -0
  165. package/dist/tools/image-processor.js +30 -0
  166. package/dist/tools/image-processor.js.map +1 -0
  167. package/dist/tools/image.d.ts +8 -0
  168. package/dist/tools/image.d.ts.map +1 -1
  169. package/dist/tools/image.js +47 -0
  170. package/dist/tools/image.js.map +1 -1
  171. package/dist/tools/index.d.ts +25 -2
  172. package/dist/tools/index.d.ts.map +1 -1
  173. package/dist/tools/index.js +13 -5
  174. package/dist/tools/index.js.map +1 -1
  175. package/dist/tools/read.d.ts +10 -2
  176. package/dist/tools/read.d.ts.map +1 -1
  177. package/dist/tools/read.js +87 -28
  178. package/dist/tools/read.js.map +1 -1
  179. package/dist/tools/write.js +1 -1
  180. package/dist/tools/write.js.map +1 -1
  181. package/dist/types.d.ts +59 -5
  182. package/dist/types.d.ts.map +1 -1
  183. package/dist/types.js.map +1 -1
  184. package/package.json +43 -4
@@ -1,5 +1,6 @@
1
1
  import { copyJson } from "@earendil-works/chord";
2
2
  import { awaitWithContext, withAbortSignal } from "@earendil-works/chord/context";
3
+ import { errorMessage } from "../errors.js";
3
4
  import { agentHooks } from "./agent.js";
4
5
  import { readContextFrom } from "./context.js";
5
6
  import { closedError, scanAll, Waiters } from "./util.js";
@@ -7,6 +8,16 @@ const SCAN_PAGE_SIZE = 256;
7
8
  /** Longest delay `setTimeout` supports; longer sleeps wait in several steps. */
8
9
  const MAX_TIMER_DELAY = 2_147_483_647;
9
10
  const LIVE_STATUSES = ["pending", "running", "waiting", "completing"];
11
+ function conversationNode(id) {
12
+ return `c${id}`;
13
+ }
14
+ function conversationOf(node) {
15
+ return Number(node.slice(1));
16
+ }
17
+ /** The node a task hangs from: its owner task, or its conversation. */
18
+ function parentNode(node) {
19
+ return node.owner ?? conversationNode(node.conversationId);
20
+ }
10
21
  /**
11
22
  * Durable task scheduler of one Harness.
12
23
  *
@@ -14,8 +25,14 @@ const LIVE_STATUSES = ["pending", "running", "waiting", "completing"];
14
25
  * commit listener updates it on the Session line, so code running on the line reads exactly the committed state from it.
15
26
  *
16
27
  * Tasks and conversations form one ownership tree (spec §5.5): a task's parent is its owner task, or its conversation;
17
- * a conversation's parent is its owner task, if any. Walks up that tree decide cascades, idle scopes, and whether a
18
- * task's ordinary owned work is live, which holds its outcome as `completing` and delays its abort handler.
28
+ * a conversation's parent is its owner task, if any. Walks up that tree decide what reaches one task: a cancelling
29
+ * owner, a scope. `#below` lists the tree downward around live work, so what lies below one node is a walk down from
30
+ * it: a task's live ordinary owned work, which holds its outcome as `completing` and delays its abort handler, a
31
+ * scope's tasks, a cascade's reach. No pass visits every live task; the sets below hold each pass's candidates.
32
+ *
33
+ * The ownership indexes are bounded by live work: an owner edge and an ended task's ownership fields are kept only while
34
+ * something is listed below them in `#below`, so only the chains above live tasks are known, whole. Work that appears
35
+ * later below a dropped node finds it unknown and loads its chain from storage again.
19
36
  *
20
37
  * Invariant: every task transition is decided and written by one callback serialized on the Session line. That covers
21
38
  * reservation, marks, runtime commits, finalization, and the synchronous step before each phase, which applies the
@@ -51,18 +68,43 @@ export class TaskScheduler {
51
68
  #idleWaiters = new Waiters();
52
69
  /** Definition whose migration failed per task; retried only once the registry resolves another definition. */
53
70
  #failedMigrations = new Map();
54
- /** Owner task of each loaded conversation, `null` when ownerless. */
71
+ /** Owner task of each known conversation, `null` when ownerless. */
55
72
  #edges = new Map();
56
- /** Tasks that own a loaded conversation; their nodes stay in `#settled` once terminal. */
57
- #conversationOwners = new Set();
58
- /** Ownership fields of terminal tasks that walks pass through. */
73
+ /** Ownership fields of the known terminal tasks, which walks pass through. */
59
74
  #settled = new Map();
75
+ /**
76
+ * Ended tasks and conversations found unneeded, dropped by `#sweep()` at the end of the commit's publication unless
77
+ * needed again by then: a chain load installs nodes before the work below them is listed.
78
+ */
79
+ #dropQueue = new Set();
80
+ /**
81
+ * The tree downward around live work: each node's child nodes that are live tasks or have nodes listed below them.
82
+ * A conversation's parent is its owner task; ownerless ones with work are in `#roots`.
83
+ */
84
+ #below = new Map();
85
+ #roots = new Set();
86
+ /** Known tasks whose chain may not be loaded; every other known task's chain is, and stays, loaded. */
87
+ #unloaded = new Set();
88
+ /** Live tasks with cancellation intent, the sources of abort cascades (spec §5.4). */
89
+ #intent = new Set();
90
+ /** Live tasks that are not `completing`, the candidates of a reservation pass; ones found waiting leave it. */
91
+ #runnable = new Set();
92
+ /** `completing` tasks the next finalize pass checks: those that started holding, and holders of ended work. */
93
+ #finalizeChecks = new Set();
94
+ /** Live waiting tasks by each task in their `on`. */
95
+ #waiters = new Map();
60
96
  /**
61
97
  * `failFast` waiters the next reconcile checks for a failed task in `on`: at open, when they start waiting, and when
62
98
  * one of their tasks fails.
63
99
  */
64
100
  #failFastChecks = new Set();
101
+ /**
102
+ * Live tasks with `abandonOnRestart` found at open, from an earlier Harness. The first reservation pass abort-marks
103
+ * them with reason `restart`, so nothing of them or below them runs a phase again.
104
+ */
105
+ #abandoned = new Set();
65
106
  #reconcileScheduled = false;
107
+ #sweepScheduled = false;
66
108
  #cascadePending = false;
67
109
  #unsubscribeRegistry = () => { };
68
110
  #enabled = false;
@@ -86,7 +128,7 @@ export class TaskScheduler {
86
128
  }
87
129
  /** Load live tasks and change surviving `running` tasks back to `pending`. Dispatches nothing. */
88
130
  async open(context) {
89
- this.#session.subscribeCommits((publication) => this.#observe(publication));
131
+ this.#session.observeCommits((publication) => this.#observe(publication));
90
132
  this.#session.subscribeClose(() => this.#seal());
91
133
  this.#unsubscribeRegistry = this.#registry.subscribe(() => this.#kick());
92
134
  await this.#session.commitWith(async (tx) => {
@@ -97,7 +139,9 @@ export class TaskScheduler {
97
139
  }
98
140
  for (const records of scans) {
99
141
  for (const record of records) {
100
- this.#live.set(record.id, record);
142
+ this.#track(undefined, record);
143
+ if (record.abandonOnRestart === true)
144
+ this.#abandoned.add(record.id);
101
145
  if (record.state.status === "running") {
102
146
  tx.setTask(withState(record, { status: "pending", checkpoint: record.state.checkpoint }));
103
147
  }
@@ -116,16 +160,23 @@ export class TaskScheduler {
116
160
  this.#enabled = true;
117
161
  this.#kick();
118
162
  }
119
- /** Wait for every invocation signalled by `#seal()`. Writes nothing. */
163
+ /**
164
+ * Signal every invocation and wait for them; writes nothing. Close calls this after every close listener has run, so
165
+ * waits, watches, and streams a task holds end with the Session's reason, not as cancelled by its signal.
166
+ */
120
167
  async join() {
168
+ for (const invocation of this.#invocations.values())
169
+ invocation.controller.abort();
121
170
  await Promise.allSettled([...this.#invocations.values()].map((invocation) => invocation.done));
122
171
  }
123
172
  /**
124
173
  * Commit the abort mark, or settle a task that no registered definition can take as `orphaned` when nothing it owns
125
174
  * is live, then join the run invocation seen on the line; the commit listener signalled it. The abort invocation
126
- * starts once the task's ordinary owned work is gone. A `completing` task is only marked.
175
+ * starts once the task's ordinary owned work is gone. A `completing` task is only marked. A request replaces a
176
+ * `restart` mark, so a task abandoned after a restart that waits for its definition is orphaned; with `keepRestart`,
177
+ * as for an owner's own cleanup, such a task keeps its mark and waits on.
127
178
  */
128
- async abort(id, context) {
179
+ async abort(id, context, keepRestart = false) {
129
180
  const marked = await this.#session.commitWith(async (tx) => {
130
181
  const current = await tx.task(id);
131
182
  if (current === undefined)
@@ -133,9 +184,12 @@ export class TaskScheduler {
133
184
  if (current.state.status === "terminal")
134
185
  return { result: "terminal" };
135
186
  const invocation = this.#invocations.get(id);
187
+ // A restart-marked task has no run invocation; it only waits for its abort.
188
+ if (keepRestart && current.abortReason === "restart")
189
+ return { result: "marked" };
136
190
  if (invocation === undefined && current.state.status !== "completing") {
137
191
  await this.#loadScopes(false);
138
- if (!this.#ownedLive().has(id)) {
192
+ if (!this.#hasOwnedLive(id)) {
139
193
  const resolution = this.#resolve(current, this.#registry.snapshot());
140
194
  if (resolution.kind === "blocked") {
141
195
  await this.#terminate(tx, current, { status: "orphaned", reason: resolution.reason });
@@ -143,20 +197,22 @@ export class TaskScheduler {
143
197
  }
144
198
  }
145
199
  }
146
- if (!current.abortRequested)
147
- tx.setTask({ ...current, abortRequested: true });
200
+ if (!current.abortRequested || current.abortReason !== undefined)
201
+ tx.setTask(withAbortMark(current));
148
202
  return { result: "marked", run: invocation?.mode === "run" ? invocation : undefined };
149
203
  }, context);
150
- // The commit listener signalled the run; join it.
151
- if (marked.run !== undefined)
152
- await awaitWithContext(marked.run.done, context);
204
+ // The commit listener signalled the run; join it. A run that ignores its signal can outlive a failed Session,
205
+ // which ends the wait instead.
206
+ if (marked.run !== undefined) {
207
+ await awaitWithContext(Promise.race([marked.run.done, this.#session.failed]), context);
208
+ }
153
209
  return marked.result;
154
210
  }
155
211
  async waitForTask(id, context) {
156
212
  // Check and register on the line so no terminal publication falls between them.
157
213
  const found = await this.#session.readOnLine(async () => {
158
214
  if (this.#closing)
159
- throw closedError();
215
+ throw closedError(this.#session);
160
216
  if (this.#live.has(id))
161
217
  return { promise: this.#taskWaiters.add(id, context) };
162
218
  const record = await this.#storage.task(id, context);
@@ -172,7 +228,7 @@ export class TaskScheduler {
172
228
  */
173
229
  waitForIdle(conversationId, context) {
174
230
  if (this.#closing)
175
- return Promise.reject(closedError());
231
+ return Promise.reject(closedError(this.#session));
176
232
  if (this.#idle(conversationId))
177
233
  return Promise.resolve();
178
234
  this.#scheduleReconcile();
@@ -188,14 +244,13 @@ export class TaskScheduler {
188
244
  const queued = await this.#loadScopes(true);
189
245
  const scope = { conversation: conversationId };
190
246
  const reached = [];
191
- for (const record of this.#live.values()) {
192
- if (record.background && !background)
247
+ const walk = this.#walkDown(conversationNode(conversationId), { crossBackground: background });
248
+ for (const { node, live } of walk) {
249
+ if (live === undefined || (node.background && !background))
193
250
  continue;
194
- if (this.#inScope(parentOf(record), scope, background) !== true)
195
- continue;
196
- reached.push(record.id);
197
- if (!record.abortRequested)
198
- tx.setTask({ ...record, abortRequested: true });
251
+ reached.push(live.id);
252
+ if (!live.abortRequested || live.abortReason !== undefined)
253
+ tx.setTask(withAbortMark(live));
199
254
  }
200
255
  for (const id of queued) {
201
256
  if (this.#inScope({ conversation: id }, scope, background) === true)
@@ -213,6 +268,12 @@ export class TaskScheduler {
213
268
  const updated = [];
214
269
  const failed = [];
215
270
  let changed = false;
271
+ // Edges first, so the tasks a commit creates in a new conversation hang from it.
272
+ for (const change of publication.changes) {
273
+ if (change.type === "conversation" && !this.#edges.has(change.value.id)) {
274
+ this.#setEdge(change.value.id, change.value.owner?.taskId ?? null);
275
+ }
276
+ }
216
277
  for (const change of publication.changes) {
217
278
  if (change.type !== "task")
218
279
  continue;
@@ -221,12 +282,10 @@ export class TaskScheduler {
221
282
  const previous = this.#live.get(record.id);
222
283
  if (failedOutcome(record) && (previous === undefined || !failedOutcome(previous)))
223
284
  failed.push(record.id);
285
+ this.#track(previous, record);
224
286
  if (record.state.status === "terminal") {
225
- this.#live.delete(record.id);
226
287
  this.#failedMigrations.delete(record.id);
227
288
  this.#failFastChecks.delete(record.id);
228
- if (this.#conversationOwners.has(record.id))
229
- this.#settled.set(record.id, nodeOf(record));
230
289
  this.#taskWaiters.resolve(record.id, record);
231
290
  // Its owner may finalize now.
232
291
  this.#scheduleReconcile();
@@ -239,6 +298,9 @@ export class TaskScheduler {
239
298
  if (invocation?.mode === "run")
240
299
  invocation.controller.abort();
241
300
  }
301
+ // A request replacing a `restart` mark upgrades the marks of the work below.
302
+ if (previous?.abortReason !== undefined && record.abortReason === undefined)
303
+ this.#cascadePending = true;
242
304
  const status = record.state.status;
243
305
  if (status === "completing" && previous?.state.status !== "completing") {
244
306
  if (cancellationIntent(record))
@@ -249,42 +311,43 @@ export class TaskScheduler {
249
311
  this.#failFastChecks.add(record.id);
250
312
  this.#scheduleReconcile();
251
313
  }
252
- this.#live.set(record.id, record);
253
- updated.push(record);
314
+ // Its owners and place never change: what is above it is checked once, when it becomes live.
315
+ if (previous === undefined)
316
+ updated.push(record);
254
317
  }
255
318
  for (const id of failed) {
256
- for (const record of this.#live.values()) {
257
- const state = record.state;
258
- if (state.status === "waiting" && state.policy === "failFast" && state.on.includes(id)) {
259
- this.#failFastChecks.add(record.id);
319
+ for (const waiter of this.#waiters.get(id) ?? []) {
320
+ const state = this.#live.get(waiter)?.state;
321
+ if (state?.status === "waiting" && state.policy === "failFast") {
322
+ this.#failFastChecks.add(waiter);
260
323
  this.#scheduleReconcile();
261
324
  }
262
325
  }
263
326
  }
264
- for (const change of publication.changes) {
265
- if (change.type === "conversation" && !this.#edges.has(change.value.id)) {
266
- this.#setEdge(change.value.id, change.value.owner?.taskId ?? null);
267
- }
268
- }
269
327
  for (const change of publication.changes) {
270
328
  // A queued input below a cancelled owner is withdrawn, even after its cascade.
271
329
  if (change.type !== "submission" || change.value.status !== "queued" || change.value.type !== "input")
272
330
  continue;
273
- const up = { conversation: change.value.conversationId };
274
- if (!this.#chainKnown(up) || this.#belowCancelled(up))
331
+ // Only an owner with cancellation intent withdraws it; a later intent's mark runs the cascade then.
332
+ if (this.#intent.size === 0)
333
+ continue;
334
+ const conversation = change.value.conversationId;
335
+ if (!this.#known(conversationNode(conversation)) || this.#belowCancelled({ conversation })) {
275
336
  this.#cascadePending = true;
337
+ }
276
338
  }
277
339
  for (const record of updated) {
278
340
  // Work created below a cancelled owner, even after its cascade, is aborted too.
279
- if (!this.#chainKnown(parentOf(record)))
341
+ if (this.#unloaded.has(record.id))
280
342
  this.#scheduleReconcile();
281
343
  else if (!record.background && !record.abortRequested && this.#belowCancelled(parentOf(record))) {
282
344
  this.#cascadePending = true;
283
345
  }
284
346
  }
285
- // Also retries, with the next commit of any kind, a cascade whose commit failed.
347
+ // A cascade that a reservation or step found pending runs with the next commit of any kind.
286
348
  if (this.#cascadePending)
287
349
  this.#scheduleReconcile();
350
+ this.#sweep();
288
351
  if (!changed)
289
352
  return;
290
353
  this.#settleIdle();
@@ -383,17 +446,28 @@ export class TaskScheduler {
383
446
  if (this.#closing)
384
447
  return;
385
448
  const queued = await this.#loadScopes(cascade);
386
- const marked = new Set();
387
- const mark = (record) => {
388
- if (record.abortRequested || marked.has(record.id))
449
+ // A cascade from an abandoned owner passes its `restart` reason on; any other intent marks, or upgrades a
450
+ // `restart` mark, as a request. Collected first, so a request wins over a `restart` mark of the same pass.
451
+ const marks = new Map();
452
+ const mark = (record, reason) => {
453
+ const staged = marks.get(record.id);
454
+ if (staged !== undefined && staged.reason === undefined)
455
+ return;
456
+ if (record.abortRequested && (record.abortReason === undefined || reason !== undefined))
389
457
  return;
390
- marked.add(record.id);
391
- tx.setTask({ ...record, abortRequested: true });
458
+ marks.set(record.id, { record, reason });
392
459
  };
393
- // Loading edges can reveal a cancelled owner, so marks are derived on every pass.
394
- for (const record of this.#live.values()) {
395
- if (!record.background && this.#belowCancelled(parentOf(record)))
396
- mark(record);
460
+ // Every live foreground task below a live owner with cancellation intent, which is its nearest one: a walk down
461
+ // from each owner with intent does not enter another, whose own walk covers what is below it. Derived again on
462
+ // every pass, as loading edges can reveal a cancelled owner.
463
+ for (const id of this.#intent) {
464
+ const owner = this.#live.get(id);
465
+ // Only an owner abandoned after a restart, and nothing else, passes its reason on.
466
+ const restart = owner.abortReason === "restart" && !failedOutcome(owner);
467
+ for (const { node, live } of this.#walkDown(id, { enter: (below) => !this.#intent.has(below) })) {
468
+ if (live !== undefined && !node.background)
469
+ mark(live, restart ? "restart" : undefined);
470
+ }
397
471
  }
398
472
  for (const id of checks) {
399
473
  const waiter = this.#live.get(id);
@@ -403,9 +477,11 @@ export class TaskScheduler {
403
477
  for (const member of waiter.state.on) {
404
478
  const record = this.#live.get(member);
405
479
  if (record !== undefined && !failedOutcome(record))
406
- mark(record);
480
+ mark(record, undefined);
407
481
  }
408
482
  }
483
+ for (const { record, reason } of marks.values())
484
+ tx.setTask(withAbortMark(record, reason));
409
485
  for (const id of queued)
410
486
  if (this.#belowCancelled({ conversation: id }))
411
487
  await this.#withdrawInputs(tx, id);
@@ -413,12 +489,9 @@ export class TaskScheduler {
413
489
  }, this.#context);
414
490
  }
415
491
  catch (error) {
416
- // Any pass may have staged marks, so a failed one is retried with the next commit.
417
- this.#cascadePending = true;
418
- for (const id of checks)
419
- this.#failFastChecks.add(id);
420
- if (!this.#closing)
421
- this.#report(error);
492
+ // No extension code runs in this commit: a throw is a storage failure, a host callback, or a bug. None is fixed by
493
+ // running the pass again, so it fails the Session, which reports it.
494
+ this.#failSession(error);
422
495
  }
423
496
  this.#settleIdle();
424
497
  }
@@ -437,53 +510,86 @@ export class TaskScheduler {
437
510
  * Harness cleanup here.
438
511
  */
439
512
  async #finalize(tx) {
440
- for (;;) {
441
- const overlay = overlayOf(tx);
442
- const owned = this.#ownedLive(overlay);
443
- const done = [...this.#liveRecords(overlay)].filter((record) => record.state.status === "completing" && !owned.has(record.id));
444
- if (done.length === 0)
445
- return;
446
- for (const record of done) {
513
+ // A worklist: the tasks that started holding, and the holders of work that ended. Finalizing one can free the
514
+ // nearest live task above it, which goes on the list. Tasks finalized here have no live work below them, so walks
515
+ // down skip them.
516
+ // In rounds: the holders the tasks of one round free are checked once, in the next, so a holder of many tasks that
517
+ // end together is walked once, not once per task.
518
+ let round = [...this.#finalizeChecks];
519
+ this.#finalizeChecks.clear();
520
+ const done = new Set();
521
+ const overlay = overlayOf(tx);
522
+ while (round.length > 0) {
523
+ const next = new Set();
524
+ for (const id of round) {
525
+ const record = overlay.tasks.get(id) ?? this.#live.get(id);
526
+ if (record?.state.status !== "completing" || done.has(id))
527
+ continue;
528
+ if (this.#hasOwnedLive(id, overlay, done))
529
+ continue;
447
530
  const outcome = record.state.outcome;
448
- tx.setTask(withState(record, { status: "terminal", outcome }));
531
+ const terminal = withState(record, { status: "terminal", outcome });
532
+ tx.setTask(terminal);
533
+ overlay.tasks.set(id, terminal);
534
+ done.add(id);
449
535
  // REMINDER: only the scheduler writes `faulted` and `orphaned` (spec §5.4); their cleanup waits for this commit.
450
536
  if (outcome.status === "faulted" || outcome.status === "orphaned") {
451
537
  await this.#settleOutcome(tx, record, outcome);
452
538
  }
539
+ const above = this.#nearestLiveAbove(record, overlay);
540
+ if (above?.state.status === "completing")
541
+ next.add(above.id);
453
542
  }
543
+ round = [...next];
454
544
  }
455
545
  }
456
546
  /**
457
- * Load the owner chains of every live task and, with `queued`, of every conversation with queued submissions, on the
458
- * Session line; returns the latter. Reads committed Storage directly, so it may run inside a commit callback.
547
+ * Load the owner chains of the tasks in `#unloaded` and, with `queued`, of every conversation with queued inputs,
548
+ * on the Session line; returns the latter. Reads committed Storage directly, so it may run inside a commit callback.
459
549
  */
460
550
  async #loadScopes(queued) {
461
- for (const record of [...this.#live.values()]) {
462
- if (!this.#chainKnown(parentOf(record)))
463
- await this.#loadChain(parentOf(record));
551
+ for (const id of [...this.#unloaded]) {
552
+ // An earlier load in this loop may have passed it.
553
+ if (this.#unloaded.has(id))
554
+ await this.#loadChain({ task: id });
464
555
  }
465
556
  if (!queued)
466
557
  return [];
467
558
  const submissions = await scanAll((cursor) => this.#storage.scanSubmissions({ status: "queued" }, SCAN_PAGE_SIZE, cursor, this.#context));
468
- const conversations = [...new Set(submissions.map((submission) => submission.conversationId))];
559
+ const inputs = submissions.filter((submission) => submission.type === "input");
560
+ const conversations = [...new Set(inputs.map((submission) => submission.conversationId))];
469
561
  for (const id of conversations)
470
562
  await this.#loadChain({ conversation: id });
471
563
  return conversations;
472
564
  }
473
- /** Load the owner edges and task nodes from `start` up to its ownerless root. */
565
+ /**
566
+ * Load the owner edges and task nodes from `start` up to its ownerless root. A known task outside `#unloaded` ends the
567
+ * walk, as its chain is loaded; the `#unloaded` tasks passed have a loaded chain afterwards. Loading a node can reveal
568
+ * a cancelled owner, so it makes the next reconcile derive cascade marks.
569
+ */
474
570
  async #loadChain(start, overlay) {
571
+ const passed = [];
475
572
  let at = start;
476
573
  while (at !== undefined) {
477
574
  if ("task" in at) {
478
- let node = this.#node(at.task, overlay);
575
+ const id = at.task;
576
+ let node = this.#live.get(id) ?? this.#settled.get(id);
577
+ // Created in this commit: the load from its own record covers what is above it.
578
+ if (node === undefined && overlay?.tasks.has(id) === true)
579
+ break;
580
+ if (node !== undefined && !this.#unloaded.has(id))
581
+ break;
479
582
  if (node === undefined) {
480
- const record = await this.#storage.task(at.task, this.#context);
583
+ const record = await this.#storage.task(id, this.#context);
481
584
  if (record === undefined)
482
585
  return;
483
586
  node = nodeOf(record);
484
587
  if (record.state.status === "terminal")
485
- this.#settled.set(record.id, node);
588
+ this.#settle(id, node);
589
+ // A loaded node can reveal a cancelling owner: the next reconcile derives marks again.
590
+ this.#scheduleReconcile();
486
591
  }
592
+ passed.push(id);
487
593
  at = parentOf(node);
488
594
  }
489
595
  else {
@@ -492,15 +598,202 @@ export class TaskScheduler {
492
598
  const record = await this.#storage.conversation(at.conversation, this.#context);
493
599
  edge = record?.owner?.taskId ?? null;
494
600
  this.#setEdge(at.conversation, edge);
601
+ // A loaded node can reveal a cancelling owner: the next reconcile derives marks again.
602
+ this.#scheduleReconcile();
495
603
  }
496
604
  at = edge === null ? undefined : { task: edge };
497
605
  }
498
606
  }
607
+ for (const id of passed)
608
+ this.#unloaded.delete(id);
499
609
  }
500
610
  #setEdge(conversationId, owner) {
501
611
  this.#edges.set(conversationId, owner);
502
- if (owner !== null)
503
- this.#conversationOwners.add(owner);
612
+ // The conversation now hangs from its owner, or is a root; with nothing below it, it waits for the sweep.
613
+ this.#relist(conversationNode(conversationId));
614
+ this.#dropIfUnneeded(conversationNode(conversationId));
615
+ }
616
+ /** Know a terminal task a chain load read, so walks pass through it; its own chain is not loaded yet. */
617
+ #settle(id, node) {
618
+ this.#settled.set(id, node);
619
+ this.#unloaded.add(id);
620
+ this.#relist(id);
621
+ this.#dropIfUnneeded(id);
622
+ }
623
+ /** Queue an ended task or a conversation for the sweep when nothing is listed below it. */
624
+ #dropIfUnneeded(node) {
625
+ if (typeof node !== "string" && this.#live.has(node))
626
+ return;
627
+ if (this.#below.has(node))
628
+ return;
629
+ this.#dropQueue.add(node);
630
+ this.#scheduleSweep();
631
+ }
632
+ /**
633
+ * Sweep in a job of its own on the Session line, after the operation underway: a rejected commit, one with nothing to
634
+ * write, and a read publish nothing, so the sweep at the end of `#observe` would not run for what they loaded.
635
+ */
636
+ #scheduleSweep() {
637
+ if (this.#sweepScheduled || this.#closing)
638
+ return;
639
+ this.#sweepScheduled = true;
640
+ this.#session
641
+ .readOnLine(async () => {
642
+ this.#sweepScheduled = false;
643
+ this.#sweep();
644
+ })
645
+ // Rejected only once the Session is closing or failed, when nothing is kept anyway.
646
+ .catch(() => { });
647
+ }
648
+ /**
649
+ * Drop the queued nodes that are still unneeded: not live, nothing listed below. Runs once a publication is fully
650
+ * tracked, and as a job of its own on the line, never inside an operation: work a commit creates below a chain its
651
+ * own callback loaded keeps that chain. A flat loop: unlisting already queued every node on the way up that lost its
652
+ * last listed node.
653
+ */
654
+ #sweep() {
655
+ for (const node of this.#dropQueue) {
656
+ if (this.#below.has(node))
657
+ continue;
658
+ if (typeof node === "string") {
659
+ this.#edges.delete(conversationOf(node));
660
+ this.#roots.delete(conversationOf(node));
661
+ }
662
+ else if (!this.#live.has(node)) {
663
+ this.#settled.delete(node);
664
+ this.#unloaded.delete(node);
665
+ }
666
+ }
667
+ this.#dropQueue.clear();
668
+ }
669
+ /**
670
+ * Bring the scheduler's sets in line with a task's committed record. A task that ended may free the nearest live task
671
+ * above it: one holding its outcome finalizes, an abort-marked one runs its abort handler; its waiters may run.
672
+ */
673
+ #track(previous, record) {
674
+ const id = record.id;
675
+ if (previous?.state.status === "waiting")
676
+ for (const on of previous.state.on)
677
+ deleteFrom(this.#waiters, on, id);
678
+ if (record.state.status === "terminal") {
679
+ const node = nodeOf(record);
680
+ this.#live.delete(id);
681
+ this.#intent.delete(id);
682
+ this.#runnable.delete(id);
683
+ this.#finalizeChecks.delete(id);
684
+ this.#settled.set(id, node);
685
+ // Before anything is forgotten: the walk up to the task it held passes the ended tasks between them. Its chain is
686
+ // loaded, as every commit that ends a task loads the chains of `#unloaded` first.
687
+ this.#release(node);
688
+ for (const waiter of this.#waiters.get(id) ?? [])
689
+ this.#runnable.add(waiter);
690
+ this.#relist(id);
691
+ this.#dropIfUnneeded(id);
692
+ return;
693
+ }
694
+ this.#live.set(id, record);
695
+ if (previous === undefined) {
696
+ if (!this.#known(parentNode(record)))
697
+ this.#unloaded.add(id);
698
+ this.#relist(id);
699
+ }
700
+ if (cancellationIntent(record))
701
+ this.#intent.add(id);
702
+ else
703
+ this.#intent.delete(id);
704
+ if (record.state.status === "completing") {
705
+ this.#runnable.delete(id);
706
+ this.#finalizeChecks.add(id);
707
+ }
708
+ else {
709
+ this.#runnable.add(id);
710
+ }
711
+ if (record.state.status === "waiting")
712
+ for (const on of record.state.on)
713
+ addTo(this.#waiters, on, id);
714
+ }
715
+ /** The nearest live task above ended ordinary work may be free now: one holding its outcome, or an aborting one. */
716
+ #release(node, overlay) {
717
+ const above = this.#nearestLiveAbove(node, overlay);
718
+ if (above?.state.status === "completing")
719
+ this.#finalizeChecks.add(above.id);
720
+ else if (above?.abortRequested === true)
721
+ this.#runnable.add(above.id);
722
+ return above;
723
+ }
724
+ /**
725
+ * List `start` under its parent node in `#below` while it is a live task or has nodes listed below it, and unlist it
726
+ * otherwise; then its parent, as far up as that changes. So `#below` holds exactly the tree around live work; a
727
+ * parent that loses its last listed node is queued for the sweep. A loop, as ownership chains can be deeper than the
728
+ * call stack.
729
+ */
730
+ #relist(start) {
731
+ let node = start;
732
+ for (;;) {
733
+ const needed = (typeof node !== "string" && this.#live.has(node)) || this.#below.has(node);
734
+ let parent;
735
+ if (typeof node === "string") {
736
+ parent = this.#edges.get(conversationOf(node));
737
+ }
738
+ else {
739
+ const known = this.#live.get(node) ?? this.#settled.get(node);
740
+ if (known === undefined)
741
+ return;
742
+ parent = parentNode(known);
743
+ }
744
+ if (parent === undefined)
745
+ return;
746
+ if (parent === null) {
747
+ const root = conversationOf(node);
748
+ if (needed)
749
+ this.#roots.add(root);
750
+ else
751
+ this.#roots.delete(root);
752
+ return;
753
+ }
754
+ if ((this.#below.get(parent)?.has(node) === true) === needed)
755
+ return;
756
+ if (needed) {
757
+ const first = !this.#below.has(parent);
758
+ addTo(this.#below, parent, node);
759
+ if (!first)
760
+ return;
761
+ }
762
+ else {
763
+ deleteFrom(this.#below, parent, node);
764
+ if (this.#below.has(parent))
765
+ return;
766
+ this.#dropIfUnneeded(parent);
767
+ }
768
+ node = parent;
769
+ }
770
+ }
771
+ /**
772
+ * The nearest live task above ended ordinary work `node`, in the overlay's view: the one that work held, if any. Work
773
+ * at or below a background task holds nothing above it.
774
+ */
775
+ #nearestLiveAbove(node, overlay) {
776
+ for (let current = node; !current.background;) {
777
+ const owner = current.owner ?? this.#edge(current.conversationId, overlay);
778
+ if (owner === undefined || owner === null)
779
+ return undefined;
780
+ const record = overlay?.tasks.get(owner) ?? this.#live.get(owner);
781
+ if (record !== undefined && record.state.status !== "terminal")
782
+ return record;
783
+ const above = record ?? this.#settled.get(owner);
784
+ if (above === undefined)
785
+ return undefined;
786
+ current = above;
787
+ }
788
+ return undefined;
789
+ }
790
+ /** Whether the chain above `node` is loaded, without walking: known tasks outside `#unloaded` have a loaded chain. */
791
+ #known(node) {
792
+ if (typeof node !== "string") {
793
+ return (this.#live.has(node) || this.#settled.has(node)) && !this.#unloaded.has(node);
794
+ }
795
+ const owner = this.#edges.get(conversationOf(node));
796
+ return owner === null || (owner !== undefined && this.#known(owner));
504
797
  }
505
798
  /** Owner task of a conversation, `null` when ownerless, `undefined` while not loaded. */
506
799
  #edge(id, overlay) {
@@ -533,51 +826,96 @@ export class TaskScheduler {
533
826
  }
534
827
  }
535
828
  }
536
- /** Whether every owner above `start` is loaded. */
537
- #chainKnown(start, overlay) {
538
- for (const step of this.#above(start, overlay))
539
- if ("unknown" in step)
540
- return false;
541
- return true;
542
- }
543
- /** Live records, with the overlay's candidates replacing committed ones; terminal candidates are gone. */
544
- *#liveRecords(overlay) {
545
- for (const record of this.#live.values()) {
546
- const candidate = overlay?.tasks.get(record.id) ?? record;
547
- if (candidate.state.status !== "terminal")
548
- yield candidate;
549
- }
550
- if (overlay === undefined)
551
- return;
552
- for (const record of overlay.tasks.values()) {
553
- if (!this.#live.has(record.id) && record.state.status !== "terminal")
554
- yield record;
829
+ /**
830
+ * The tasks below `start` that a walk down reaches: those `#below` lists and, under an overlay, the commit's new tasks,
831
+ * with its candidates in place of committed records. Background tasks are reached but not entered, unless
832
+ * `crossBackground`, and neither are the tasks `enter` rejects. A loop, as chains can be deeper than the call stack.
833
+ */
834
+ *#walkDown(start, options = {}) {
835
+ const { overlay, crossBackground = false, enter } = options;
836
+ const links = overlay === undefined ? undefined : this.#links(overlay);
837
+ const pending = [start];
838
+ for (let at = pending.pop(); at !== undefined; at = pending.pop()) {
839
+ for (const children of [this.#below.get(at), links?.get(at)]) {
840
+ for (const child of children ?? []) {
841
+ if (typeof child === "string") {
842
+ pending.push(child);
843
+ continue;
844
+ }
845
+ const record = overlay?.tasks.get(child) ?? this.#live.get(child);
846
+ const node = record ?? this.#settled.get(child);
847
+ if (node === undefined)
848
+ continue;
849
+ const live = record !== undefined && record.state.status !== "terminal" ? record : undefined;
850
+ yield { id: child, node, live };
851
+ if ((crossBackground || !node.background) && (enter === undefined || enter(child)))
852
+ pending.push(child);
853
+ }
854
+ }
555
855
  }
556
856
  }
557
857
  /**
558
- * Every task with live ordinary owned work (spec §5.5), mapped to that work: each live non-background task counts for
559
- * every owner task above it up to and including the first background one. Owner chains must be loaded.
858
+ * The links walks down need for the commit's new tasks: each one under its parent node, then each node above it that
859
+ * `#below` does not list yet, until one it lists, a live task (always listed), or the chain's end.
560
860
  */
561
- #ownedLive(overlay) {
562
- const owned = new Map();
563
- for (const record of this.#liveRecords(overlay)) {
564
- if (record.background)
861
+ #links(overlay) {
862
+ if (overlay.links !== undefined)
863
+ return overlay.links;
864
+ const links = new Map();
865
+ for (const record of overlay.tasks.values()) {
866
+ if (this.#live.has(record.id) || this.#settled.has(record.id))
565
867
  continue;
566
- for (const step of this.#above(parentOf(record), overlay)) {
567
- if ("unknown" in step)
868
+ let child = record.id;
869
+ let parent = parentNode(record);
870
+ while (parent !== undefined && parent !== null && links.get(parent)?.has(child) !== true) {
871
+ addTo(links, parent, child);
872
+ if (typeof parent !== "string" && (this.#live.has(parent) || overlay.tasks.has(parent)))
568
873
  break;
569
- if (!("task" in step))
570
- continue;
571
- const below = owned.get(step.task);
572
- if (below === undefined)
573
- owned.set(step.task, [record.id]);
574
- else
575
- below.push(record.id);
576
- if (step.node.background)
874
+ let above;
875
+ if (typeof parent === "string") {
876
+ above = this.#edge(conversationOf(parent), overlay);
877
+ }
878
+ else {
879
+ const fields = this.#settled.get(parent);
880
+ above = fields === undefined ? undefined : parentNode(fields);
881
+ }
882
+ if (above !== undefined && above !== null && this.#below.get(above)?.has(parent) === true)
577
883
  break;
884
+ child = parent;
885
+ parent = above;
578
886
  }
579
887
  }
580
- return owned;
888
+ overlay.links = links;
889
+ return links;
890
+ }
891
+ /**
892
+ * The live ordinary owned work of `owner` (spec §5.5): the live non-background tasks below it that a walk down reaches
893
+ * without entering a background task; with `first`, only the first. Candidates in `overlay` replace committed records;
894
+ * the walk does not enter tasks in `done`, which have none. Owner chains must be loaded.
895
+ */
896
+ #ownedLive(owner, options = {}) {
897
+ const { overlay, first = false, done } = options;
898
+ const found = [];
899
+ const walk = this.#walkDown(owner, {
900
+ ...(overlay === undefined ? {} : { overlay }),
901
+ ...(done === undefined ? {} : { enter: (id) => !done.has(id) }),
902
+ });
903
+ for (const { id, node, live } of walk) {
904
+ if (live === undefined || node.background)
905
+ continue;
906
+ found.push(id);
907
+ if (first)
908
+ break;
909
+ }
910
+ return found;
911
+ }
912
+ #hasOwnedLive(owner, overlay, done) {
913
+ const options = {
914
+ first: true,
915
+ ...(overlay === undefined ? {} : { overlay }),
916
+ ...(done === undefined ? {} : { done }),
917
+ };
918
+ return this.#ownedLive(owner, options).length > 0;
581
919
  }
582
920
  /**
583
921
  * Whether ordinary traversal from `scope` reaches `start`: walking up reaches the scope's conversation, or an
@@ -603,32 +941,47 @@ export class TaskScheduler {
603
941
  * background owner without it. Terminal owners never cascade (spec §5.4).
604
942
  */
605
943
  #belowCancelled(start) {
944
+ return this.#cancellingOwner(start) !== undefined;
945
+ }
946
+ /** The nearest live owner above `start` whose cancellation intent reaches it, if any; see `#belowCancelled`. */
947
+ #cancellingOwner(start) {
948
+ // Nothing cancels, the common case: no walk.
949
+ if (this.#intent.size === 0)
950
+ return undefined;
606
951
  for (const step of this.#above(start)) {
607
952
  if ("unknown" in step)
608
- return false;
953
+ return undefined;
609
954
  if (!("task" in step))
610
955
  continue;
611
956
  const live = this.#live.get(step.task);
612
957
  if (live !== undefined && cancellationIntent(live))
613
- return true;
958
+ return live;
614
959
  if (step.node.background)
615
- return false;
960
+ return undefined;
616
961
  }
617
- return false;
962
+ return undefined;
963
+ }
964
+ /** Fail the Session, which reports it, for a throw in one of the scheduler's own commits; ignored once closing. */
965
+ #failSession(error) {
966
+ if (!this.#closing)
967
+ this.#session.fail(error);
618
968
  }
619
- /** Close listener: runs synchronously once admission is sealed, before `join()`. */
969
+ /** Close listener: runs synchronously once admission is sealed, before `join()`, or when the Session fails. */
620
970
  #seal() {
621
971
  this.#closing = true;
622
- this.#unsubscribeRegistry();
623
- const error = closedError();
972
+ try {
973
+ this.#unsubscribeRegistry();
974
+ }
975
+ catch (error) {
976
+ this.#report(error);
977
+ }
978
+ const error = closedError(this.#session);
624
979
  this.#taskWaiters.rejectAll(error);
625
980
  this.#idleWaiters.rejectAll(error);
626
981
  this.#contexts.clear();
627
982
  if (this.#expiry !== undefined)
628
983
  clearTimeout(this.#expiry.timer);
629
984
  this.#expiry = undefined;
630
- for (const invocation of this.#invocations.values())
631
- invocation.controller.abort();
632
985
  }
633
986
  #kick() {
634
987
  this.#dirty = true;
@@ -647,38 +1000,64 @@ export class TaskScheduler {
647
1000
  }
648
1001
  }
649
1002
  catch (error) {
650
- if (!this.#closing)
651
- this.#report(error);
1003
+ // As in `#reconcile()`: the reservation commit runs no extension code.
1004
+ this.#failSession(error);
652
1005
  }
653
1006
  finally {
654
1007
  this.#draining = false;
655
- // A wakeup that arrived during a failed pass still needs its pass.
1008
+ // A wakeup that arrived after the loop's last check still needs its pass.
656
1009
  if (this.#dirty)
657
1010
  this.#kick();
658
1011
  }
659
1012
  }
660
- /** Reserve every eligible task in one commit; orphan abort-marked tasks no definition can take. */
1013
+ /**
1014
+ * Reserve every eligible task in one commit; orphan abort-marked tasks no definition can take, unless abandoned after
1015
+ * a restart. The first pass after open only abort-marks the abandoned tasks, so later passes see the marks.
1016
+ */
661
1017
  async #reserve() {
662
1018
  const reservations = [];
1019
+ let abandoned = [];
663
1020
  try {
664
1021
  await this.#session.commitWith(async (tx) => {
665
1022
  if (!this.#enabled || this.#closing)
666
1023
  return;
1024
+ for (const id of this.#abandoned) {
1025
+ const record = this.#live.get(id);
1026
+ if (record !== undefined && !record.abortRequested)
1027
+ tx.setTask(withAbortMark(record, "restart"));
1028
+ }
1029
+ if (this.#abandoned.size > 0) {
1030
+ abandoned = [...this.#abandoned];
1031
+ return;
1032
+ }
667
1033
  await this.#loadScopes(false);
668
- const owned = this.#ownedLive();
669
1034
  // Taken once per pass, and only when some task is a candidate.
670
1035
  let snapshot;
671
- for (const record of [...this.#live.values()]) {
672
- if (this.#invocations.has(record.id) || this.#waitingOn(record, owned).length > 0)
1036
+ for (const id of [...this.#runnable]) {
1037
+ const record = this.#live.get(id);
1038
+ if (record === undefined || record.state.status === "completing" || this.#invocations.has(id)) {
1039
+ this.#runnable.delete(id);
673
1040
  continue;
674
- if (record.state.status === "completing")
1041
+ }
1042
+ // It returns once what it waits for ends, its record changes, or the work below it ends.
1043
+ if (this.#waitingOn(record, true).length > 0) {
1044
+ this.#runnable.delete(id);
675
1045
  continue;
1046
+ }
676
1047
  const runnable = record;
677
1048
  const mode = record.abortRequested ? "abort" : "run";
1049
+ // Work below an owner with cancellation intent waits for its cascade mark instead of running a phase; schedule
1050
+ // the cascade, which may not have run yet.
1051
+ if (mode === "run" && !record.background && this.#belowCancelled(parentOf(record))) {
1052
+ this.#cascadePending = true;
1053
+ this.#scheduleReconcile();
1054
+ continue;
1055
+ }
678
1056
  snapshot ??= this.#registry.snapshot();
679
1057
  const resolution = this.#resolve(runnable, snapshot);
680
1058
  if (resolution.kind === "blocked") {
681
- if (mode === "abort") {
1059
+ // An abandoned task waits for its definition, so its abort handler can clean up.
1060
+ if (mode === "abort" && record.abortReason === undefined) {
682
1061
  await this.#terminate(tx, record, { status: "orphaned", reason: resolution.reason });
683
1062
  }
684
1063
  continue;
@@ -689,7 +1068,9 @@ export class TaskScheduler {
689
1068
  checkpoint: resolution.record.state.checkpoint,
690
1069
  }));
691
1070
  }
692
- // Registered on the line, so marks and later reservations see it and close joins it.
1071
+ // Registered on the line, so marks and later reservations see it and close joins it; `#end` makes it a
1072
+ // candidate again.
1073
+ this.#runnable.delete(id);
693
1074
  const invocation = this.#createInvocation(record, mode);
694
1075
  reservations.push({ invocation, task: resolution.task, snapshot });
695
1076
  }
@@ -698,22 +1079,31 @@ export class TaskScheduler {
698
1079
  catch (error) {
699
1080
  for (const { invocation } of reservations) {
700
1081
  this.#invocations.delete(invocation.taskId);
1082
+ this.#runnable.add(invocation.taskId);
701
1083
  invocation.finish();
702
1084
  }
703
1085
  throw error;
704
1086
  }
1087
+ // Marked, or already marked or gone: done with them. Reserve again, now seeing the marks.
1088
+ for (const id of abandoned)
1089
+ this.#abandoned.delete(id);
1090
+ if (abandoned.length > 0)
1091
+ this.#dirty = true;
705
1092
  return reservations;
706
1093
  }
707
1094
  /**
708
1095
  * Live tasks a task waits for before its next invocation: its live ordinary owned work when abort-marked, since
709
1096
  * abort runs bottom-up, otherwise the live part of the `on` of a wait.
710
1097
  */
711
- #waitingOn(record, owned) {
1098
+ #waitingOn(record, first = false) {
712
1099
  if (record.abortRequested)
713
- return owned.get(record.id) ?? [];
1100
+ return this.#ownedLive(record.id, { first });
714
1101
  if (record.state.status !== "waiting")
715
1102
  return [];
716
- return record.state.on.filter((id) => this.#live.has(id));
1103
+ if (!first)
1104
+ return record.state.on.filter((id) => this.#live.has(id));
1105
+ const live = record.state.on.find((id) => this.#live.has(id));
1106
+ return live === undefined ? [] : [live];
717
1107
  }
718
1108
  /** Resolve the record's definition by kind, migrating an older stored version. */
719
1109
  #resolve(record, snapshot) {
@@ -750,6 +1140,21 @@ export class TaskScheduler {
750
1140
  return { reason: "migration_failed", error: failed.error };
751
1141
  return { task, migrates: true };
752
1142
  }
1143
+ /** Sizes of the ownership indexes, for tests that bound the scheduler's memory. */
1144
+ get indexSizes() {
1145
+ let below = 0;
1146
+ for (const children of this.#below.values())
1147
+ below += children.size;
1148
+ return {
1149
+ live: this.#live.size,
1150
+ settled: this.#settled.size,
1151
+ edges: this.#edges.size,
1152
+ below,
1153
+ roots: this.#roots.size,
1154
+ unloaded: this.#unloaded.size,
1155
+ dropQueue: this.#dropQueue.size,
1156
+ };
1157
+ }
753
1158
  /**
754
1159
  * Scheduling state and every live task with its derived state, read on the Session line. Runs no task code: a
755
1160
  * pending migration shows as `ready` with `migrates`, and only a migration the scheduler already tried, or one that
@@ -757,20 +1162,18 @@ export class TaskScheduler {
757
1162
  */
758
1163
  async inspect(snapshot) {
759
1164
  await this.#loadScopes(false);
760
- const owned = this.#ownedLive();
761
1165
  const tasks = [];
762
- for (const record of this.#live.values()) {
763
- tasks.push({ record, state: this.#inspectTask(record, snapshot, owned) });
764
- }
1166
+ for (const record of this.#live.values())
1167
+ tasks.push({ record, state: this.#inspectTask(record, snapshot) });
765
1168
  const scheduling = this.#closing ? "closing" : this.#enabled ? "running" : "paused";
766
1169
  return { scheduling, tasks };
767
1170
  }
768
- #inspectTask(record, snapshot, owned) {
1171
+ #inspectTask(record, snapshot) {
769
1172
  if (this.#invocations.has(record.id))
770
1173
  return { kind: "running" };
771
1174
  if (record.state.status === "completing")
772
1175
  return { kind: "completing" };
773
- const on = this.#waitingOn(record, owned);
1176
+ const on = this.#waitingOn(record);
774
1177
  if (on.length > 0)
775
1178
  return { kind: "waiting", on };
776
1179
  const fit = this.#fit(record, snapshot.task(record.kind));
@@ -849,6 +1252,12 @@ export class TaskScheduler {
849
1252
  // 3. abort mark: end; a fresh abort invocation starts once the task's ordinary owned work is gone.
850
1253
  if (current.abortRequested)
851
1254
  return false;
1255
+ // An owner's cancellation intent ends it too, before its cascade marks it; reservation then holds it back.
1256
+ if (!current.background && this.#belowCancelled(parentOf(current))) {
1257
+ this.#cascadePending = true;
1258
+ this.#scheduleReconcile();
1259
+ return false;
1260
+ }
852
1261
  if (previous === undefined)
853
1262
  return true;
854
1263
  // 4. uncaught error.
@@ -909,16 +1318,17 @@ export class TaskScheduler {
909
1318
  return current;
910
1319
  this.#end(invocation);
911
1320
  if (decision !== false) {
912
- const message = decision.fault instanceof Error ? decision.fault.message : String(decision.fault);
1321
+ const message = errorMessage(decision.fault);
913
1322
  await this.#terminate(tx, current, { status: "faulted", error: { message } });
914
1323
  }
915
1324
  return undefined;
916
1325
  }, this.#context);
917
1326
  }
918
1327
  catch (error) {
1328
+ // The step writes the scheduler's own decision, a fault or a terminal record; a throw there would otherwise
1329
+ // leave the task running, to be reserved and run again.
919
1330
  this.#end(invocation);
920
- if (!this.#closing)
921
- this.#report(error);
1331
+ this.#failSession(error);
922
1332
  return undefined;
923
1333
  }
924
1334
  }
@@ -928,7 +1338,7 @@ export class TaskScheduler {
928
1338
  */
929
1339
  async #terminate(tx, record, outcome) {
930
1340
  await this.#loadScopes(false);
931
- if (this.#ownedLive(overlayOf(tx)).has(record.id)) {
1341
+ if (this.#hasOwnedLive(record.id, overlayOf(tx))) {
932
1342
  tx.setTask(withState(record, { status: "completing", outcome }));
933
1343
  return;
934
1344
  }
@@ -948,7 +1358,7 @@ export class TaskScheduler {
948
1358
  await this.#loadScopes(false);
949
1359
  for (const record of overlay.tasks.values())
950
1360
  await this.#loadChain(parentOf(record), overlay);
951
- if (this.#ownedLive(overlay).has(current.id)) {
1361
+ if (this.#hasOwnedLive(current.id, overlay)) {
952
1362
  tx.setTask(withState(current, { status: "completing", outcome: next.outcome }));
953
1363
  return;
954
1364
  }
@@ -956,37 +1366,102 @@ export class TaskScheduler {
956
1366
  tx.setTask(withState(current, next));
957
1367
  }
958
1368
  /**
959
- * A wait names existing tasks other than the waiter and its owners, which could never finish first; `failFast` only
960
- * tasks the waiter owns. An abort handler cannot wait.
1369
+ * A wait names existing tasks, and `failFast` only tasks the waiter owns. It may not close a cycle in the wait-for
1370
+ * graph, whose edges run from each task to what it waits on and to its live ordinary owned work (spec §5.5); waiting
1371
+ * on itself or on an owner holding it are the shortest such cycles. An abort handler cannot wait.
1372
+ *
1373
+ * Invariant: the wait-for graph is acyclic. Only a wait adds an edge between existing tasks, as a new task has only
1374
+ * incoming edges, so checking each wait maintains it.
961
1375
  */
962
1376
  async #validateWait(tx, invocation, current, on, policy) {
963
1377
  if (invocation.mode === "abort")
964
1378
  throw new Error(`Abort handler of task ${current.id} cannot wait`);
965
1379
  const overlay = overlayOf(tx);
966
- await this.#loadChain(parentOf(current));
967
- const owners = new Set();
968
- for (const step of this.#above(parentOf(current)))
969
- if ("task" in step)
970
- owners.add(step.task);
1380
+ const targets = [];
971
1381
  for (const id of on) {
972
- if (id === current.id || owners.has(id)) {
973
- throw new Error(`Task ${current.id} cannot wait on itself or its owner ${id}`);
974
- }
975
1382
  const member = overlay.tasks.get(id) ?? this.#live.get(id) ?? (await this.#storage.task(id, this.#context));
976
1383
  if (member === undefined)
977
1384
  throw new Error(`Task ${id} does not exist`);
978
1385
  if (policy === "failFast" && member.owner !== current.id) {
979
1386
  throw new Error(`Task ${current.id} can wait failFast only on tasks it owns; ${id} is not one`);
980
1387
  }
1388
+ // A live child already holds the waiter (task-owned tasks are never background): waiting on it adds no edge. A
1389
+ // terminal task has no edges.
1390
+ if (member.state.status !== "terminal" && member.owner !== current.id)
1391
+ targets.push(id);
1392
+ }
1393
+ if (targets.length === 0)
1394
+ return;
1395
+ const cycle = await this.#waitCycle(current.id, targets, overlay);
1396
+ if (cycle !== undefined) {
1397
+ throw new Error(`Task ${current.id} cannot wait on ${cycle[0]}: ${[current.id, ...cycle].join(" -> ")} is a wait cycle`);
1398
+ }
1399
+ }
1400
+ /**
1401
+ * The path from a member of `on` back to `task` in the wait-for graph, if the wait would close a cycle (spec §5.1).
1402
+ * An edge X -> Y means X cannot settle before Y: Y is in the `on` of waiting X that is not abort-marked, or Y is live
1403
+ * ordinary owned work of X,
1404
+ * which holds X as `completing`. Each task is visited once and each subtree walked once, so a check is bounded by the
1405
+ * live work it reaches.
1406
+ */
1407
+ async #waitCycle(task, on, overlay) {
1408
+ // Owned work is found by walks down, which need the chains of every live task loaded.
1409
+ await this.#loadScopes(false);
1410
+ const from = new Map();
1411
+ const walked = new Set();
1412
+ const pending = [];
1413
+ const reach = (id, via) => {
1414
+ if (from.has(id))
1415
+ return;
1416
+ from.set(id, via);
1417
+ pending.push(id);
1418
+ };
1419
+ const path = (end) => {
1420
+ const steps = [];
1421
+ for (let at = end; at !== undefined; at = from.get(at))
1422
+ steps.unshift(at);
1423
+ return steps;
1424
+ };
1425
+ for (const id of on)
1426
+ reach(id, undefined);
1427
+ for (let id = pending.pop(); id !== undefined; id = pending.pop()) {
1428
+ if (id === task)
1429
+ return path(id);
1430
+ const record = overlay.tasks.get(id) ?? this.#live.get(id);
1431
+ if (record === undefined || record.state.status === "terminal")
1432
+ continue;
1433
+ // An abort-marked task waits only for its owned work, as abort runs bottom-up (`#waitingOn`).
1434
+ if (record.state.status === "waiting" && !record.abortRequested) {
1435
+ for (const member of record.state.on)
1436
+ reach(member, id);
1437
+ }
1438
+ if (walked.has(id))
1439
+ continue;
1440
+ walked.add(id);
1441
+ // Below a walked task the walk does not go again: its own walk covered what lies below.
1442
+ const found = [];
1443
+ const walk = this.#walkDown(id, { overlay, enter: (below) => !walked.has(below) });
1444
+ for (const { id: below, node, live } of walk) {
1445
+ if (live === undefined || node.background)
1446
+ continue;
1447
+ found.push(below);
1448
+ reach(below, id);
1449
+ }
1450
+ for (const below of found)
1451
+ walked.add(below);
981
1452
  }
1453
+ return undefined;
982
1454
  }
983
1455
  /** End an invocation: its runtime operations reject from now on, its signal aborts, its watches stop, and its task is free. */
984
1456
  #end(invocation) {
985
1457
  if (invocation.ended)
986
1458
  return;
987
1459
  invocation.ended = true;
988
- if (this.#invocations.get(invocation.taskId) === invocation)
1460
+ if (this.#invocations.get(invocation.taskId) === invocation) {
989
1461
  this.#invocations.delete(invocation.taskId);
1462
+ if (this.#live.has(invocation.taskId))
1463
+ this.#runnable.add(invocation.taskId);
1464
+ }
990
1465
  for (const watch of invocation.watches)
991
1466
  void watch.stop();
992
1467
  // Pending waits bound to the invocation, such as a tool's waitForTask(), reject with it.
@@ -995,10 +1470,20 @@ export class TaskScheduler {
995
1470
  /** No live non-background task in the scope; a task whose owner edges are not loaded yet counts as inside. */
996
1471
  #idle(conversationId) {
997
1472
  const scope = conversationId === undefined ? { roots: true } : { conversation: conversationId };
998
- for (const record of this.#live.values()) {
999
- if (!record.background && this.#inScope(parentOf(record), scope) !== false)
1473
+ // Tasks whose chain is not loaded are not listed below their scope yet; one counts as inside unless its walk up
1474
+ // decides otherwise.
1475
+ for (const id of this.#unloaded) {
1476
+ const record = this.#live.get(id);
1477
+ if (record !== undefined && !record.background && this.#inScope(parentOf(record), scope) !== false)
1000
1478
  return false;
1001
1479
  }
1480
+ const starts = conversationId === undefined ? this.#roots : [conversationId];
1481
+ for (const start of starts) {
1482
+ for (const { node, live } of this.#walkDown(conversationNode(start))) {
1483
+ if (live !== undefined && !node.background)
1484
+ return false;
1485
+ }
1486
+ }
1002
1487
  return true;
1003
1488
  }
1004
1489
  // ─── Invocation runtime ──────────────────────────────────────────────────
@@ -1070,6 +1555,34 @@ export class TaskScheduler {
1070
1555
  snapshotAsOf: ((...args) => this.#read(invocation, () => sessionMethod(this.#session, "snapshotAsOf")(...args))),
1071
1556
  getTask: ((id, context) => this.#read(invocation, () => this.#session.readOnLine(() => this.#storage.task(id, context)))),
1072
1557
  waitForTask: ((id, context) => this.#read(invocation, () => this.waitForTask(id, withAbortSignal(invocation.controller.signal, context)))),
1558
+ ownedTasks: (context) => this.#read(invocation, () => this.#session.readOnLine(async () => {
1559
+ // The step that ends the invocation may have run on the line before this.
1560
+ if (invocation.ended)
1561
+ throw endedError(invocation);
1562
+ context.abortSignal?.throwIfAborted();
1563
+ // Every live task is listed under its parent node, its owner task when it has one.
1564
+ const owned = [];
1565
+ for (const node of this.#below.get(invocation.taskId) ?? []) {
1566
+ const record = typeof node === "string" ? undefined : this.#live.get(node);
1567
+ if (record?.owner === invocation.taskId)
1568
+ owned.push(record);
1569
+ }
1570
+ // Copies, as from storage: the scheduler reads its own.
1571
+ return owned
1572
+ .sort((a, b) => a.id - b.id)
1573
+ .map((record) => copyJson(record));
1574
+ })),
1575
+ abortOwned: (id, context) => this.#read(invocation, async () => {
1576
+ const bound = withAbortSignal(invocation.controller.signal, context);
1577
+ const record = await this.#session.readOnLine(() => this.#storage.task(id, bound));
1578
+ if (record?.owner !== invocation.taskId) {
1579
+ throw new Error(`Task ${id} is not owned by task ${invocation.taskId}`);
1580
+ }
1581
+ if (record.state.status === "terminal")
1582
+ return;
1583
+ await this.abort(id, bound, true);
1584
+ await this.waitForTask(id, bound);
1585
+ }),
1073
1586
  outcomes: ((ids, context) => this.#read(invocation, () => this.#session.readOnLine(async () => {
1074
1587
  const outcomes = [];
1075
1588
  for (const id of ids) {
@@ -1142,7 +1655,7 @@ export class TaskScheduler {
1142
1655
  if (invocation.ended)
1143
1656
  throw endedError(invocation);
1144
1657
  if (this.#closing)
1145
- throw closedError();
1658
+ throw closedError(this.#session);
1146
1659
  const found = this.#live.get(invocation.taskId);
1147
1660
  if (found === undefined)
1148
1661
  throw new Error(`Task ${invocation.taskId} is terminal`);
@@ -1186,6 +1699,11 @@ export class TaskScheduler {
1186
1699
  return watch;
1187
1700
  }
1188
1701
  }
1702
+ /** The record with an abort mark: `restart` for an abandonment; without a reason, a request, which replaces `restart`. */
1703
+ function withAbortMark(record, reason) {
1704
+ const { abortReason: _reason, ...rest } = record;
1705
+ return { ...rest, abortRequested: true, ...(reason === undefined ? {} : { abortReason: reason }) };
1706
+ }
1189
1707
  /** A live owner's durable cancellation intent: its abort mark, or a held outcome other than `completed`. */
1190
1708
  function cancellationIntent(record) {
1191
1709
  return record.state.status !== "terminal" && (record.abortRequested || failedOutcome(record));
@@ -1210,6 +1728,21 @@ function overlayOf(tx) {
1210
1728
  const edges = new Map(tx.stagedConversations().map((record) => [record.id, record.owner?.taskId ?? null]));
1211
1729
  return { tasks, edges };
1212
1730
  }
1731
+ function addTo(map, key, value) {
1732
+ const values = map.get(key);
1733
+ if (values === undefined)
1734
+ map.set(key, new Set([value]));
1735
+ else
1736
+ values.add(value);
1737
+ }
1738
+ function deleteFrom(map, key, value) {
1739
+ const values = map.get(key);
1740
+ if (values === undefined)
1741
+ return;
1742
+ values.delete(value);
1743
+ if (values.size === 0)
1744
+ map.delete(key);
1745
+ }
1213
1746
  /** Own memo entry only; memo names such as `toString` must not resolve to inherited properties. */
1214
1747
  function memoOf(record, name) {
1215
1748
  const memos = record?.memos;