@earendil-works/pi-durable 1.3.0 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (257) hide show
  1. package/CHANGELOG.md +227 -0
  2. package/README.md +287 -82
  3. package/dist/codemode/codemode.d.ts +4 -4
  4. package/dist/codemode/codemode.d.ts.map +1 -1
  5. package/dist/codemode/codemode.js +21 -16
  6. package/dist/codemode/codemode.js.map +1 -1
  7. package/dist/codemode/models.d.ts.map +1 -1
  8. package/dist/codemode/models.js +6 -4
  9. package/dist/codemode/models.js.map +1 -1
  10. package/dist/documents.d.ts +1 -1
  11. package/dist/documents.d.ts.map +1 -1
  12. package/dist/documents.js.map +1 -1
  13. package/dist/env/index.d.ts +12 -10
  14. package/dist/env/index.d.ts.map +1 -1
  15. package/dist/env/index.js.map +1 -1
  16. package/dist/env/node-watch.d.ts +3 -3
  17. package/dist/env/node-watch.d.ts.map +1 -1
  18. package/dist/env/node-watch.js +18 -16
  19. package/dist/env/node-watch.js.map +1 -1
  20. package/dist/env/node.d.ts +2 -2
  21. package/dist/env/node.d.ts.map +1 -1
  22. package/dist/env/node.js +12 -4
  23. package/dist/env/node.js.map +1 -1
  24. package/dist/errors.d.ts +18 -2
  25. package/dist/errors.d.ts.map +1 -1
  26. package/dist/errors.js +26 -3
  27. package/dist/errors.js.map +1 -1
  28. package/dist/harness/agent.d.ts +13 -2
  29. package/dist/harness/agent.d.ts.map +1 -1
  30. package/dist/harness/agent.js +48 -6
  31. package/dist/harness/agent.js.map +1 -1
  32. package/dist/harness/compaction.d.ts +3 -1
  33. package/dist/harness/compaction.d.ts.map +1 -1
  34. package/dist/harness/compaction.js +38 -6
  35. package/dist/harness/compaction.js.map +1 -1
  36. package/dist/harness/context.d.ts +23 -7
  37. package/dist/harness/context.d.ts.map +1 -1
  38. package/dist/harness/context.js +71 -25
  39. package/dist/harness/context.js.map +1 -1
  40. package/dist/harness/define.d.ts +7 -3
  41. package/dist/harness/define.d.ts.map +1 -1
  42. package/dist/harness/define.js +4 -0
  43. package/dist/harness/define.js.map +1 -1
  44. package/dist/harness/generation.d.ts +38 -5
  45. package/dist/harness/generation.d.ts.map +1 -1
  46. package/dist/harness/generation.js +497 -108
  47. package/dist/harness/generation.js.map +1 -1
  48. package/dist/harness/harness.d.ts +7 -2
  49. package/dist/harness/harness.d.ts.map +1 -1
  50. package/dist/harness/harness.js +147 -57
  51. package/dist/harness/harness.js.map +1 -1
  52. package/dist/harness/inbox.d.ts +7 -3
  53. package/dist/harness/inbox.d.ts.map +1 -1
  54. package/dist/harness/inbox.js +7 -4
  55. package/dist/harness/inbox.js.map +1 -1
  56. package/dist/harness/json.d.ts +20 -4
  57. package/dist/harness/json.d.ts.map +1 -1
  58. package/dist/harness/json.js +23 -5
  59. package/dist/harness/json.js.map +1 -1
  60. package/dist/harness/live.d.ts +122 -16
  61. package/dist/harness/live.d.ts.map +1 -1
  62. package/dist/harness/live.js +93 -12
  63. package/dist/harness/live.js.map +1 -1
  64. package/dist/harness/partial-changes.d.ts +54 -0
  65. package/dist/harness/partial-changes.d.ts.map +1 -0
  66. package/dist/harness/partial-changes.js +162 -0
  67. package/dist/harness/partial-changes.js.map +1 -0
  68. package/dist/harness/prompt.d.ts +1 -1
  69. package/dist/harness/prompt.d.ts.map +1 -1
  70. package/dist/harness/prompt.js +1 -1
  71. package/dist/harness/prompt.js.map +1 -1
  72. package/dist/harness/registry-reader.d.ts +11 -0
  73. package/dist/harness/registry-reader.d.ts.map +1 -0
  74. package/dist/harness/registry-reader.js +25 -0
  75. package/dist/harness/registry-reader.js.map +1 -0
  76. package/dist/harness/scheduler.d.ts +86 -14
  77. package/dist/harness/scheduler.d.ts.map +1 -1
  78. package/dist/harness/scheduler.js +1459 -369
  79. package/dist/harness/scheduler.js.map +1 -1
  80. package/dist/harness/submissions.d.ts +23 -11
  81. package/dist/harness/submissions.d.ts.map +1 -1
  82. package/dist/harness/submissions.js +104 -36
  83. package/dist/harness/submissions.js.map +1 -1
  84. package/dist/harness/task-graph.d.ts +28 -8
  85. package/dist/harness/task-graph.d.ts.map +1 -1
  86. package/dist/harness/task-graph.js +85 -36
  87. package/dist/harness/task-graph.js.map +1 -1
  88. package/dist/harness/telemetry.d.ts +307 -0
  89. package/dist/harness/telemetry.d.ts.map +1 -0
  90. package/dist/harness/telemetry.js +136 -0
  91. package/dist/harness/telemetry.js.map +1 -0
  92. package/dist/harness/tool.d.ts +2 -0
  93. package/dist/harness/tool.d.ts.map +1 -1
  94. package/dist/harness/tool.js +223 -61
  95. package/dist/harness/tool.js.map +1 -1
  96. package/dist/harness/transcript.d.ts +72 -0
  97. package/dist/harness/transcript.d.ts.map +1 -0
  98. package/dist/harness/transcript.js +264 -0
  99. package/dist/harness/transcript.js.map +1 -0
  100. package/dist/harness/types.d.ts +302 -48
  101. package/dist/harness/types.d.ts.map +1 -1
  102. package/dist/harness/types.js.map +1 -1
  103. package/dist/harness/util.d.ts +7 -1
  104. package/dist/harness/util.d.ts.map +1 -1
  105. package/dist/harness/util.js +13 -0
  106. package/dist/harness/util.js.map +1 -1
  107. package/dist/harness/view.d.ts +43 -22
  108. package/dist/harness/view.d.ts.map +1 -1
  109. package/dist/harness/view.js +130 -71
  110. package/dist/harness/view.js.map +1 -1
  111. package/dist/index.d.ts +13 -9
  112. package/dist/index.d.ts.map +1 -1
  113. package/dist/index.js +7 -5
  114. package/dist/index.js.map +1 -1
  115. package/dist/search.d.ts +19 -3
  116. package/dist/search.d.ts.map +1 -1
  117. package/dist/search.js +107 -44
  118. package/dist/search.js.map +1 -1
  119. package/dist/session/cached-storage.d.ts +71 -0
  120. package/dist/session/cached-storage.d.ts.map +1 -0
  121. package/dist/session/cached-storage.js +269 -0
  122. package/dist/session/cached-storage.js.map +1 -0
  123. package/dist/session/conversation-tails.d.ts +33 -0
  124. package/dist/session/conversation-tails.d.ts.map +1 -0
  125. package/dist/session/conversation-tails.js +101 -0
  126. package/dist/session/conversation-tails.js.map +1 -0
  127. package/dist/session/keys.d.ts +68 -0
  128. package/dist/session/keys.d.ts.map +1 -0
  129. package/dist/session/keys.js +161 -0
  130. package/dist/session/keys.js.map +1 -0
  131. package/dist/session/session.d.ts +181 -29
  132. package/dist/session/session.d.ts.map +1 -1
  133. package/dist/session/session.js +1073 -183
  134. package/dist/session/session.js.map +1 -1
  135. package/dist/session/transaction.d.ts +86 -4
  136. package/dist/session/transaction.d.ts.map +1 -1
  137. package/dist/session/transaction.js +250 -38
  138. package/dist/session/transaction.js.map +1 -1
  139. package/dist/session/writer.d.ts +74 -0
  140. package/dist/session/writer.d.ts.map +1 -0
  141. package/dist/session/writer.js +294 -0
  142. package/dist/session/writer.js.map +1 -0
  143. package/dist/storage/indexeddb.d.ts +68 -0
  144. package/dist/storage/indexeddb.d.ts.map +1 -0
  145. package/dist/storage/indexeddb.js +881 -0
  146. package/dist/storage/indexeddb.js.map +1 -0
  147. package/dist/storage/jsonl/storage.d.ts +30 -2
  148. package/dist/storage/jsonl/storage.d.ts.map +1 -1
  149. package/dist/storage/jsonl/storage.js +88 -24
  150. package/dist/storage/jsonl/storage.js.map +1 -1
  151. package/dist/storage/memory.d.ts +7 -2
  152. package/dist/storage/memory.d.ts.map +1 -1
  153. package/dist/storage/memory.js +29 -3
  154. package/dist/storage/memory.js.map +1 -1
  155. package/dist/storage/sqlite/cloudflare.d.ts +2 -51
  156. package/dist/storage/sqlite/cloudflare.d.ts.map +1 -1
  157. package/dist/storage/sqlite/cloudflare.js +2 -133
  158. package/dist/storage/sqlite/cloudflare.js.map +1 -1
  159. package/dist/storage/sqlite/database.d.ts +1 -36
  160. package/dist/storage/sqlite/database.d.ts.map +1 -1
  161. package/dist/storage/sqlite/database.js.map +1 -1
  162. package/dist/storage/sqlite/index.d.ts +1 -1
  163. package/dist/storage/sqlite/index.d.ts.map +1 -1
  164. package/dist/storage/sqlite/index.js.map +1 -1
  165. package/dist/storage/sqlite/migrations.d.ts +2 -5
  166. package/dist/storage/sqlite/migrations.d.ts.map +1 -1
  167. package/dist/storage/sqlite/migrations.js +28 -28
  168. package/dist/storage/sqlite/migrations.js.map +1 -1
  169. package/dist/storage/sqlite/node.d.ts +2 -37
  170. package/dist/storage/sqlite/node.d.ts.map +1 -1
  171. package/dist/storage/sqlite/node.js +3 -174
  172. package/dist/storage/sqlite/node.js.map +1 -1
  173. package/dist/storage/sqlite/schema.d.ts +1 -3
  174. package/dist/storage/sqlite/schema.d.ts.map +1 -1
  175. package/dist/storage/sqlite/schema.js +2 -9
  176. package/dist/storage/sqlite/schema.js.map +1 -1
  177. package/dist/storage/sqlite/storage.d.ts +15 -3
  178. package/dist/storage/sqlite/storage.d.ts.map +1 -1
  179. package/dist/storage/sqlite/storage.js +110 -36
  180. package/dist/storage/sqlite/storage.js.map +1 -1
  181. package/dist/storage/store/commit.d.ts +33 -0
  182. package/dist/storage/store/commit.d.ts.map +1 -0
  183. package/dist/storage/store/commit.js +286 -0
  184. package/dist/storage/store/commit.js.map +1 -0
  185. package/dist/storage/store/documents.d.ts +43 -0
  186. package/dist/storage/store/documents.d.ts.map +1 -0
  187. package/dist/storage/store/documents.js +102 -0
  188. package/dist/storage/store/documents.js.map +1 -0
  189. package/dist/storage/store/index.d.ts +2 -0
  190. package/dist/storage/store/index.d.ts.map +1 -0
  191. package/dist/storage/store/index.js +2 -0
  192. package/dist/storage/store/index.js.map +1 -0
  193. package/dist/storage/store/queries.d.ts +49 -0
  194. package/dist/storage/store/queries.d.ts.map +1 -0
  195. package/dist/storage/store/queries.js +93 -0
  196. package/dist/storage/store/queries.js.map +1 -0
  197. package/dist/storage/store/storage.d.ts +59 -0
  198. package/dist/storage/store/storage.d.ts.map +1 -0
  199. package/dist/storage/store/storage.js +438 -0
  200. package/dist/storage/store/storage.js.map +1 -0
  201. package/dist/storage/store/tables.d.ts +54 -0
  202. package/dist/storage/store/tables.d.ts.map +1 -0
  203. package/dist/storage/store/tables.js +108 -0
  204. package/dist/storage/store/tables.js.map +1 -0
  205. package/dist/testing/commit-writes.d.ts +5 -0
  206. package/dist/testing/commit-writes.d.ts.map +1 -0
  207. package/dist/testing/commit-writes.js +12 -0
  208. package/dist/testing/commit-writes.js.map +1 -0
  209. package/dist/testing/env-conformance.js +7 -7
  210. package/dist/testing/env-conformance.js.map +1 -1
  211. package/dist/testing/index.d.ts +1 -0
  212. package/dist/testing/index.d.ts.map +1 -1
  213. package/dist/testing/index.js +1 -0
  214. package/dist/testing/index.js.map +1 -1
  215. package/dist/testing/storage-benchmark.d.ts.map +1 -1
  216. package/dist/testing/storage-benchmark.js +18 -17
  217. package/dist/testing/storage-benchmark.js.map +1 -1
  218. package/dist/testing/storage-conformance.d.ts.map +1 -1
  219. package/dist/testing/storage-conformance.js +369 -67
  220. package/dist/testing/storage-conformance.js.map +1 -1
  221. package/dist/timers.d.ts +21 -0
  222. package/dist/timers.d.ts.map +1 -0
  223. package/dist/timers.js +72 -0
  224. package/dist/timers.js.map +1 -0
  225. package/dist/tool-search/tool-search.d.ts +6 -12
  226. package/dist/tool-search/tool-search.d.ts.map +1 -1
  227. package/dist/tool-search/tool-search.js +7 -7
  228. package/dist/tool-search/tool-search.js.map +1 -1
  229. package/dist/tools/bash.d.ts +12 -12
  230. package/dist/tools/bash.d.ts.map +1 -1
  231. package/dist/tools/bash.js +16 -15
  232. package/dist/tools/bash.js.map +1 -1
  233. package/dist/tools/edit.d.ts +7 -7
  234. package/dist/tools/edit.d.ts.map +1 -1
  235. package/dist/tools/edit.js +9 -9
  236. package/dist/tools/edit.js.map +1 -1
  237. package/dist/tools/index.d.ts +32 -32
  238. package/dist/tools/read.d.ts +6 -6
  239. package/dist/tools/read.d.ts.map +1 -1
  240. package/dist/tools/read.js +19 -8
  241. package/dist/tools/read.js.map +1 -1
  242. package/dist/tools/write.d.ts +4 -4
  243. package/dist/tools/write.d.ts.map +1 -1
  244. package/dist/tools/write.js +5 -5
  245. package/dist/tools/write.js.map +1 -1
  246. package/dist/types.d.ts +331 -64
  247. package/dist/types.d.ts.map +1 -1
  248. package/dist/types.js.map +1 -1
  249. package/package.json +27 -10
  250. package/dist/harness/events.d.ts +0 -186
  251. package/dist/harness/events.d.ts.map +0 -1
  252. package/dist/harness/events.js +0 -373
  253. package/dist/harness/events.js.map +0 -1
  254. package/dist/session/observation.d.ts +0 -42
  255. package/dist/session/observation.d.ts.map +0 -1
  256. package/dist/session/observation.js +0 -281
  257. package/dist/session/observation.js.map +0 -1
@@ -1,13 +1,35 @@
1
1
  import { copyJson } from "@earendil-works/chord";
2
- import { awaitWithContext, withAbortSignal } from "@earendil-works/chord/context";
3
- import { errorMessage } from "../errors.js";
4
- import { agentHooks } from "./agent.js";
5
- import { readContextFrom } from "./context.js";
6
- import { closedError, scanAll, Waiters } from "./util.js";
2
+ import { awaitWithContext, withAbortSignal, withoutAbortSignal, withTelemetry } from "@earendil-works/chord/context";
3
+ import { ownedSlice } from "@earendil-works/chord/delta";
4
+ import { createTypedMetrics, createTypedSpanStarter, NOOP_TELEMETRY_CONTEXT, NOOP_TELEMETRY_METRICS, } from "@earendil-works/pi-telemetry";
5
+ import { errorMessage, ReadOnlySession } from "../errors.js";
6
+ import { ALL_KEYS, conversationKey, INTENT_KEY, ownedKey, ROOTS_KEY, taskKey } from "../session/keys.js";
7
+ import { agentHooks, DEFAULT_RESTART_LIMIT } from "./agent.js";
8
+ import { advanceRange, rangeEntry, readContextFrom } from "./context.js";
9
+ import { DURABLE_TELEMETRY_SCHEMA } from "./telemetry.js";
10
+ import { closedError, conversationEntries, optionalExtension, scanAll, Waiters } from "./util.js";
7
11
  const SCAN_PAGE_SIZE = 256;
8
- /** Longest delay `setTimeout` supports; longer sleeps wait in several steps. */
9
- const MAX_TIMER_DELAY = 2_147_483_647;
10
- const LIVE_STATUSES = ["pending", "running", "waiting", "completing"];
12
+ const NO_KEYS = [];
13
+ const LIVE_STATUSES = ["pending", "running", "waiting", "sleeping", "completing"];
14
+ /** The part of a record `blockedReason()` reads. */
15
+ export function blockedSubject(record) {
16
+ return {
17
+ id: record.id,
18
+ kind: record.kind,
19
+ version: record.version,
20
+ abortRequested: record.abortRequested,
21
+ ...(record.abortReason === undefined ? {} : { abortReason: record.abortReason }),
22
+ status: record.state.status,
23
+ };
24
+ }
25
+ /** Continue with the next phase, the first one. */
26
+ const CONTINUE = { kind: "continue" };
27
+ /** Rule 5: the phase made durable progress; `#progress()` decides once the decision stands. */
28
+ const PROGRESS = { kind: "progress" };
29
+ /** End the invocation without a write. */
30
+ const END = { kind: "end" };
31
+ /** End it: an owner's cancellation intent reaches it before its cascade did; schedule the cascade. */
32
+ const CASCADE = { kind: "cascade" };
11
33
  function conversationNode(id) {
12
34
  return `c${id}`;
13
35
  }
@@ -21,8 +43,9 @@ function parentNode(node) {
21
43
  /**
22
44
  * Durable task scheduler of one Harness.
23
45
  *
24
- * `#live` mirrors every committed non-terminal task record: pending, running, waiting, and completing. The synchronous
25
- * commit listener updates it on the Session line, so code running on the line reads exactly the committed state from it.
46
+ * `#live` mirrors every committed non-terminal task record: pending, running, waiting, sleeping, and completing. The
47
+ * synchronous commit listener updates it on the Session line, so code running on the line reads exactly the committed
48
+ * state from it.
26
49
  *
27
50
  * Tasks and conversations form one ownership tree (spec §5.5): a task's parent is its owner task, or its conversation;
28
51
  * a conversation's parent is its owner task, if any. Walks up that tree decide what reaches one task: a cancelling
@@ -35,11 +58,22 @@ function parentNode(node) {
35
58
  * later below a dropped node finds it unknown and loads its chain from storage again.
36
59
  *
37
60
  * Invariant: every task transition is decided and written by one callback serialized on the Session line. That covers
38
- * reservation, marks, runtime commits, finalization, and the synchronous step before each phase, which applies the
39
- * precedence rules and writes a fault or handover. Handlers and joins run off the line. An invocation ends inside the
40
- * step that decides its end, so a runtime commit it queued either lands before that decision or is rejected.
61
+ * reservation, marks, runtime commits, finalization, and the step before each phase, which applies the precedence rules
62
+ * and writes a fault or handover. Handlers and joins run off the line. An invocation ends inside the step that decides
63
+ * its end, so a runtime commit it queued either lands before that decision or is rejected.
64
+ *
65
+ * Decisions stay serialized; their storage overlaps (group commit). The Session stores the commits of several jobs in
66
+ * one call while the line runs on, and this memory changes only as each commit is published. So every decision reads
67
+ * it in a decision region (`#region()`): synchronous, recording the keys it reads (a task's record, the live work below
68
+ * a node, the cancellation intent set), staging nothing; when one of them has an unpublished writer, the region waits
69
+ * for its publication and runs again, and only a region that read nothing unpublished is applied. Every commit writes
70
+ * the memberships its task changes alter (`membershipKeys()`), so a walk that a pending commit would change waits.
41
71
  */
42
72
  export class TaskScheduler {
73
+ #telemetry;
74
+ #activeMetric;
75
+ #durationMetric;
76
+ #resumed = new Set();
43
77
  #session;
44
78
  #storage;
45
79
  #registry;
@@ -48,8 +82,12 @@ export class TaskScheduler {
48
82
  #settings;
49
83
  #env;
50
84
  #now;
85
+ #onWake;
86
+ #timers;
87
+ #writeAtClose;
51
88
  #report;
52
89
  #settleOutcome;
90
+ #limitsRestarts;
53
91
  #withdrawInputs;
54
92
  #conversation;
55
93
  #context;
@@ -60,7 +98,21 @@ export class TaskScheduler {
60
98
  * first idle check after `settings.contextRetentionMs` of idleness, and at close.
61
99
  */
62
100
  #contexts = new Map();
63
- /** Timer for the earliest expiry of an idle context, only where timers can be unreferenced (see `#scheduleExpiry`). */
101
+ /**
102
+ * The idle kept contexts by `idleSince`, oldest first: the expiry order, so neither an expiry nor finding the next one
103
+ * scans `#contexts`. Kept sorted also when the Harness clock goes back (see `#markIdle`).
104
+ */
105
+ #idleOrder = new Map();
106
+ /** At least the newest `idleSince` in `#idleOrder` (drops do not lower it): an older one added means the clock went back. */
107
+ #idleNewest = Number.NEGATIVE_INFINITY;
108
+ /**
109
+ * A chain load resolved what `#idle()` treats as unknown for a live task whose chain is not loaded, which can make a
110
+ * scope idle without a task change below it; the next idle check then covers every scope.
111
+ */
112
+ #unknownsChanged = false;
113
+ /** Idle checks that covered every scope (`idleState`, a test diagnostic). */
114
+ #fullIdleChecks = 0;
115
+ /** Background timer for the earliest expiry of an idle context, unless the host declined it (see `#scheduleExpiry`). */
64
116
  #expiry;
65
117
  #invocations = new Map();
66
118
  #taskWaiters = new Waiters();
@@ -103,15 +155,39 @@ export class TaskScheduler {
103
155
  * them with reason `restart`, so nothing of them or below them runs a phase again.
104
156
  */
105
157
  #abandoned = new Set();
158
+ /** Sleeping live tasks, due or not: the candidates of `#wakeDue()`. */
159
+ #sleeping = new Set();
160
+ /** Timer for the earliest sleep, without `onWake` while scheduling runs. */
161
+ #timer;
162
+ /** Last value given to `onWake`, so an unchanged one is not repeated. */
163
+ #lastWake;
164
+ #wakeScheduled = false;
165
+ #blockedListeners = new Set();
166
+ /** A registry snapshot that threw for the task graph was reported; it is read per record, so only once. */
167
+ #snapshotReported = false;
106
168
  #reconcileScheduled = false;
169
+ /** Reconcile passes underway with work (they may overlap): their commits finalize held outcomes and mark cascades. */
170
+ #reconcilingWork = 0;
107
171
  #sweepScheduled = false;
172
+ /**
173
+ * Nodes chain loads installed for jobs whose commits are not published yet, with how many such jobs: the sweep keeps
174
+ * them, as work a pending commit creates below them needs them.
175
+ */
176
+ #pins = new Map();
177
+ /** Keys the decision region underway reads (`#region()`); set only while one runs. */
178
+ #reading;
108
179
  #cascadePending = false;
109
180
  #unsubscribeRegistry = () => { };
110
181
  #enabled = false;
182
+ #readOnly;
111
183
  #closing = false;
112
184
  #dirty = false;
113
185
  #draining = false;
114
186
  constructor(options) {
187
+ this.#telemetry = options.telemetry ?? NOOP_TELEMETRY_CONTEXT;
188
+ const metrics = createTypedMetrics(this.#telemetry.metrics ?? NOOP_TELEMETRY_METRICS, [DURABLE_TELEMETRY_SCHEMA]);
189
+ this.#activeMetric = metrics.upDownCounter("pi.durable.tasks.active");
190
+ this.#durationMetric = metrics.histogram("pi.durable.phase.duration");
115
191
  this.#session = options.session;
116
192
  this.#storage = options.storage;
117
193
  this.#registry = options.registry;
@@ -120,18 +196,31 @@ export class TaskScheduler {
120
196
  this.#settings = options.settings;
121
197
  this.#env = options.env;
122
198
  this.#now = options.now;
199
+ this.#onWake = options.onWake;
200
+ this.#timers = options.timers;
201
+ this.#writeAtClose = options.writeAtClose;
123
202
  this.#report = options.report;
124
203
  this.#settleOutcome = options.settleOutcome;
204
+ this.#limitsRestarts = options.limitsRestarts;
125
205
  this.#withdrawInputs = options.withdrawInputs;
126
206
  this.#conversation = options.conversation;
127
207
  this.#context = options.context;
208
+ this.#readOnly = options.readOnly === true;
128
209
  }
129
- /** Load live tasks and change surviving `running` tasks back to `pending`. Dispatches nothing. */
210
+ /**
211
+ * Load live tasks and change surviving `running` tasks back to `pending`, counting the restart. Dispatches nothing;
212
+ * a task cut off `settings.restartLimit` times ends when it is next reserved. Read-only, it only loads them: a
213
+ * `running` task stays as committed, running in another process.
214
+ */
130
215
  async open(context) {
131
- this.#session.observeCommits((publication) => this.#observe(publication));
216
+ this.#session.followCommits((publication) => this.#observe(publication));
132
217
  this.#session.subscribeClose(() => this.#seal());
133
- this.#unsubscribeRegistry = this.#registry.subscribe(() => this.#kick());
218
+ this.#unsubscribeRegistry = this.#registry.subscribe(() => {
219
+ this.#notifyBlocked();
220
+ this.#kick();
221
+ });
134
222
  await this.#session.commitWith(async (tx) => {
223
+ await this.#published();
135
224
  // Every table read before the first write.
136
225
  const scans = [];
137
226
  for (const status of LIVE_STATUSES) {
@@ -140,10 +229,15 @@ export class TaskScheduler {
140
229
  for (const records of scans) {
141
230
  for (const record of records) {
142
231
  this.#track(undefined, record);
232
+ this.#resumed.add(record.id);
233
+ if (this.#readOnly)
234
+ continue;
143
235
  if (record.abandonOnRestart === true)
144
236
  this.#abandoned.add(record.id);
145
237
  if (record.state.status === "running") {
146
- tx.setTask(withState(record, { status: "pending", checkpoint: record.state.checkpoint }));
238
+ const restarts = (record.restarts ?? 0) + 1;
239
+ const pending = withState(record, { status: "pending", checkpoint: record.state.checkpoint });
240
+ tx.setTask({ ...pending, restarts });
147
241
  }
148
242
  if (record.state.status === "waiting" && record.state.policy === "failFast") {
149
243
  this.#failFastChecks.add(record.id);
@@ -151,23 +245,133 @@ export class TaskScheduler {
151
245
  }
152
246
  }
153
247
  }, context);
248
+ if (this.#readOnly)
249
+ return;
154
250
  // Derive abort marks a crash left unapplied below cancelled owners, and finalize held outcomes.
155
251
  this.#cascadePending = true;
156
252
  this.#scheduleReconcile();
253
+ this.#scheduleWake();
157
254
  }
158
- /** Enable scheduling. Idempotent; the kick does nothing once closing. */
255
+ /**
256
+ * Enable scheduling, and make sleeping tasks that are due runnable: a host wakeup, such as an alarm, calls this.
257
+ * Idempotent; the kick does nothing once closing.
258
+ */
159
259
  resume() {
260
+ if (this.#readOnly)
261
+ throw new ReadOnlySession("running tasks");
160
262
  this.#enabled = true;
263
+ this.#wakeDue();
161
264
  this.#kick();
162
265
  }
163
266
  /**
164
- * Signal every invocation and wait for them; writes nothing. Close calls this after every close listener has run, so
165
- * waits, watches, and streams a task holds end with the Session's reason, not as cancelled by its signal.
267
+ * Why a live task cannot run here, if it cannot: its definition is missing, too old, or failed to migrate. A running
268
+ * task, an abort-marked one, and one holding its outcome are never blocked.
269
+ */
270
+ blockedReason(record) {
271
+ const status = record.status;
272
+ if (status === "running" || status === "completing" || status === "terminal")
273
+ return undefined;
274
+ // An abort request runs its handler or orphans the task; a `restart` mark waits for the definition, blocked.
275
+ if (record.abortRequested && record.abortReason === undefined)
276
+ return undefined;
277
+ let snapshot;
278
+ try {
279
+ snapshot = this.#registry.snapshot();
280
+ }
281
+ catch (error) {
282
+ if (!this.#snapshotReported)
283
+ this.#report(error, REGISTRY);
284
+ this.#snapshotReported = true;
285
+ return undefined;
286
+ }
287
+ const fit = this.#fit(record, snapshot.task(record.kind));
288
+ if ("reason" in fit)
289
+ return fit.reason;
290
+ return fit.migrates && fit.task.definition.migrate === undefined ? "migration_failed" : undefined;
291
+ }
292
+ /** Call `listener` when blocked reasons may have changed without a commit: a registry change or a failed migration. */
293
+ subscribeBlocked(listener) {
294
+ this.#blockedListeners.add(listener);
295
+ return () => this.#blockedListeners.delete(listener);
296
+ }
297
+ /**
298
+ * When the Session next needs a process: `active` while some task can run now or is running, `at` for the earliest
299
+ * sleep not yet due. Synchronous, from memory; a task whose chain is not loaded yet counts as it would alone.
300
+ */
301
+ wake() {
302
+ // A read-only Harness runs nothing, so it never needs a process.
303
+ if (this.#readOnly)
304
+ return { active: false };
305
+ const now = this.#now();
306
+ // A reconcile pass with work to do is work too: it finalizes held outcomes and marks cascades, at open as
307
+ // recovery. A process that dies before it lands must be woken again. A pass with nothing to do is not.
308
+ const active = this.#invocations.size > 0 || this.#reconcilingWork > 0 || this.#reconcileWork() || this.#runsNow(now);
309
+ // The earliest sleep not yet due: only `#sleeping` holds sleeping tasks, so no pass visits every live task.
310
+ let at;
311
+ for (const id of this.#sleeping) {
312
+ const record = this.#live.get(id);
313
+ const state = record?.state;
314
+ if (state?.status === "sleeping" && !record.abortRequested && state.until > now) {
315
+ at = at === undefined ? state.until : Math.min(at, state.until);
316
+ }
317
+ }
318
+ return at === undefined ? { active } : { active, at };
319
+ }
320
+ /** Whether a live task can run now: the scan `wake()` needs only while no invocation runs. */
321
+ #runsNow(now) {
322
+ let snapshot;
323
+ let snapshotFailed = false;
324
+ for (const record of this.#live.values()) {
325
+ const state = record.state;
326
+ if (state.status === "completing")
327
+ continue;
328
+ if (state.status === "waiting" && !record.abortRequested && state.on.some((id) => this.#live.has(id)))
329
+ continue;
330
+ if (state.status === "sleeping" && !record.abortRequested && state.until > now)
331
+ continue;
332
+ // An abort invocation waits for the live work below it, which counts itself.
333
+ if (record.abortRequested && this.#waitingOn(record, true).length > 0)
334
+ continue;
335
+ // A task no definition can take waits for a registry change, except an abort-marked one, which is orphaned.
336
+ if (!record.abortRequested || record.abortReason !== undefined) {
337
+ if (snapshot === undefined && !snapshotFailed) {
338
+ try {
339
+ snapshot = this.#registry.snapshot();
340
+ }
341
+ catch (error) {
342
+ snapshotFailed = true;
343
+ this.#report(error, REGISTRY);
344
+ }
345
+ }
346
+ if (snapshot !== undefined && "reason" in this.#fit(record, snapshot.task(record.kind)))
347
+ continue;
348
+ }
349
+ return true;
350
+ }
351
+ return false;
352
+ }
353
+ /**
354
+ * Signal every invocation and wait for them, then write the tasks they ran back as `pending` (`writeAtClose`), as a
355
+ * close is not a crash. Close calls this after every close listener has run, so waits, watches, and streams a task
356
+ * holds end with the Session's reason, not as cancelled by its signal.
166
357
  */
167
358
  async join() {
168
- for (const invocation of this.#invocations.values())
359
+ const stopped = [...this.#invocations.values()];
360
+ for (const invocation of stopped)
169
361
  invocation.controller.abort();
170
- await Promise.allSettled([...this.#invocations.values()].map((invocation) => invocation.done));
362
+ await Promise.allSettled(stopped.map((invocation) => invocation.done));
363
+ if (stopped.length === 0)
364
+ return;
365
+ // A close is not a crash: a task it stopped is `pending` again, so the next open counts no restart for it. Read
366
+ // on the line after commits admitted before the seal, such as an abort mark, so the write keeps them. A failure
367
+ // leaves them `running`, counted as if cut off; the Storage guard fails the Session and reports it.
368
+ await this.#writeAtClose(() => stopped.flatMap((invocation) => {
369
+ const record = this.#live.get(invocation.taskId);
370
+ if (record?.state.status !== "running")
371
+ return [];
372
+ const value = withState(record, { status: "pending", checkpoint: record.state.checkpoint });
373
+ return [{ type: "task", value }];
374
+ })).catch(() => { });
171
375
  }
172
376
  /**
173
377
  * Commit the abort mark, or settle a task that no registered definition can take as `orphaned` when nothing it owns
@@ -189,12 +393,14 @@ export class TaskScheduler {
189
393
  return { result: "marked" };
190
394
  if (invocation === undefined && current.state.status !== "completing") {
191
395
  await this.#loadScopes(false);
192
- if (!this.#hasOwnedLive(id)) {
193
- const resolution = this.#resolve(current, this.#registry.snapshot());
194
- if (resolution.kind === "blocked") {
195
- await this.#terminate(tx, current, { status: "orphaned", reason: resolution.reason });
196
- return { result: "marked" };
197
- }
396
+ const owns = await this.#region(() => this.#hasOwnedLive(id), this.#reloadScopes);
397
+ // Resolved once that decision stands, as it may run `migrate`.
398
+ const resolution = owns
399
+ ? undefined
400
+ : this.#resolve(current, this.#registry.snapshot());
401
+ if (resolution?.kind === "blocked") {
402
+ await this.#terminate(tx, current, { status: "orphaned", reason: resolution.reason });
403
+ return { result: "marked" };
198
404
  }
199
405
  }
200
406
  if (!current.abortRequested || current.abortReason !== undefined)
@@ -210,12 +416,13 @@ export class TaskScheduler {
210
416
  }
211
417
  async waitForTask(id, context) {
212
418
  // Check and register on the line so no terminal publication falls between them.
213
- const found = await this.#session.readOnLine(async () => {
419
+ const found = await this.#session.readOnLine(async (storage) => {
214
420
  if (this.#closing)
215
421
  throw closedError(this.#session);
216
- if (this.#live.has(id))
422
+ if (await this.#region(() => this.#liveRecord(id) !== undefined)) {
217
423
  return { promise: this.#taskWaiters.add(id, context) };
218
- const record = await this.#storage.task(id, context);
424
+ }
425
+ const record = await storage.task(id, context);
219
426
  if (record === undefined)
220
427
  throw new Error(`Task ${id} does not exist`);
221
428
  return { promise: Promise.resolve(record) };
@@ -241,6 +448,7 @@ export class TaskScheduler {
241
448
  */
242
449
  async abortConversation(conversationId, background, context) {
243
450
  const reached = await this.#session.commitWith(async (tx) => {
451
+ await this.#published();
244
452
  const queued = await this.#loadScopes(true);
245
453
  const scope = { conversation: conversationId };
246
454
  const reached = [];
@@ -263,11 +471,100 @@ export class TaskScheduler {
263
471
  await this.waitForTask(id, context);
264
472
  await this.waitForIdle(conversationId, context);
265
473
  }
474
+ /**
475
+ * The ownership memberships a commit's task and conversation changes write (group commit §4.2), at its admission on
476
+ * the line: for each task created or made terminal, every node from its parent up to and including the nearest live
477
+ * task, or the root conversation, and then the roots too, conservatively, as a pending end may unlist the root a
478
+ * creation finds listed; for each conversation created under a task, that task; and the intent set when
479
+ * a change sets, upgrades, or clears a mark or a held or ended outcome other than `completed`. Loads the chains it
480
+ * walks; the nodes it installs stay until the commit is published.
481
+ */
482
+ membershipKeys(tx) {
483
+ const tasks = tx.stagedTasks();
484
+ const conversations = tx.stagedConversations();
485
+ if (tasks.length === 0 && conversations.every((record) => record.owner === undefined))
486
+ return NO_KEYS;
487
+ return this.#membershipKeys(tx, tasks, conversations);
488
+ }
489
+ async #membershipKeys(tx, tasks, conversations) {
490
+ const keys = new Set();
491
+ const overlay = overlayOf(tx);
492
+ const created = new Set(tx.createdTaskIds());
493
+ const reached = new Map();
494
+ for (const record of tasks) {
495
+ if (intentChanges(this.#live.get(record.id), record))
496
+ keys.add(INTENT_KEY);
497
+ if (!created.has(record.id) && record.state.status !== "terminal")
498
+ continue;
499
+ await this.#loadChain(parentOf(record), overlay);
500
+ if ((await this.#chainKeys(parentNode(record), overlay, keys, reached)) !== undefined)
501
+ keys.add(ROOTS_KEY);
502
+ }
503
+ for (const conversation of conversations) {
504
+ const owner = conversation.owner?.taskId;
505
+ if (owner === undefined)
506
+ continue;
507
+ await this.#loadChain({ task: owner }, overlay);
508
+ await this.#chainKeys(owner, overlay, keys, reached);
509
+ }
510
+ return [...keys];
511
+ }
512
+ /**
513
+ * Add the memberships from `start` up to and including the nearest live task, or the root conversation, which it
514
+ * returns. `reached` keeps, per node walked, where its walk ended, so a chain several tasks of a commit share is
515
+ * walked once. Whether an ancestor is live can change under a pending commit, so it is read after a touch of the
516
+ * ancestor's record: a pending end must not stop the chain early. Ownership fields are immutable, so not keyed.
517
+ */
518
+ async #chainKeys(start, overlay, keys, reached) {
519
+ const walked = [];
520
+ let root;
521
+ let node = start;
522
+ while (node !== undefined && node !== null) {
523
+ const key = ownedKey(node);
524
+ // Walked up from here already, for another task of the commit.
525
+ if (reached.has(key)) {
526
+ root = reached.get(key);
527
+ break;
528
+ }
529
+ keys.add(key);
530
+ walked.push(key);
531
+ if (typeof node === "string") {
532
+ const conversation = conversationOf(node);
533
+ node = this.#edge(conversation, overlay);
534
+ if (node === null)
535
+ root = conversation;
536
+ continue;
537
+ }
538
+ let record = overlay.tasks.get(node);
539
+ if (record === undefined) {
540
+ const gate = this.#session.touch(taskKey(node));
541
+ if (gate !== undefined) {
542
+ await gate;
543
+ // It may have ended meanwhile, and its node been swept: load the chain above it again.
544
+ await this.#loadChain({ task: node }, overlay);
545
+ }
546
+ record = this.#live.get(node);
547
+ }
548
+ if (record !== undefined && record.state.status !== "terminal")
549
+ break;
550
+ const above = record ?? this.#settled.get(node);
551
+ node = above === undefined ? undefined : parentNode(above);
552
+ }
553
+ for (const key of walked)
554
+ reached.set(key, root);
555
+ return root;
556
+ }
266
557
  // ─── Scheduling ────────────────────────────────────────────────────────
267
558
  #observe(publication) {
559
+ // A commit held in memory changes documents only.
560
+ if (publication.kind === "memory")
561
+ return;
562
+ this.#advanceContexts(publication);
268
563
  const updated = [];
269
564
  const failed = [];
270
565
  let changed = false;
566
+ // The conversations whose idleness the commit's task changes can change; undefined when a chain is not known.
567
+ let dirty = new Set();
271
568
  // Edges first, so the tasks a commit creates in a new conversation hang from it.
272
569
  for (const change of publication.changes) {
273
570
  if (change.type === "conversation" && !this.#edges.has(change.value.id)) {
@@ -283,6 +580,11 @@ export class TaskScheduler {
283
580
  if (failedOutcome(record) && (previous === undefined || !failedOutcome(previous)))
284
581
  failed.push(record.id);
285
582
  this.#track(previous, record);
583
+ // Idleness counts live tasks, so only a task that became live or ended can change it. Collected before `#sweep()`,
584
+ // which may forget the chain of a task that ended.
585
+ const liveness = previous === undefined || record.state.status === "terminal";
586
+ if (liveness && dirty !== undefined && !this.#addScopesAbove(record, dirty))
587
+ dirty = undefined;
286
588
  if (record.state.status === "terminal") {
287
589
  this.#failedMigrations.delete(record.id);
288
590
  this.#failFastChecks.delete(record.id);
@@ -293,10 +595,11 @@ export class TaskScheduler {
293
595
  }
294
596
  if (record.abortRequested && previous?.abortRequested !== true) {
295
597
  this.#cascadePending = true;
296
- // Signal a run invocation of the newly marked task; its next step ends it.
598
+ // Signal a run invocation of the newly marked task; its next step ends it. Abort listeners are task code, which
599
+ // never runs inside a publication: the signal follows in a microtask, after host commit listeners.
297
600
  const invocation = this.#invocations.get(record.id);
298
601
  if (invocation?.mode === "run")
299
- invocation.controller.abort();
602
+ queueMicrotask(() => invocation.controller.abort());
300
603
  }
301
604
  // A request replacing a `restart` mark upgrades the marks of the work below.
302
605
  if (previous?.abortReason !== undefined && record.abortReason === undefined)
@@ -350,9 +653,60 @@ export class TaskScheduler {
350
653
  this.#sweep();
351
654
  if (!changed)
352
655
  return;
353
- this.#settleIdle();
656
+ this.#settleIdle(dirty);
354
657
  this.#kick();
355
658
  }
659
+ /**
660
+ * Add the conversations above `record` to `scopes`: those whose idleness a change of it can change. False when a node
661
+ * on the way up is not known. A conversation already in `scopes` has everything above it there too.
662
+ */
663
+ #addScopesAbove(record, scopes) {
664
+ for (const step of this.#above(parentOf(record))) {
665
+ if ("unknown" in step)
666
+ return false;
667
+ if (!("conversation" in step))
668
+ continue;
669
+ if (scopes.has(step.conversation))
670
+ return true;
671
+ scopes.add(step.conversation);
672
+ }
673
+ return true;
674
+ }
675
+ /**
676
+ * Extend each kept context range with the entries a stored commit appended to its conversation, so a later read
677
+ * needs no Storage scan. Entries are published in ID order.
678
+ */
679
+ #advanceContexts(publication) {
680
+ if (this.#contexts.size === 0)
681
+ return;
682
+ let appended;
683
+ for (const change of publication.changes) {
684
+ if (change.type !== "entry" || !this.#contexts.has(change.value.conversationId))
685
+ continue;
686
+ appended ??= new Map();
687
+ const entries = appended.get(change.value.conversationId);
688
+ if (entries === undefined)
689
+ appended.set(change.value.conversationId, [change.value]);
690
+ else
691
+ entries.push(change.value);
692
+ }
693
+ if (appended === undefined)
694
+ return;
695
+ for (const [conversationId, entries] of appended) {
696
+ const kept = this.#contexts.get(conversationId);
697
+ // An idle range past its retention is dropped here too: entry-only commits do not run `#settleIdle()`, and a
698
+ // host may decline the expiry timer. `expiresAt` was fixed when it went idle, so no settings are read here.
699
+ if (kept.idleSince !== undefined && kept.expiresAt !== undefined && kept.expiresAt <= this.#now()) {
700
+ this.#dropContext(conversationId);
701
+ continue;
702
+ }
703
+ const range = advanceRange(kept.range, entries);
704
+ if (range === undefined)
705
+ this.#dropContext(conversationId);
706
+ else
707
+ kept.range = range;
708
+ }
709
+ }
356
710
  /**
357
711
  * `settings.contextRetentionMs`, or 0 when the host's settings throw. Kept contexts are only a cache, so dropping them
358
712
  * is safe, while a throw here would escape commit listeners, reconciliation, and the expiry timer.
@@ -362,69 +716,116 @@ export class TaskScheduler {
362
716
  return this.#settings().contextRetentionMs;
363
717
  }
364
718
  catch (error) {
365
- this.#report(error);
719
+ this.#report(error, SETTINGS);
366
720
  return 0;
367
721
  }
368
722
  }
369
- /** Resolve idle waiters, and drop each kept context whose conversation has been idle for the retention period. */
370
- #settleIdle() {
371
- for (const conversationId of this.#idleWaiters.keys()) {
372
- if (this.#idle(conversationId))
373
- this.#idleWaiters.resolve(conversationId);
374
- }
723
+ /**
724
+ * After task changes below the conversations in `scopes` (every conversation when undefined): resolve their idle
725
+ * waiters and the Harness-wide ones, drop the kept contexts whose idle period passed the retention, then start or end
726
+ * the idle periods of the kept contexts in `scopes`. A task change can change the idleness only of the conversations
727
+ * above it, so no check visits every conversation.
728
+ */
729
+ #settleIdle(scopes) {
730
+ const all = scopes === undefined || this.#unknownsChanged;
731
+ this.#unknownsChanged = false;
732
+ if (all)
733
+ this.#fullIdleChecks++;
734
+ this.#resolveIdleWaiters(all ? undefined : scopes);
375
735
  const now = this.#now();
376
736
  const retention = this.#contextRetentionMs();
377
- for (const [conversationId, kept] of this.#contexts) {
378
- if (kept.idleSince !== undefined && now - kept.idleSince >= retention) {
379
- this.#contexts.delete(conversationId);
380
- }
381
- else if (!this.#idle(conversationId)) {
737
+ // Expired first: a context past its retention is dropped even when the same commit makes it busy again.
738
+ this.#expireContexts(now, retention);
739
+ for (const conversationId of all ? [...this.#contexts.keys()] : scopes) {
740
+ const kept = this.#contexts.get(conversationId);
741
+ if (kept === undefined)
742
+ continue;
743
+ if (!this.#idle(conversationId)) {
744
+ if (kept.idleSince === undefined)
745
+ continue;
382
746
  kept.idleSince = undefined;
747
+ this.#idleOrder.delete(conversationId);
383
748
  }
384
- else if (retention > 0) {
385
- kept.idleSince ??= now;
749
+ else if (retention <= 0) {
750
+ this.#dropContext(conversationId);
386
751
  }
387
- else {
388
- this.#contexts.delete(conversationId);
752
+ else if (kept.idleSince === undefined) {
753
+ kept.idleSince = now;
754
+ kept.expiresAt = now + retention;
755
+ this.#markIdle(conversationId, now);
389
756
  }
390
757
  }
391
758
  this.#scheduleExpiry();
392
759
  }
760
+ /** Resolve the idle waiters of the conversations in `scopes` (all when undefined) and the Harness-wide ones. */
761
+ #resolveIdleWaiters(scopes) {
762
+ for (const conversationId of this.#idleWaiters.keys()) {
763
+ if (conversationId !== undefined && scopes !== undefined && !scopes.has(conversationId))
764
+ continue;
765
+ if (this.#idle(conversationId))
766
+ this.#idleWaiters.resolve(conversationId);
767
+ }
768
+ }
393
769
  /**
394
- * Run `#settleIdle()` when the earliest idle context expires. The timer is unreferenced, so it never keeps the process
395
- * alive. Where timers cannot be unreferenced, as in Cloudflare Workers, none is kept: a pending timer could keep a
396
- * Durable Object from being evicted, and eviction frees the contexts. There, task changes alone check expiry.
770
+ * Drop the kept contexts idle for at least `retention`, oldest first, with the retention setting of now; with no
771
+ * retention, every idle one, also those whose idle period a clock that went back makes start after now.
397
772
  */
398
- #scheduleExpiry() {
399
- const retention = this.#contextRetentionMs();
400
- let at;
401
- for (const kept of this.#contexts.values()) {
402
- if (kept.idleSince !== undefined && (at === undefined || kept.idleSince + retention < at)) {
403
- at = kept.idleSince + retention;
404
- }
773
+ #expireContexts(now, retention) {
774
+ for (const [conversationId, idleSince] of this.#idleOrder) {
775
+ if (retention > 0 && now - idleSince < retention)
776
+ return;
777
+ this.#dropContext(conversationId);
405
778
  }
779
+ }
780
+ /**
781
+ * Start the idle period of a kept context in the expiry order. An idle period older than the newest one kept, which
782
+ * a clock that went back starts, sorts the order again, and the newest is taken from what is kept then.
783
+ */
784
+ #markIdle(conversationId, idleSince) {
785
+ this.#idleOrder.delete(conversationId);
786
+ this.#idleOrder.set(conversationId, idleSince);
787
+ if (this.#idleOrder.size === 1 || idleSince >= this.#idleNewest) {
788
+ this.#idleNewest = idleSince;
789
+ return;
790
+ }
791
+ const sorted = [...this.#idleOrder].sort((a, b) => a[1] - b[1]);
792
+ this.#idleOrder.clear();
793
+ for (const [id, since] of sorted)
794
+ this.#idleOrder.set(id, since);
795
+ this.#idleNewest = sorted.at(-1)[1];
796
+ }
797
+ /** Forget a kept context, idle or not, and its place in the expiry order. */
798
+ #dropContext(conversationId) {
799
+ this.#contexts.delete(conversationId);
800
+ this.#idleOrder.delete(conversationId);
801
+ }
802
+ /**
803
+ * Drop the expired contexts when the earliest idle context expires, on a background timer, which never keeps the
804
+ * process alive. Where the host declines it, as the default does in Cloudflare Workers, none is kept: a pending timer
805
+ * could keep a Durable Object from being evicted, and eviction frees the contexts. There, task changes alone check
806
+ * expiry.
807
+ */
808
+ #scheduleExpiry() {
809
+ const oldest = this.#idleOrder.values().next();
810
+ const at = oldest.done === true ? undefined : oldest.value + this.#contextRetentionMs();
406
811
  if (this.#expiry !== undefined && this.#expiry.at === at)
407
812
  return;
408
- if (this.#expiry !== undefined)
409
- clearTimeout(this.#expiry.timer);
813
+ this.#expiry?.disarm();
410
814
  this.#expiry = undefined;
411
815
  if (at === undefined || this.#closing)
412
816
  return;
413
- const timer = setTimeout(() => {
817
+ const disarm = this.#timers.background(() => {
414
818
  this.#expiry = undefined;
415
- this.#settleIdle();
416
- }, Math.min(Math.max(0, at - this.#now()), MAX_TIMER_DELAY));
417
- const unref = timer.unref;
418
- if (typeof unref !== "function") {
419
- clearTimeout(timer);
420
- return;
421
- }
422
- unref.call(timer);
423
- this.#expiry = { at, timer };
819
+ this.#expireContexts(this.#now(), this.#contextRetentionMs());
820
+ this.#scheduleExpiry();
821
+ }, at - this.#now());
822
+ if (disarm !== undefined)
823
+ this.#expiry = { at, disarm };
424
824
  }
425
825
  // ─── Ownership ───────────────────────────────────────────────────────────
426
826
  #scheduleReconcile() {
427
- if (this.#reconcileScheduled || this.#closing)
827
+ // A read-only reconcile could only fail: what it writes is refused, and its own commits fail the Session.
828
+ if (this.#reconcileScheduled || this.#closing || this.#readOnly)
428
829
  return;
429
830
  this.#reconcileScheduled = true;
430
831
  queueMicrotask(() => void this.#reconcile());
@@ -437,54 +838,25 @@ export class TaskScheduler {
437
838
  */
438
839
  async #reconcile() {
439
840
  this.#reconcileScheduled = false;
440
- const cascade = this.#cascadePending;
441
- this.#cascadePending = false;
442
- const checks = [...this.#failFastChecks];
443
- this.#failFastChecks.clear();
841
+ const work = this.#reconcileWork();
842
+ if (work)
843
+ this.#reconcilingWork++;
444
844
  try {
445
- await this.#session.commitWith(async (tx) => {
845
+ await this.#session.commitInternal(async (tx) => {
446
846
  if (this.#closing)
447
847
  return;
848
+ // A cascade scans the queued inputs, which waits for every unpublished commit: a request made before the scan
849
+ // is covered by it, and none is published while the job holds the line after it.
850
+ const cascade = this.#cascadePending;
448
851
  const queued = await this.#loadScopes(cascade);
449
- // A cascade from an abandoned owner passes its `restart` reason on; any other intent marks, or upgrades a
450
- // `restart` mark, as a request. Collected first, so a request wins over a `restart` mark of the same pass.
451
- const marks = new Map();
452
- const mark = (record, reason) => {
453
- const staged = marks.get(record.id);
454
- if (staged !== undefined && staged.reason === undefined)
455
- return;
456
- if (record.abortRequested && (record.abortReason === undefined || reason !== undefined))
457
- return;
458
- marks.set(record.id, { record, reason });
459
- };
460
- // Every live foreground task below a live owner with cancellation intent, which is its nearest one: a walk down
461
- // from each owner with intent does not enter another, whose own walk covers what is below it. Derived again on
462
- // every pass, as loading edges can reveal a cancelled owner.
463
- for (const id of this.#intent) {
464
- const owner = this.#live.get(id);
465
- // Only an owner abandoned after a restart, and nothing else, passes its reason on.
466
- const restart = owner.abortReason === "restart" && !failedOutcome(owner);
467
- for (const { node, live } of this.#walkDown(id, { enter: (below) => !this.#intent.has(below) })) {
468
- if (live !== undefined && !node.background)
469
- mark(live, restart ? "restart" : undefined);
470
- }
471
- }
472
- for (const id of checks) {
473
- const waiter = this.#live.get(id);
474
- if (waiter?.state.status !== "waiting" || !(await this.#anyFailed(waiter.state.on)))
475
- continue;
476
- // Every other live task: the failed one keeps its own outcome.
477
- for (const member of waiter.state.on) {
478
- const record = this.#live.get(member);
479
- if (record !== undefined && !failedOutcome(record))
480
- mark(record, undefined);
481
- }
482
- }
852
+ if (cascade)
853
+ this.#cascadePending = false;
854
+ const marks = await this.#marks();
483
855
  for (const { record, reason } of marks.values())
484
856
  tx.setTask(withAbortMark(record, reason));
485
- for (const id of queued)
486
- if (this.#belowCancelled({ conversation: id }))
487
- await this.#withdrawInputs(tx, id);
857
+ const withdrawn = await this.#region(() => queued.filter((id) => this.#belowCancelled({ conversation: id })), this.#reloadScopes);
858
+ for (const id of withdrawn)
859
+ await this.#withdrawInputs(tx, id);
488
860
  await this.#finalize(tx);
489
861
  }, this.#context);
490
862
  }
@@ -493,59 +865,172 @@ export class TaskScheduler {
493
865
  // running the pass again, so it fails the Session, which reports it.
494
866
  this.#failSession(error);
495
867
  }
496
- this.#settleIdle();
868
+ if (work)
869
+ this.#reconcilingWork--;
870
+ this.#scheduleWake();
871
+ // Its commit's task changes were settled at their publication; a waiter that asked for this pass checks again, and
872
+ // what its chain loads resolved can make any scope idle.
873
+ if (this.#unknownsChanged) {
874
+ this.#settleIdle(undefined);
875
+ return;
876
+ }
877
+ this.#resolveIdleWaiters(undefined);
878
+ // A pass checks expiry too, for hosts that decline the expiry timer.
879
+ this.#expireContexts(this.#now(), this.#contextRetentionMs());
880
+ this.#scheduleExpiry();
497
881
  }
498
- /** Whether any of `ids` holds or ended with an outcome other than `completed`. */
499
- async #anyFailed(ids) {
500
- for (const id of ids) {
501
- const record = this.#live.get(id) ?? (await this.#storage.task(id, this.#context));
502
- if (record !== undefined && failedOutcome(record))
503
- return true;
882
+ /**
883
+ * The abort marks committed records imply (spec §5.4, §5.5): every live foreground task below a live owner with
884
+ * cancellation intent, which is its nearest one (a walk down from each owner with intent does not enter another,
885
+ * whose own walk covers what is below it), and every other live task in the `on` of a `failFast` waiter one of whose
886
+ * tasks failed. A cascade from an abandoned owner passes its `restart` reason on; any other intent marks, or upgrades
887
+ * a `restart` mark, as a request. Collected first, so a request wins over a `restart` mark of the same pass. Derived
888
+ * again on every pass, as loading edges can reveal a cancelled owner. Takes the `failFast` checks once its decision
889
+ * stands.
890
+ */
891
+ async #marks() {
892
+ // Terminal tasks in a checked `on`, read from Storage: each one's outcome is final.
893
+ const ended = new Map();
894
+ for (;;) {
895
+ const plan = await this.#region(() => this.#planMarks(ended), this.#reloadScopes);
896
+ if ("missing" in plan) {
897
+ for (const id of plan.missing)
898
+ ended.set(id, (await this.#session.lineStorage.task(id, this.#context)) ?? null);
899
+ continue;
900
+ }
901
+ for (const id of plan.checked)
902
+ this.#failFastChecks.delete(id);
903
+ return plan.marks;
504
904
  }
505
- return false;
905
+ }
906
+ /** `#marks()`' decision region; the `failFast` members it needs from Storage first, when `ended` lacks them. */
907
+ #planMarks(ended) {
908
+ const marks = new Map();
909
+ // A mark rewrites the whole record, so the record it copies is read as a task record, keyed: a commit of the task
910
+ // that is not published yet makes the region wait and run again, rather than stage a mark over a stale checkpoint.
911
+ const mark = (found, reason) => {
912
+ const record = this.#liveRecord(found.id);
913
+ const staged = marks.get(record.id);
914
+ if (staged !== undefined && staged.reason === undefined)
915
+ return;
916
+ if (record.abortRequested && (record.abortReason === undefined || reason !== undefined))
917
+ return;
918
+ marks.set(record.id, { record, reason });
919
+ };
920
+ this.#record(INTENT_KEY);
921
+ for (const id of this.#intent) {
922
+ const owner = this.#liveRecord(id);
923
+ // Only an owner abandoned after a restart, and nothing else, passes its reason on.
924
+ const restart = owner.abortReason === "restart" && !failedOutcome(owner);
925
+ for (const { node, live } of this.#walkDown(id, { enter: (below) => !this.#intent.has(below) })) {
926
+ if (live !== undefined && !node.background)
927
+ mark(live, restart ? "restart" : undefined);
928
+ }
929
+ }
930
+ const checked = [...this.#failFastChecks];
931
+ const missing = [];
932
+ for (const id of checked) {
933
+ const waiter = this.#liveRecord(id);
934
+ if (waiter?.state.status !== "waiting")
935
+ continue;
936
+ let failed = false;
937
+ for (const member of waiter.state.on) {
938
+ const record = this.#liveRecord(member) ?? ended.get(member);
939
+ if (record === undefined)
940
+ missing.push(member);
941
+ else if (record !== null && failedOutcome(record))
942
+ failed = true;
943
+ }
944
+ if (!failed)
945
+ continue;
946
+ // Every other live task: the failed one keeps its own outcome.
947
+ for (const member of waiter.state.on) {
948
+ const record = this.#liveRecord(member);
949
+ if (record !== undefined && !failedOutcome(record))
950
+ mark(record, undefined);
951
+ }
952
+ }
953
+ return missing.length > 0 ? { missing } : { marks, checked };
954
+ }
955
+ /** Whether a reconcile pass would have work: a cascade to derive, held outcomes to check, or `failFast` waits. */
956
+ #reconcileWork() {
957
+ return this.#cascadePending || this.#finalizeChecks.size > 0 || this.#failFastChecks.size > 0;
506
958
  }
507
959
  /**
508
960
  * Write the terminal record of every `completing` task without live ordinary owned work. Finalizing one can free its
509
961
  * owner, so this repeats over the commit's candidates until nothing changes. A held scheduler outcome gets its
510
- * Harness cleanup here.
962
+ * Harness cleanup here. In rounds, one decision region each: the holders the tasks of one round free are checked once,
963
+ * in the next, so a holder of many tasks that end together is walked once, not once per task. The first round takes
964
+ * the `completing` tasks to check once its decision stands.
511
965
  */
512
966
  async #finalize(tx) {
513
- // A worklist: the tasks that started holding, and the holders of work that ended. Finalizing one can free the
514
- // nearest live task above it, which goes on the list. Tasks finalized here have no live work below them, so walks
515
- // down skip them.
516
- // In rounds: the holders the tasks of one round free are checked once, in the next, so a holder of many tasks that
517
- // end together is walked once, not once per task.
518
- let round = [...this.#finalizeChecks];
519
- this.#finalizeChecks.clear();
520
967
  const done = new Set();
968
+ // The commit's candidates, kept in step with the terminal records each round stages.
521
969
  const overlay = overlayOf(tx);
522
- while (round.length > 0) {
523
- const next = new Set();
524
- for (const id of round) {
525
- const record = overlay.tasks.get(id) ?? this.#live.get(id);
526
- if (record?.state.status !== "completing" || done.has(id))
527
- continue;
528
- if (this.#hasOwnedLive(id, overlay, done))
529
- continue;
530
- const outcome = record.state.outcome;
531
- const terminal = withState(record, { status: "terminal", outcome });
970
+ let round;
971
+ for (;;) {
972
+ const ids = round;
973
+ const plan = await this.#region(() => this.#planRound(ids ?? [...this.#finalizeChecks], overlay, done), this.#reloadScopes);
974
+ if (ids === undefined)
975
+ this.#finalizeChecks.clear();
976
+ for (const { record, terminal } of plan.finalized) {
532
977
  tx.setTask(terminal);
533
- overlay.tasks.set(id, terminal);
534
- done.add(id);
978
+ overlay.tasks.set(record.id, terminal);
979
+ done.add(record.id);
980
+ const outcome = record.state.outcome;
535
981
  // REMINDER: only the scheduler writes `faulted` and `orphaned` (spec §5.4); their cleanup waits for this commit.
536
982
  if (outcome.status === "faulted" || outcome.status === "orphaned") {
537
983
  await this.#settleOutcome(tx, record, outcome);
538
984
  }
985
+ }
986
+ if (plan.next.length === 0)
987
+ return;
988
+ round = plan.next;
989
+ }
990
+ }
991
+ /**
992
+ * One finalize round's decision: which of `ids` are `completing` without live ordinary owned work below them, in the
993
+ * commit's candidates with the round's own finalizations applied in order, and the holders above them to check next.
994
+ * Tasks finalized have no live work below them, so walks down skip them. It applies the round's terminal records to
995
+ * `overlay` while it decides and takes them back before it returns, so a run again starts from the same candidates.
996
+ */
997
+ #planRound(ids, overlay, finalized) {
998
+ const tasks = overlay.tasks;
999
+ // The tasks finalized so far, earlier rounds' and this one's, without copying the former.
1000
+ const round = new Set();
1001
+ const done = { has: (id) => round.has(id) || finalized.has(id) };
1002
+ const ended = [];
1003
+ const next = new Set();
1004
+ try {
1005
+ for (const id of ids) {
1006
+ const staged = tasks.get(id);
1007
+ const record = staged ?? this.#liveRecord(id);
1008
+ if (record?.state.status !== "completing" || done.has(id))
1009
+ continue;
1010
+ if (this.#hasOwnedLive(id, overlay, done))
1011
+ continue;
1012
+ const terminal = withState(record, { status: "terminal", outcome: record.state.outcome });
1013
+ tasks.set(id, terminal);
1014
+ round.add(id);
1015
+ ended.push({ record, terminal, staged: staged !== undefined });
539
1016
  const above = this.#nearestLiveAbove(record, overlay);
540
1017
  if (above?.state.status === "completing")
541
1018
  next.add(above.id);
542
1019
  }
543
- round = [...next];
544
1020
  }
1021
+ finally {
1022
+ for (const { record, staged } of ended) {
1023
+ if (staged)
1024
+ tasks.set(record.id, record);
1025
+ else
1026
+ tasks.delete(record.id);
1027
+ }
1028
+ }
1029
+ return { finalized: ended, next: [...next] };
545
1030
  }
546
1031
  /**
547
1032
  * Load the owner chains of the tasks in `#unloaded` and, with `queued`, of every conversation with queued inputs,
548
- * on the Session line; returns the latter. Reads committed Storage directly, so it may run inside a commit callback.
1033
+ * on the Session line; returns the latter. Reads through the line's Storage, so it may run inside a commit callback.
549
1034
  */
550
1035
  async #loadScopes(queued) {
551
1036
  for (const id of [...this.#unloaded]) {
@@ -555,7 +1040,8 @@ export class TaskScheduler {
555
1040
  }
556
1041
  if (!queued)
557
1042
  return [];
558
- const submissions = await scanAll((cursor) => this.#storage.scanSubmissions({ status: "queued" }, SCAN_PAGE_SIZE, cursor, this.#context));
1043
+ // A scan, which cannot be keyed: it waits for every unpublished commit.
1044
+ const submissions = await scanAll((cursor) => this.#session.lineStorage.scanSubmissions({ status: "queued" }, SCAN_PAGE_SIZE, cursor, this.#context));
559
1045
  const inputs = submissions.filter((submission) => submission.type === "input");
560
1046
  const conversations = [...new Set(inputs.map((submission) => submission.conversationId))];
561
1047
  for (const id of conversations)
@@ -565,48 +1051,76 @@ export class TaskScheduler {
565
1051
  /**
566
1052
  * Load the owner edges and task nodes from `start` up to its ownerless root. A known task outside `#unloaded` ends the
567
1053
  * walk, as its chain is loaded; the `#unloaded` tasks passed have a loaded chain afterwards. Loading a node can reveal
568
- * a cancelled owner, so it makes the next reconcile derive cascade marks.
1054
+ * a cancelled owner, so it makes the next reconcile derive cascade marks. The nodes it installs stay until the job's
1055
+ * commit is published (`#pin()`).
569
1056
  */
570
1057
  async #loadChain(start, overlay) {
571
1058
  const passed = [];
1059
+ let installed = false;
572
1060
  let at = start;
573
- while (at !== undefined) {
574
- if ("task" in at) {
575
- const id = at.task;
576
- let node = this.#live.get(id) ?? this.#settled.get(id);
577
- // Created in this commit: the load from its own record covers what is above it.
578
- if (node === undefined && overlay?.tasks.has(id) === true)
579
- break;
580
- if (node !== undefined && !this.#unloaded.has(id))
581
- break;
582
- if (node === undefined) {
583
- const record = await this.#storage.task(id, this.#context);
584
- if (record === undefined)
585
- return;
586
- node = nodeOf(record);
587
- if (record.state.status === "terminal")
588
- this.#settle(id, node);
589
- // A loaded node can reveal a cancelling owner: the next reconcile derives marks again.
590
- this.#scheduleReconcile();
1061
+ try {
1062
+ while (at !== undefined) {
1063
+ if ("task" in at) {
1064
+ const id = at.task;
1065
+ let node = this.#live.get(id) ?? this.#settled.get(id);
1066
+ // Created in this commit: the load from its own record covers what is above it.
1067
+ if (node === undefined && overlay?.tasks.has(id) === true)
1068
+ break;
1069
+ if (node !== undefined && !this.#unloaded.has(id))
1070
+ break;
1071
+ if (node === undefined) {
1072
+ // Through the line's Storage: a task whose creation or end is not published yet waits for it.
1073
+ const record = await this.#session.lineStorage.task(id, this.#context);
1074
+ if (record === undefined)
1075
+ return;
1076
+ node = nodeOf(record);
1077
+ if (record.state.status === "terminal") {
1078
+ this.#settle(id, node);
1079
+ this.#pin(id);
1080
+ installed = true;
1081
+ }
1082
+ // A loaded node can reveal a cancelling owner: the next reconcile derives marks again.
1083
+ this.#scheduleReconcile();
1084
+ }
1085
+ passed.push(id);
1086
+ at = parentOf(node);
591
1087
  }
592
- passed.push(id);
593
- at = parentOf(node);
594
- }
595
- else {
596
- let edge = this.#edge(at.conversation, overlay);
597
- if (edge === undefined) {
598
- const record = await this.#storage.conversation(at.conversation, this.#context);
599
- edge = record?.owner?.taskId ?? null;
600
- this.#setEdge(at.conversation, edge);
601
- // A loaded node can reveal a cancelling owner: the next reconcile derives marks again.
602
- this.#scheduleReconcile();
1088
+ else {
1089
+ let edge = this.#edge(at.conversation, overlay);
1090
+ if (edge === undefined) {
1091
+ // Through the line's Storage: a conversation whose creation is not published yet waits for it, so a missing
1092
+ // record is never mistaken for an ownerless one.
1093
+ const record = await this.#session.lineStorage.conversation(at.conversation, this.#context);
1094
+ edge = record?.owner?.taskId ?? null;
1095
+ this.#setEdge(at.conversation, edge);
1096
+ this.#pin(conversationNode(at.conversation));
1097
+ installed = true;
1098
+ // A loaded node can reveal a cancelling owner: the next reconcile derives marks again.
1099
+ this.#scheduleReconcile();
1100
+ }
1101
+ at = edge === null ? undefined : { task: edge };
603
1102
  }
604
- at = edge === null ? undefined : { task: edge };
1103
+ }
1104
+ }
1105
+ finally {
1106
+ // `#idle()` counts a live task whose chain is not loaded as inside every scope it may be in: what this load
1107
+ // resolved, also when it stopped early, can make such a scope idle without a task change below it.
1108
+ if (passed.some((id) => this.#live.has(id)) || (installed && this.#liveUnloaded())) {
1109
+ this.#unknownsChanged = true;
1110
+ // A waiter such a task held is checked again at the pass's end, also when this load was not a reconcile's.
1111
+ this.#scheduleReconcile();
605
1112
  }
606
1113
  }
607
1114
  for (const id of passed)
608
1115
  this.#unloaded.delete(id);
609
1116
  }
1117
+ /** Whether a live task's chain is not loaded yet: one `#idle()` counts as inside every scope it may be in. */
1118
+ #liveUnloaded() {
1119
+ for (const id of this.#unloaded)
1120
+ if (this.#live.has(id))
1121
+ return true;
1122
+ return false;
1123
+ }
610
1124
  #setEdge(conversationId, owner) {
611
1125
  this.#edges.set(conversationId, owner);
612
1126
  // The conversation now hangs from its owner, or is a root; with nothing below it, it waits for the sweep.
@@ -653,7 +1167,8 @@ export class TaskScheduler {
653
1167
  */
654
1168
  #sweep() {
655
1169
  for (const node of this.#dropQueue) {
656
- if (this.#below.has(node))
1170
+ // Queued again once its pins go.
1171
+ if (this.#below.has(node) || this.#pins.has(node))
657
1172
  continue;
658
1173
  if (typeof node === "string") {
659
1174
  this.#edges.delete(conversationOf(node));
@@ -666,18 +1181,93 @@ export class TaskScheduler {
666
1181
  }
667
1182
  this.#dropQueue.clear();
668
1183
  }
1184
+ /**
1185
+ * Run `decide`, a decision region (group commit §6): synchronous, reading scheduler memory and recording the keys it
1186
+ * reads, staging nothing and changing no scheduler state. When a key it read has an unpublished writer, its result,
1187
+ * or its error, is dropped: once that writer is published, `refresh` reloads what the region needs and it runs
1188
+ * again. A waiting job holds the line, so unpublished writes only shrink, and the runs converge. What a region read
1189
+ * that had no unpublished writer gets none for the rest of the job, so its result stands while the job applies it.
1190
+ */
1191
+ async #region(decide, refresh) {
1192
+ for (;;) {
1193
+ const keys = new Set();
1194
+ this.#reading = keys;
1195
+ let result;
1196
+ try {
1197
+ result = { value: decide() };
1198
+ }
1199
+ catch (error) {
1200
+ result = { error };
1201
+ }
1202
+ finally {
1203
+ this.#reading = undefined;
1204
+ }
1205
+ const gates = [];
1206
+ for (const key of keys) {
1207
+ const gate = this.#session.touch(key);
1208
+ if (gate !== undefined)
1209
+ gates.push(gate);
1210
+ }
1211
+ if (gates.length === 0) {
1212
+ if ("error" in result)
1213
+ throw result.error;
1214
+ return result.value;
1215
+ }
1216
+ await Promise.all(gates);
1217
+ await refresh?.();
1218
+ }
1219
+ }
1220
+ /** Load the chains a region walks, again after it waited: what was published meanwhile may list new work. */
1221
+ #reloadScopes = () => this.#loadScopes(false);
1222
+ /** In a decision region, note that it reads `key`; anywhere else, nothing. */
1223
+ #record(key) {
1224
+ this.#reading?.add(key);
1225
+ }
1226
+ /** A live task's committed record, read by a decision region. */
1227
+ #liveRecord(id) {
1228
+ this.#record(taskKey(id));
1229
+ return this.#live.get(id);
1230
+ }
1231
+ /**
1232
+ * Keep `node`, which a chain load of the job on the line installed, until that job's commit is published or fails:
1233
+ * work it creates below the node needs it, and the sweep would drop it as unneeded before then.
1234
+ */
1235
+ #pin(node) {
1236
+ this.#pins.set(node, (this.#pins.get(node) ?? 0) + 1);
1237
+ this.#session.onJobEnd(() => {
1238
+ const count = this.#pins.get(node) - 1;
1239
+ if (count > 0) {
1240
+ this.#pins.set(node, count);
1241
+ return;
1242
+ }
1243
+ this.#pins.delete(node);
1244
+ this.#dropIfUnneeded(node);
1245
+ });
1246
+ }
669
1247
  /**
670
1248
  * Bring the scheduler's sets in line with a task's committed record. A task that ended may free the nearest live task
671
1249
  * above it: one holding its outcome finalizes, an abort-marked one runs its abort handler; its waiters may run.
672
1250
  */
673
1251
  #track(previous, record) {
674
1252
  const id = record.id;
1253
+ const invocation = this.#invocations.get(id);
1254
+ if (invocation !== undefined)
1255
+ invocation.record = record;
1256
+ if (record.state.status === "terminal") {
1257
+ if (previous !== undefined && !this.#closing)
1258
+ this.#countActive(-1, record.kind);
1259
+ this.#resumed.delete(id);
1260
+ }
1261
+ else if (previous === undefined && !this.#closing) {
1262
+ this.#countActive(1, record.kind);
1263
+ }
675
1264
  if (previous?.state.status === "waiting")
676
1265
  for (const on of previous.state.on)
677
1266
  deleteFrom(this.#waiters, on, id);
678
1267
  if (record.state.status === "terminal") {
679
1268
  const node = nodeOf(record);
680
1269
  this.#live.delete(id);
1270
+ this.#sleeping.delete(id);
681
1271
  this.#intent.delete(id);
682
1272
  this.#runnable.delete(id);
683
1273
  this.#finalizeChecks.delete(id);
@@ -711,6 +1301,10 @@ export class TaskScheduler {
711
1301
  if (record.state.status === "waiting")
712
1302
  for (const on of record.state.on)
713
1303
  addTo(this.#waiters, on, id);
1304
+ if (record.state.status === "sleeping")
1305
+ this.#sleeping.add(id);
1306
+ else
1307
+ this.#sleeping.delete(id);
714
1308
  }
715
1309
  /** The nearest live task above ended ordinary work may be free now: one holding its outcome, or an aborting one. */
716
1310
  #release(node, overlay) {
@@ -773,11 +1367,12 @@ export class TaskScheduler {
773
1367
  * at or below a background task holds nothing above it.
774
1368
  */
775
1369
  #nearestLiveAbove(node, overlay) {
1370
+ this.#record(INTENT_KEY);
776
1371
  for (let current = node; !current.background;) {
777
1372
  const owner = current.owner ?? this.#edge(current.conversationId, overlay);
778
1373
  if (owner === undefined || owner === null)
779
1374
  return undefined;
780
- const record = overlay?.tasks.get(owner) ?? this.#live.get(owner);
1375
+ const record = overlay?.tasks.get(owner) ?? this.#liveRecord(owner);
781
1376
  if (record !== undefined && record.state.status !== "terminal")
782
1377
  return record;
783
1378
  const above = record ?? this.#settled.get(owner);
@@ -797,7 +1392,10 @@ export class TaskScheduler {
797
1392
  }
798
1393
  /** Owner task of a conversation, `null` when ownerless, `undefined` while not loaded. */
799
1394
  #edge(id, overlay) {
800
- return overlay?.edges.has(id) === true ? overlay.edges.get(id) : this.#edges.get(id);
1395
+ if (overlay?.edges.has(id) === true)
1396
+ return overlay.edges.get(id);
1397
+ this.#record(conversationKey(id));
1398
+ return this.#edges.get(id);
801
1399
  }
802
1400
  #node(id, overlay) {
803
1401
  return overlay?.tasks.get(id) ?? this.#live.get(id) ?? this.#settled.get(id);
@@ -836,6 +1434,7 @@ export class TaskScheduler {
836
1434
  const links = overlay === undefined ? undefined : this.#links(overlay);
837
1435
  const pending = [start];
838
1436
  for (let at = pending.pop(); at !== undefined; at = pending.pop()) {
1437
+ this.#record(ownedKey(at));
839
1438
  for (const children of [this.#below.get(at), links?.get(at)]) {
840
1439
  for (const child of children ?? []) {
841
1440
  if (typeof child === "string") {
@@ -923,6 +1522,7 @@ export class TaskScheduler {
923
1522
  * edge is not loaded.
924
1523
  */
925
1524
  #inScope(start, scope, crossBackground = false) {
1525
+ this.#record(INTENT_KEY);
926
1526
  for (const step of this.#above(start)) {
927
1527
  if ("unknown" in step)
928
1528
  return undefined;
@@ -945,6 +1545,7 @@ export class TaskScheduler {
945
1545
  }
946
1546
  /** The nearest live owner above `start` whose cancellation intent reaches it, if any; see `#belowCancelled`. */
947
1547
  #cancellingOwner(start) {
1548
+ this.#record(INTENT_KEY);
948
1549
  // Nothing cancels, the common case: no walk.
949
1550
  if (this.#intent.size === 0)
950
1551
  return undefined;
@@ -961,29 +1562,149 @@ export class TaskScheduler {
961
1562
  }
962
1563
  return undefined;
963
1564
  }
1565
+ /**
1566
+ * Count a task in or out of `pi.durable.tasks.active`. Commit observers count at a commit's publication, which must not
1567
+ * throw, so a throwing adapter is reported.
1568
+ */
1569
+ #countActive(delta, kind) {
1570
+ try {
1571
+ this.#activeMetric.add(delta, { task: kind });
1572
+ }
1573
+ catch (error) {
1574
+ this.#report(error, TELEMETRY);
1575
+ }
1576
+ }
964
1577
  /** Fail the Session, which reports it, for a throw in one of the scheduler's own commits; ignored once closing. */
965
- #failSession(error) {
1578
+ #failSession(error, context) {
966
1579
  if (!this.#closing)
967
- this.#session.fail(error);
1580
+ this.#session.fail(error, context);
968
1581
  }
969
1582
  /** Close listener: runs synchronously once admission is sealed, before `join()`, or when the Session fails. */
970
1583
  #seal() {
1584
+ for (const record of this.#live.values())
1585
+ this.#countActive(-1, record.kind);
1586
+ this.#resumed.clear();
971
1587
  this.#closing = true;
972
1588
  try {
973
1589
  this.#unsubscribeRegistry();
974
1590
  }
975
1591
  catch (error) {
976
- this.#report(error);
1592
+ this.#report(error, REGISTRY);
977
1593
  }
978
1594
  const error = closedError(this.#session);
979
1595
  this.#taskWaiters.rejectAll(error);
980
1596
  this.#idleWaiters.rejectAll(error);
981
1597
  this.#contexts.clear();
982
- if (this.#expiry !== undefined)
983
- clearTimeout(this.#expiry.timer);
1598
+ this.#idleOrder.clear();
1599
+ this.#expiry?.disarm();
984
1600
  this.#expiry = undefined;
1601
+ this.#timer?.disarm();
1602
+ this.#timer = undefined;
1603
+ }
1604
+ /**
1605
+ * Recompute the wake in a microtask the line never waits for, with no order relative to host commit listeners (one
1606
+ * already queued serves later changes too): arm the built-in timer, or tell `onWake` of a change.
1607
+ */
1608
+ #scheduleWake() {
1609
+ if (this.#wakeScheduled || this.#closing || this.#readOnly)
1610
+ return;
1611
+ this.#wakeScheduled = true;
1612
+ queueMicrotask(() => {
1613
+ this.#wakeScheduled = false;
1614
+ if (this.#closing)
1615
+ return;
1616
+ const wake = this.wake();
1617
+ if (this.#onWake === undefined) {
1618
+ this.#armTimer(this.#enabled ? wake.at : undefined);
1619
+ return;
1620
+ }
1621
+ const last = this.#lastWake;
1622
+ if (last !== undefined && last.active === wake.active && last.at === wake.at)
1623
+ return;
1624
+ try {
1625
+ this.#onWake(wake);
1626
+ // Only a delivered wake counts: after a throw, the next change or kick offers it again.
1627
+ this.#lastWake = wake;
1628
+ }
1629
+ catch (error) {
1630
+ this.#report(error, ON_WAKE);
1631
+ }
1632
+ });
1633
+ }
1634
+ /**
1635
+ * Keep the timer for the earliest sleep. It keeps the process alive, unlike the context expiry timer: a process with a
1636
+ * sleeping task has work left, such as a retry backoff. Rechecks the clock when it fires, so a wake before `until`
1637
+ * only arms it again.
1638
+ */
1639
+ #armTimer(at) {
1640
+ if (this.#timer !== undefined && this.#timer.at === at)
1641
+ return;
1642
+ this.#timer?.disarm();
1643
+ this.#timer = undefined;
1644
+ if (at === undefined || this.#closing)
1645
+ return;
1646
+ const disarm = this.#timers.foreground(() => {
1647
+ this.#timer = undefined;
1648
+ this.#wakeDue();
1649
+ this.#kick();
1650
+ }, at - this.#now());
1651
+ this.#timer = { at, disarm };
1652
+ }
1653
+ /** Make every sleeping task whose time has come a reservation candidate. */
1654
+ #wakeDue() {
1655
+ for (const id of this.#sleeping) {
1656
+ const record = this.#live.get(id);
1657
+ if (record !== undefined && !this.#notDue(record))
1658
+ this.#runnable.add(id);
1659
+ }
1660
+ }
1661
+ /**
1662
+ * The report info of a task's failure, with the extension that installed its definition: the code that failed is
1663
+ * that extension's. Built-in tasks have none.
1664
+ */
1665
+ /** What a report about `task`'s own code says of it: the task, and the extension that installed its definition. */
1666
+ #aboutTask(task) {
1667
+ let extension;
1668
+ if ("extension" in task)
1669
+ extension = task.extension;
1670
+ else {
1671
+ try {
1672
+ const snapshot = this.#registry.snapshot();
1673
+ extension = installerOf(snapshot, snapshot.task(task.kind));
1674
+ }
1675
+ catch {
1676
+ // A registry that throws is reported where it is read for scheduling; a report goes out without the name.
1677
+ }
1678
+ }
1679
+ return { task: reportTask(task), ...optionalExtension(extension) };
1680
+ }
1681
+ /** `settings.restartLimit`, or the default when the host's settings throw. */
1682
+ #restartLimit() {
1683
+ try {
1684
+ return this.#settings().restartLimit;
1685
+ }
1686
+ catch (error) {
1687
+ this.#report(error, SETTINGS);
1688
+ return DEFAULT_RESTART_LIMIT;
1689
+ }
1690
+ }
1691
+ #notifyBlocked() {
1692
+ for (const listener of [...this.#blockedListeners]) {
1693
+ try {
1694
+ listener();
1695
+ }
1696
+ catch (error) {
1697
+ this.#report(error, { kind: "listener_error" });
1698
+ }
1699
+ }
1700
+ }
1701
+ /** A sleep whose `until` the clock has not reached; an abort mark ends a sleep. */
1702
+ #notDue(record) {
1703
+ const state = record.state;
1704
+ return state.status === "sleeping" && !record.abortRequested && state.until > this.#now();
985
1705
  }
986
1706
  #kick() {
1707
+ this.#scheduleWake();
987
1708
  this.#dirty = true;
988
1709
  if (this.#draining || !this.#enabled || this.#closing)
989
1710
  return;
@@ -1016,45 +1737,57 @@ export class TaskScheduler {
1016
1737
  */
1017
1738
  async #reserve() {
1018
1739
  const reservations = [];
1740
+ const faulted = [];
1019
1741
  let abandoned = [];
1020
1742
  try {
1021
- await this.#session.commitWith(async (tx) => {
1743
+ await this.#session.commitInternal(async (tx) => {
1022
1744
  if (!this.#enabled || this.#closing)
1023
1745
  return;
1024
- for (const id of this.#abandoned) {
1025
- const record = this.#live.get(id);
1026
- if (record !== undefined && !record.abortRequested)
1027
- tx.setTask(withAbortMark(record, "restart"));
1028
- }
1029
1746
  if (this.#abandoned.size > 0) {
1030
- abandoned = [...this.#abandoned];
1747
+ const ids = [...this.#abandoned];
1748
+ const unmarked = await this.#region(() => ids
1749
+ .map((id) => this.#liveRecord(id))
1750
+ .filter((record) => record !== undefined && !record.abortRequested));
1751
+ for (const record of unmarked)
1752
+ tx.setTask(withAbortMark(record, "restart"));
1753
+ abandoned = ids;
1031
1754
  return;
1032
1755
  }
1033
1756
  await this.#loadScopes(false);
1757
+ // Every pass picks up sleeps whose time has come: a pass started by any change may be the first to run after
1758
+ // a deadline, and the wake it schedules no longer keeps a timer for a due sleep.
1759
+ this.#wakeDue();
1760
+ const plan = await this.#region(() => this.#planReservation(), this.#reloadScopes);
1761
+ for (const id of plan.dropped)
1762
+ this.#runnable.delete(id);
1763
+ // Work below an owner with cancellation intent waits for its cascade mark instead of running a phase; schedule
1764
+ // the cascade, which may not have run yet.
1765
+ if (plan.cascade) {
1766
+ this.#cascadePending = true;
1767
+ this.#scheduleReconcile();
1768
+ }
1034
1769
  // Taken once per pass, and only when some task is a candidate.
1035
1770
  let snapshot;
1036
- for (const id of [...this.#runnable]) {
1037
- const record = this.#live.get(id);
1038
- if (record === undefined || record.state.status === "completing" || this.#invocations.has(id)) {
1039
- this.#runnable.delete(id);
1040
- continue;
1041
- }
1042
- // It returns once what it waits for ends, its record changes, or the work below it ends.
1043
- if (this.#waitingOn(record, true).length > 0) {
1044
- this.#runnable.delete(id);
1045
- continue;
1046
- }
1047
- const runnable = record;
1771
+ for (const record of plan.ready) {
1772
+ const id = record.id;
1048
1773
  const mode = record.abortRequested ? "abort" : "run";
1049
- // Work below an owner with cancellation intent waits for its cascade mark instead of running a phase; schedule
1050
- // the cascade, which may not have run yet.
1051
- if (mode === "run" && !record.background && this.#belowCancelled(parentOf(record))) {
1052
- this.#cascadePending = true;
1053
- this.#scheduleReconcile();
1774
+ // Work cut off `restartLimit` times in a row would run forever: an abort handler, or a phase of a kind that
1775
+ // does not end itself at the limit, ends the task `faulted` instead.
1776
+ // The settings are read only for a task that was cut off at all.
1777
+ const cutOff = record.restarts !== undefined && record.restarts >= this.#restartLimit();
1778
+ if (cutOff && (mode === "abort" || !this.#limitsRestarts(record.kind))) {
1779
+ const message = mode === "abort"
1780
+ ? `Abort handler of task ${id} (${record.kind}) was cut off ${record.restarts} times`
1781
+ : `Task ${id} (${record.kind}) was cut off ${record.restarts} times without progress`;
1782
+ await this.#terminate(tx, record, {
1783
+ status: "faulted",
1784
+ error: { message, detail: { reason: "restart_limit" } },
1785
+ });
1786
+ faulted.push({ record, message });
1054
1787
  continue;
1055
1788
  }
1056
1789
  snapshot ??= this.#registry.snapshot();
1057
- const resolution = this.#resolve(runnable, snapshot);
1790
+ const resolution = this.#resolve(record, snapshot);
1058
1791
  if (resolution.kind === "blocked") {
1059
1792
  // An abandoned task waits for its definition, so its abort handler can clean up.
1060
1793
  if (mode === "abort" && record.abortReason === undefined) {
@@ -1062,7 +1795,7 @@ export class TaskScheduler {
1062
1795
  }
1063
1796
  continue;
1064
1797
  }
1065
- if (resolution.record !== runnable || record.state.status !== "running") {
1798
+ if (resolution.record !== record || record.state.status !== "running") {
1066
1799
  tx.setTask(withState(resolution.record, {
1067
1800
  status: "running",
1068
1801
  checkpoint: resolution.record.state.checkpoint,
@@ -1071,7 +1804,7 @@ export class TaskScheduler {
1071
1804
  // Registered on the line, so marks and later reservations see it and close joins it; `#end` makes it a
1072
1805
  // candidate again.
1073
1806
  this.#runnable.delete(id);
1074
- const invocation = this.#createInvocation(record, mode);
1807
+ const invocation = this.#createInvocation(record, mode, installerOf(snapshot, resolution.task));
1075
1808
  reservations.push({ invocation, task: resolution.task, snapshot });
1076
1809
  }
1077
1810
  }, this.#context);
@@ -1084,6 +1817,13 @@ export class TaskScheduler {
1084
1817
  }
1085
1818
  throw error;
1086
1819
  }
1820
+ for (const { record, message } of faulted) {
1821
+ this.#report(new Error(message), {
1822
+ kind: "task_faulted",
1823
+ ...this.#aboutTask(recordTask(record)),
1824
+ reason: "restart_limit",
1825
+ });
1826
+ }
1087
1827
  // Marked, or already marked or gone: done with them. Reserve again, now seeing the marks.
1088
1828
  for (const id of abandoned)
1089
1829
  this.#abandoned.delete(id);
@@ -1091,6 +1831,39 @@ export class TaskScheduler {
1091
1831
  this.#dirty = true;
1092
1832
  return reservations;
1093
1833
  }
1834
+ /**
1835
+ * A reservation pass's decision region: the candidates it drops (gone, holding, waiting, asleep, or with an
1836
+ * invocation), whether one waits for a cascade, and the runnable records to resolve. Iterating the candidates is not
1837
+ * keyed: a pass that misses a task a pending commit makes runnable only delays it, as does skipping one with an
1838
+ * invocation without reading its record, or a waiter's check whether what it waits on is still live.
1839
+ */
1840
+ #planReservation() {
1841
+ const dropped = [];
1842
+ const ready = [];
1843
+ let cascade = false;
1844
+ for (const id of this.#runnable) {
1845
+ if (this.#invocations.has(id)) {
1846
+ dropped.push(id);
1847
+ continue;
1848
+ }
1849
+ const record = this.#liveRecord(id);
1850
+ // It returns once what it waits for ends, its record changes, or the work below it ends; a sleep at the first pass
1851
+ // after its time comes, which the timer, `resume()`, or any change starts.
1852
+ if (record === undefined ||
1853
+ record.state.status === "completing" ||
1854
+ this.#waitingOn(record, true).length > 0 ||
1855
+ this.#notDue(record)) {
1856
+ dropped.push(id);
1857
+ continue;
1858
+ }
1859
+ if (!record.abortRequested && !record.background && this.#belowCancelled(parentOf(record))) {
1860
+ cascade = true;
1861
+ continue;
1862
+ }
1863
+ ready.push(record);
1864
+ }
1865
+ return { dropped, cascade, ready };
1866
+ }
1094
1867
  /**
1095
1868
  * Live tasks a task waits for before its next invocation: its live ordinary owned work when abort-marked, since
1096
1869
  * abort runs bottom-up, otherwise the live part of the `on` of a wait.
@@ -1123,7 +1896,11 @@ export class TaskScheduler {
1123
1896
  }
1124
1897
  catch (error) {
1125
1898
  this.#failedMigrations.set(record.id, { task: fit.task, error });
1126
- this.#report(error);
1899
+ // Blocked now without a commit: the graph and the host's wake must hear it.
1900
+ queueMicrotask(() => this.#notifyBlocked());
1901
+ this.#scheduleWake();
1902
+ const extension = installerOf(snapshot, fit.task);
1903
+ this.#report(error, { kind: "migration_failed", ...this.#aboutTask({ ...recordTask(record), extension }) });
1127
1904
  return { kind: "blocked", reason: "migration_failed" };
1128
1905
  }
1129
1906
  }
@@ -1153,6 +1930,36 @@ export class TaskScheduler {
1153
1930
  roots: this.#roots.size,
1154
1931
  unloaded: this.#unloaded.size,
1155
1932
  dropQueue: this.#dropQueue.size,
1933
+ pins: this.#pins.size,
1934
+ };
1935
+ }
1936
+ /**
1937
+ * The kept contexts' idle state, for tests (a diagnostic, computed on demand): how many are kept and how many are
1938
+ * idle, how many disagree with a full recompute (an idle period without `#idle()`, or `#idle()` without one; an
1939
+ * expiry order that differs from the idle periods, or is not sorted), and how many idle checks covered every scope.
1940
+ */
1941
+ get idleState() {
1942
+ let mismatches = 0;
1943
+ let idle = 0;
1944
+ for (const [conversationId, kept] of this.#contexts) {
1945
+ if (kept.idleSince !== undefined)
1946
+ idle++;
1947
+ if ((kept.idleSince !== undefined) !== this.#idle(conversationId))
1948
+ mismatches++;
1949
+ if (this.#idleOrder.get(conversationId) !== kept.idleSince)
1950
+ mismatches++;
1951
+ }
1952
+ let previous = Number.NEGATIVE_INFINITY;
1953
+ for (const [conversationId, idleSince] of this.#idleOrder) {
1954
+ if (idleSince < previous || this.#contexts.get(conversationId)?.idleSince !== idleSince)
1955
+ mismatches++;
1956
+ previous = idleSince;
1957
+ }
1958
+ return {
1959
+ contexts: this.#contexts.size,
1960
+ idleContexts: idle,
1961
+ idleMismatches: mismatches,
1962
+ fullIdleChecks: this.#fullIdleChecks,
1156
1963
  };
1157
1964
  }
1158
1965
  /**
@@ -1161,6 +1968,7 @@ export class TaskScheduler {
1161
1968
  * cannot exist, shows as failed.
1162
1969
  */
1163
1970
  async inspect(snapshot) {
1971
+ await this.#published();
1164
1972
  await this.#loadScopes(false);
1165
1973
  const tasks = [];
1166
1974
  for (const record of this.#live.values())
@@ -1176,24 +1984,31 @@ export class TaskScheduler {
1176
1984
  const on = this.#waitingOn(record);
1177
1985
  if (on.length > 0)
1178
1986
  return { kind: "waiting", on };
1987
+ if (this.#notDue(record))
1988
+ return { kind: "sleeping", until: record.state.until };
1179
1989
  const fit = this.#fit(record, snapshot.task(record.kind));
1180
1990
  if ("reason" in fit)
1181
1991
  return { kind: "blocked", ...fit };
1182
1992
  if (fit.migrates && fit.task.definition.migrate === undefined) {
1183
1993
  return { kind: "blocked", reason: "migration_failed", error: missingMigration(record, erased(fit.task)) };
1184
1994
  }
1185
- return { kind: "ready", migrates: fit.migrates };
1995
+ return { kind: "pending", migrates: fit.migrates };
1186
1996
  }
1187
- #createInvocation(record, mode) {
1997
+ #createInvocation(record, mode, extension) {
1188
1998
  const controller = new AbortController();
1189
1999
  const { promise: done, resolve: finish } = Promise.withResolvers();
1190
2000
  const invocation = {
1191
2001
  taskId: record.id,
2002
+ kind: record.kind,
2003
+ extension,
1192
2004
  conversationId: record.conversationId,
1193
2005
  mode,
1194
2006
  controller,
2007
+ links: record.startedAt === undefined ? record.links : undefined,
1195
2008
  context: withAbortSignal(controller.signal, this.#context),
2009
+ record: record,
1196
2010
  watches: new Set(),
2011
+ timers: new Set(),
1197
2012
  ended: false,
1198
2013
  done,
1199
2014
  finish: () => finish(),
@@ -1211,10 +2026,13 @@ export class TaskScheduler {
1211
2026
  await this.#runAbort(reservation);
1212
2027
  }
1213
2028
  catch (error) {
1214
- this.#report(error);
2029
+ this.#report(error, { kind: "task_error", ...this.#aboutTask(invocation) }, invocation.context);
1215
2030
  }
1216
2031
  finally {
1217
2032
  this.#end(invocation);
2033
+ invocation.record = undefined;
2034
+ invocation.links = undefined;
2035
+ invocation.fault = undefined;
1218
2036
  invocation.finish();
1219
2037
  this.#kick();
1220
2038
  }
@@ -1226,61 +2044,128 @@ export class TaskScheduler {
1226
2044
  const state = { task: reservation.task, snapshot: reservation.snapshot, reported: undefined };
1227
2045
  const phase = { snapshot: () => state.snapshot, task: () => state.task, agent: undefined };
1228
2046
  const runtime = this.#runtime(invocation, phase);
1229
- let previous;
1230
- for (;;) {
1231
- const current = await this.#step(invocation, (tx, current) => this.#decide(tx, current, previous, state));
1232
- // Close may seal between the decision and dispatch.
1233
- if (current === undefined || this.#closing)
1234
- return;
1235
- const checkpoint = current.state.checkpoint;
1236
- // Each phase handler resolves its agent afresh, at first use.
1237
- phase.agent = undefined;
2047
+ const progress = (tx, current) => this.#progress(tx, current, state, invocation.extension, invocation.context);
2048
+ let current = await this.#step(invocation, (task) => this.#rules(task, undefined), progress);
2049
+ while (current !== undefined && !this.#closing) {
2050
+ const task = current;
2051
+ current = await this.#phase(invocation, task, async () => {
2052
+ const checkpoint = task.state.checkpoint;
2053
+ // Each phase handler resolves its agent afresh, at first use.
2054
+ phase.agent = undefined;
2055
+ let previous;
2056
+ try {
2057
+ await erased(state.task).phases[checkpoint.phase](task, runtime, invocation.context);
2058
+ previous = { checkpoint };
2059
+ }
2060
+ catch (error) {
2061
+ previous = { checkpoint, failure: { error } };
2062
+ }
2063
+ return this.#step(invocation, (next) => this.#rules(next, previous), progress);
2064
+ });
2065
+ }
2066
+ }
2067
+ /** A phase owns its decision commit, not the next handler, a durable wait, or recovery. */
2068
+ #phase(invocation, task, run) {
2069
+ const parent = task.trace === undefined
2070
+ ? this.#telemetry
2071
+ : (this.#telemetry.propagation?.extract(task.trace) ?? this.#telemetry);
2072
+ const checkpoint = task.state.checkpoint;
2073
+ const attempt = checkpoint.attempt;
2074
+ const resumed = this.#resumed.delete(task.id);
2075
+ return createTypedSpanStarter(parent, [DURABLE_TELEMETRY_SCHEMA])("pi.durable.phase", {
2076
+ "task.id": task.id,
2077
+ "task.name": task.kind,
2078
+ phase: checkpoint.phase,
2079
+ kind: invocation.mode,
2080
+ resumed,
2081
+ ...(attempt === undefined ? {} : { attempt }),
2082
+ }, async (span) => {
2083
+ if (span.propagation !== undefined && invocation.links !== undefined) {
2084
+ for (const carrier of invocation.links)
2085
+ span.addLink({ context: span.propagation.extract(carrier) });
2086
+ }
2087
+ const previous = invocation.context;
2088
+ invocation.context = withTelemetry(span, previous);
2089
+ const started = performance.now();
1238
2090
  try {
1239
- await erased(state.task).phases[checkpoint.phase](current, runtime, invocation.context);
1240
- previous = { checkpoint };
2091
+ return await run();
1241
2092
  }
1242
- catch (error) {
1243
- previous = { checkpoint, failure: { error } };
2093
+ finally {
2094
+ invocation.context = previous;
2095
+ const state = invocation.record.state;
2096
+ const terminal = invocation.fault ??
2097
+ (state.status === "terminal" || state.status === "completing" ? state.outcome : undefined);
2098
+ const outcome = terminal !== undefined
2099
+ ? terminal.status === "completed"
2100
+ ? "done"
2101
+ : terminal.status === "aborted"
2102
+ ? "aborted"
2103
+ : "faulted"
2104
+ : state.status === "waiting" || state.status === "sleeping"
2105
+ ? state.status
2106
+ : invocation.ended || this.#closing
2107
+ ? "aborted"
2108
+ : `next=${state.checkpoint.phase}`;
2109
+ span.setAttributes({ outcome });
2110
+ if (terminal?.status === "faulted" || terminal?.status === "failed" || terminal?.status === "orphaned") {
2111
+ const message = terminal.status === "orphaned" ? terminal.reason : terminal.error.message;
2112
+ span.setStatus({
2113
+ status: "error",
2114
+ error: { name: `task_${terminal.status}`, message: ownedSlice(message, 0, 500) },
2115
+ });
2116
+ }
2117
+ else if (outcome === "aborted") {
2118
+ span.setStatus({ status: "ok" });
2119
+ }
2120
+ this.#durationMetric.record(performance.now() - started, { task: task.kind, phase: checkpoint.phase, outcome }, span);
1244
2121
  }
1245
- }
2122
+ });
1246
2123
  }
1247
2124
  /**
1248
- * Precedence rules for a run invocation, on the line. Rules 1 (terminal, `completing`, or `waiting`) and 2 (closing)
1249
- * are applied by `#step`. Returns whether the invocation continues with the next phase.
2125
+ * Precedence rules for a run invocation, a decision region of its step. Rules 1 (terminal, `completing`, `waiting`, or
2126
+ * `sleeping`) and 2 (closing) are applied by `#step`. Progress (rule 5) is decided by `#progress()` once the decision
2127
+ * stands, as it takes a registry snapshot.
1250
2128
  */
1251
- #decide(tx, current, previous, state) {
2129
+ #rules(current, previous) {
1252
2130
  // 3. abort mark: end; a fresh abort invocation starts once the task's ordinary owned work is gone.
1253
2131
  if (current.abortRequested)
1254
- return false;
2132
+ return END;
1255
2133
  // An owner's cancellation intent ends it too, before its cascade marks it; reservation then holds it back.
1256
- if (!current.background && this.#belowCancelled(parentOf(current))) {
1257
- this.#cascadePending = true;
1258
- this.#scheduleReconcile();
1259
- return false;
1260
- }
2134
+ if (!current.background && this.#belowCancelled(parentOf(current)))
2135
+ return CASCADE;
1261
2136
  if (previous === undefined)
1262
- return true;
2137
+ return CONTINUE;
1263
2138
  // 4. uncaught error.
1264
2139
  if (previous.failure !== undefined)
1265
- return { fault: previous.failure.error };
2140
+ return { kind: "fault", error: previous.failure.error };
1266
2141
  // 6. no durable progress.
1267
2142
  if (jsonEqual(current.state.checkpoint, previous.checkpoint)) {
1268
2143
  const message = `Task ${current.kind} phase ${previous.checkpoint.phase} returned without durable progress`;
1269
- return { fault: new Error(message) };
2144
+ return { kind: "fault", error: new Error(message) };
1270
2145
  }
1271
- // 5. progress: refresh the snapshot; hand over to a replacement definition that can take the task.
2146
+ return PROGRESS;
2147
+ }
2148
+ /**
2149
+ * Rule 5, progress: refresh the snapshot; hand over to a replacement definition that can take the task, writing it
2150
+ * back as `pending`, or keep running under the old one, reported once per resolved definition. Returns whether the
2151
+ * invocation continues.
2152
+ */
2153
+ #progress(tx, current, state, extension, context) {
1272
2154
  state.snapshot = this.#registry.snapshot();
1273
2155
  const next = state.snapshot.task(current.kind);
1274
- if (next !== state.task) {
1275
- if (next !== undefined && canReserve(next, current)) {
1276
- tx.setTask(withState(current, { status: "pending", checkpoint: current.state.checkpoint }));
1277
- return false;
1278
- }
1279
- if (state.reported === undefined || state.reported.task !== next) {
1280
- state.reported = { task: next };
1281
- const cause = next === undefined ? "missing_task" : "incompatible_task";
1282
- this.#report(new Error(`Task ${current.id} keeps running under its old ${current.kind} definition`, { cause }));
1283
- }
2156
+ if (next === state.task)
2157
+ return true;
2158
+ if (next !== undefined && canReserve(next, current)) {
2159
+ tx.setTask(withState(current, { status: "pending", checkpoint: current.state.checkpoint }));
2160
+ return false;
2161
+ }
2162
+ if (state.reported === undefined || state.reported.task !== next) {
2163
+ state.reported = { task: next };
2164
+ this.#report(new Error(`Task ${current.id} keeps running under its old ${current.kind} definition`), {
2165
+ kind: "task_outdated",
2166
+ ...this.#aboutTask({ ...recordTask(current), extension }),
2167
+ reason: outdatedReason(next, current),
2168
+ }, context);
1284
2169
  }
1285
2170
  return true;
1286
2171
  }
@@ -1290,45 +2175,72 @@ export class TaskScheduler {
1290
2175
  const current = this.#live.get(invocation.taskId);
1291
2176
  if (current === undefined || this.#closing)
1292
2177
  return;
1293
- let failure;
1294
- try {
1295
- const phase = { snapshot: () => reservation.snapshot, task: () => reservation.task, agent: undefined };
1296
- const runtime = this.#runtime(invocation, phase);
1297
- await erased(reservation.task).abort(current, runtime, invocation.context);
1298
- }
1299
- catch (error) {
1300
- failure = { error };
1301
- }
1302
- const message = `Abort handler of task ${invocation.taskId} returned without a terminal outcome`;
1303
- await this.#step(invocation, () => ({ fault: failure?.error ?? new Error(message) }));
2178
+ await this.#phase(invocation, current, async () => {
2179
+ let failure;
2180
+ try {
2181
+ const phase = {
2182
+ snapshot: () => reservation.snapshot,
2183
+ task: () => reservation.task,
2184
+ agent: undefined,
2185
+ };
2186
+ const runtime = this.#runtime(invocation, phase);
2187
+ await erased(reservation.task).abort(current, runtime, invocation.context);
2188
+ }
2189
+ catch (error) {
2190
+ failure = { error };
2191
+ }
2192
+ const message = `Abort handler of task ${invocation.taskId} returned without a terminal outcome`;
2193
+ await this.#step(invocation, () => ({ kind: "fault", error: failure?.error ?? new Error(message) }));
2194
+ });
1304
2195
  }
1305
2196
  /**
1306
- * One synchronous decision on the Session line. A task that is no longer running (rule 1: terminal, `completing`, or
1307
- * `waiting`) or a closing Harness (rule 2) ends the invocation without a write; otherwise `decide` may stage a write
1308
- * and returns whether the invocation continues. Ending happens inside the callback, before a fault's Harness
1309
- * cleanup. A rejected step, such as admission after close, also ends the invocation.
2197
+ * One decision on the Session line, before a phase: a decision region (`#region()`) that reads the task's record. A
2198
+ * task that is no longer running (rule 1: terminal, `completing`, `waiting`, or `sleeping`) or a closing Harness (rule
2199
+ * 2) ends the invocation without a write; otherwise `plan` decides, and a decision that stands is applied: the
2200
+ * invocation continues, with `progress` for rule 5, or ends, with a fault written. Ending happens inside the callback,
2201
+ * before a fault's Harness cleanup. A rejected step, such as admission after close, also ends the invocation. Its
2202
+ * outcome goes to the invocation only, so it is a scheduler commit (`commitInternal()`).
1310
2203
  */
1311
- async #step(invocation, decide) {
2204
+ async #step(invocation, plan, progress) {
2205
+ let fault;
1312
2206
  try {
1313
- return await this.#session.commitWith(async (tx) => {
1314
- const found = this.#live.get(invocation.taskId);
1315
- const current = found?.state.status === "running" ? found : undefined;
1316
- const decision = current !== undefined && !this.#closing ? decide(tx, current) : false;
1317
- if (decision === true)
2207
+ const continued = await this.#session.commitInternal(async (tx) => {
2208
+ const decided = await this.#region(() => {
2209
+ const found = this.#liveRecord(invocation.taskId);
2210
+ if (found?.state.status !== "running" || this.#closing)
2211
+ return { plan: END };
2212
+ const current = found;
2213
+ return { current, plan: plan(current) };
2214
+ });
2215
+ const { current } = decided;
2216
+ const step = decided.plan;
2217
+ if (step.kind === "continue")
2218
+ return current;
2219
+ if (step.kind === "progress" && progress(tx, current))
1318
2220
  return current;
1319
2221
  this.#end(invocation);
1320
- if (decision !== false) {
1321
- const message = errorMessage(decision.fault);
1322
- await this.#terminate(tx, current, { status: "faulted", error: { message } });
2222
+ if (step.kind === "cascade") {
2223
+ this.#cascadePending = true;
2224
+ this.#scheduleReconcile();
2225
+ }
2226
+ if (step.kind === "fault") {
2227
+ fault = { error: step.error };
2228
+ const message = errorMessage(step.error);
2229
+ invocation.fault = { status: "faulted", error: { message } };
2230
+ await this.#terminate(tx, current, invocation.fault);
1323
2231
  }
1324
2232
  return undefined;
1325
- }, this.#context);
2233
+ }, withoutAbortSignal(invocation.context));
2234
+ // Reported once the fault is durable: a host may count it, for example toward a breaker.
2235
+ if (fault !== undefined)
2236
+ this.#report(fault.error, { kind: "task_faulted", ...this.#aboutTask(invocation), reason: "threw" }, invocation.context);
2237
+ return continued;
1326
2238
  }
1327
2239
  catch (error) {
1328
2240
  // The step writes the scheduler's own decision, a fault or a terminal record; a throw there would otherwise
1329
2241
  // leave the task running, to be reserved and run again.
1330
2242
  this.#end(invocation);
1331
- this.#failSession(error);
2243
+ this.#failSession(error, invocation.context);
1332
2244
  return undefined;
1333
2245
  }
1334
2246
  }
@@ -1338,7 +2250,7 @@ export class TaskScheduler {
1338
2250
  */
1339
2251
  async #terminate(tx, record, outcome) {
1340
2252
  await this.#loadScopes(false);
1341
- if (this.#hasOwnedLive(record.id, overlayOf(tx))) {
2253
+ if (await this.#region(() => this.#hasOwnedLive(record.id, overlayOf(tx)), this.#reloadScopes)) {
1342
2254
  tx.setTask(withState(record, { status: "completing", outcome }));
1343
2255
  return;
1344
2256
  }
@@ -1353,17 +2265,22 @@ export class TaskScheduler {
1353
2265
  async #commitState(tx, invocation, current, next) {
1354
2266
  if (next.status === "waiting")
1355
2267
  await this.#validateWait(tx, invocation, current, next.on, next.policy);
2268
+ if (next.status === "sleeping")
2269
+ validateSleep(invocation, current.id, next.until);
1356
2270
  if (next.status === "terminal") {
1357
2271
  const overlay = overlayOf(tx);
1358
2272
  await this.#loadScopes(false);
1359
2273
  for (const record of overlay.tasks.values())
1360
2274
  await this.#loadChain(parentOf(record), overlay);
1361
- if (this.#hasOwnedLive(current.id, overlay)) {
1362
- tx.setTask(withState(current, { status: "completing", outcome: next.outcome }));
2275
+ if (await this.#region(() => this.#hasOwnedLive(current.id, overlayOf(tx)), this.#reloadScopes)) {
2276
+ tx.setTask(withState(progressed(current), { status: "completing", outcome: next.outcome }));
1363
2277
  return;
1364
2278
  }
1365
2279
  }
1366
- tx.setTask(withState(current, next));
2280
+ // A state the task committed that differs from its current one is progress: its restart count starts over. The
2281
+ // same checkpoint committed again, for example as intent before an effect, is not.
2282
+ const same = jsonEqual(current.state, next);
2283
+ tx.setTask(withState(same ? current : progressed(current), next));
1367
2284
  }
1368
2285
  /**
1369
2286
  * A wait names existing tasks, and `failFast` only tasks the waiter owns. It may not close a cycle in the wait-for
@@ -1376,37 +2293,64 @@ export class TaskScheduler {
1376
2293
  async #validateWait(tx, invocation, current, on, policy) {
1377
2294
  if (invocation.mode === "abort")
1378
2295
  throw new Error(`Abort handler of task ${current.id} cannot wait`);
1379
- const overlay = overlayOf(tx);
2296
+ // Owned work is found by walks down, which need the chains of every live task loaded.
2297
+ await this.#loadScopes(false);
2298
+ // Tasks in `on` that are not live, read from Storage: terminal, or missing.
2299
+ const ended = new Map();
2300
+ for (;;) {
2301
+ const checked = await this.#region(() => this.#checkWait(current, on, policy, overlayOf(tx), ended), this.#reloadScopes);
2302
+ if (checked === undefined)
2303
+ return;
2304
+ if ("missing" in checked) {
2305
+ for (const id of checked.missing)
2306
+ ended.set(id, (await this.#session.lineStorage.task(id, this.#context)) ?? null);
2307
+ continue;
2308
+ }
2309
+ throw checked.error;
2310
+ }
2311
+ }
2312
+ /**
2313
+ * `#validateWait()`'s decision region: why the wait is refused, `undefined` when it may be stored, or the tasks in
2314
+ * `on` it needs from Storage first, when `ended` lacks them.
2315
+ */
2316
+ #checkWait(current, on, policy, overlay, ended) {
1380
2317
  const targets = [];
2318
+ const missing = [];
1381
2319
  for (const id of on) {
1382
- const member = overlay.tasks.get(id) ?? this.#live.get(id) ?? (await this.#storage.task(id, this.#context));
1383
- if (member === undefined)
1384
- throw new Error(`Task ${id} does not exist`);
2320
+ const member = overlay.tasks.get(id) ?? this.#liveRecord(id) ?? ended.get(id);
2321
+ if (member === undefined) {
2322
+ missing.push(id);
2323
+ continue;
2324
+ }
2325
+ if (member === null)
2326
+ return { error: new Error(`Task ${id} does not exist`) };
1385
2327
  if (policy === "failFast" && member.owner !== current.id) {
1386
- throw new Error(`Task ${current.id} can wait failFast only on tasks it owns; ${id} is not one`);
2328
+ return { error: new Error(`Task ${current.id} can wait failFast only on tasks it owns; ${id} is not one`) };
1387
2329
  }
1388
2330
  // A live child already holds the waiter (task-owned tasks are never background): waiting on it adds no edge. A
1389
2331
  // terminal task has no edges.
1390
2332
  if (member.state.status !== "terminal" && member.owner !== current.id)
1391
2333
  targets.push(id);
1392
2334
  }
2335
+ if (missing.length > 0)
2336
+ return { missing };
1393
2337
  if (targets.length === 0)
1394
- return;
1395
- const cycle = await this.#waitCycle(current.id, targets, overlay);
1396
- if (cycle !== undefined) {
1397
- throw new Error(`Task ${current.id} cannot wait on ${cycle[0]}: ${[current.id, ...cycle].join(" -> ")} is a wait cycle`);
1398
- }
2338
+ return undefined;
2339
+ const cycle = this.#waitCycle(current.id, targets, overlay);
2340
+ if (cycle === undefined)
2341
+ return undefined;
2342
+ return {
2343
+ error: new Error(`Task ${current.id} cannot wait on ${cycle[0]}: ${[current.id, ...cycle].join(" -> ")} is a wait cycle`),
2344
+ };
1399
2345
  }
1400
2346
  /**
1401
2347
  * The path from a member of `on` back to `task` in the wait-for graph, if the wait would close a cycle (spec §5.1).
1402
2348
  * An edge X -> Y means X cannot settle before Y: Y is in the `on` of waiting X that is not abort-marked, or Y is live
1403
- * ordinary owned work of X,
1404
- * which holds X as `completing`. Each task is visited once and each subtree walked once, so a check is bounded by the
1405
- * live work it reaches.
2349
+ * ordinary owned work of X, which holds X as `completing`. Each task is visited once and each subtree walked once,
2350
+ * so a check is bounded by the live work it reaches. Part of a decision region: it records every record it reads.
2351
+ * Owned work is found by walks down, which need the chains of every live task loaded.
1406
2352
  */
1407
- async #waitCycle(task, on, overlay) {
1408
- // Owned work is found by walks down, which need the chains of every live task loaded.
1409
- await this.#loadScopes(false);
2353
+ #waitCycle(task, on, overlay) {
1410
2354
  const from = new Map();
1411
2355
  const walked = new Set();
1412
2356
  const pending = [];
@@ -1427,7 +2371,7 @@ export class TaskScheduler {
1427
2371
  for (let id = pending.pop(); id !== undefined; id = pending.pop()) {
1428
2372
  if (id === task)
1429
2373
  return path(id);
1430
- const record = overlay.tasks.get(id) ?? this.#live.get(id);
2374
+ const record = overlay.tasks.get(id) ?? this.#liveRecord(id);
1431
2375
  if (record === undefined || record.state.status === "terminal")
1432
2376
  continue;
1433
2377
  // An abort-marked task waits only for its owned work, as abort runs bottom-up (`#waitingOn`).
@@ -1464,12 +2408,18 @@ export class TaskScheduler {
1464
2408
  }
1465
2409
  for (const watch of invocation.watches)
1466
2410
  void watch.stop();
2411
+ for (const disarm of invocation.timers)
2412
+ disarm();
2413
+ invocation.timers.clear();
1467
2414
  // Pending waits bound to the invocation, such as a tool's waitForTask(), reject with it.
1468
2415
  invocation.controller.abort(endedError(invocation));
1469
2416
  }
1470
2417
  /** No live non-background task in the scope; a task whose owner edges are not loaded yet counts as inside. */
1471
2418
  #idle(conversationId) {
1472
2419
  const scope = conversationId === undefined ? { roots: true } : { conversation: conversationId };
2420
+ // In a decision region, global idle reads the roots it starts from.
2421
+ if (conversationId === undefined)
2422
+ this.#record(ROOTS_KEY);
1473
2423
  // Tasks whose chain is not loaded are not listed below their scope yet; one counts as inside unless its walk up
1474
2424
  // decides otherwise.
1475
2425
  for (const id of this.#unloaded) {
@@ -1501,7 +2451,7 @@ export class TaskScheduler {
1501
2451
  };
1502
2452
  const hooks = {
1503
2453
  each: async (name, invoke) => {
1504
- for (const handlers of agentHooks(await agent(), phase.task().definition.name)) {
2454
+ for (const { extension, handlers } of agentHooks(await agent(), phase.task().definition.name)) {
1505
2455
  const handler = handlers[name];
1506
2456
  if (typeof handler !== "function")
1507
2457
  continue;
@@ -1511,17 +2461,20 @@ export class TaskScheduler {
1511
2461
  catch (error) {
1512
2462
  if (invocation.controller.signal.aborted)
1513
2463
  throw error;
1514
- this.#report(error);
2464
+ this.#report(error, { kind: "extension_error", extension, task: reportTask(invocation) }, invocation.context);
1515
2465
  }
1516
2466
  }
1517
2467
  },
1518
2468
  };
1519
2469
  const settings = this.#settings;
2470
+ const models = this.#models;
1520
2471
  return {
1521
2472
  taskId: invocation.taskId,
1522
2473
  conversationId: invocation.conversationId,
1523
2474
  signal: invocation.controller.signal,
1524
- models: this.#models,
2475
+ get models() {
2476
+ return models();
2477
+ },
1525
2478
  agent: (context) => invocation.ended ? Promise.reject(endedError(invocation)) : awaitWithContext(agent(), context),
1526
2479
  get settings() {
1527
2480
  return settings();
@@ -1531,11 +2484,46 @@ export class TaskScheduler {
1531
2484
  get registry() {
1532
2485
  return phase.snapshot();
1533
2486
  },
1534
- commit: (change, context) => this.#gated(invocation, async (tx, current) => {
1535
- const next = await change(tx, current);
1536
- if (next !== undefined)
1537
- await this.#commitState(tx, invocation, current, next);
1538
- }, context),
2487
+ commit: async (change, context, options) => {
2488
+ let checkpoint;
2489
+ let interrupted;
2490
+ await this.#gated(invocation, async (tx, current) => {
2491
+ const next = await change(tx, current);
2492
+ if (next !== undefined)
2493
+ await this.#commitState(tx, invocation, current, next);
2494
+ if (next?.status === "running" && current.state.checkpoint.phase !== next.checkpoint.phase) {
2495
+ checkpoint = next.checkpoint.phase;
2496
+ }
2497
+ // A task that ends for crashes says so (`task_interrupted`), so a host keeps a crash-loop signal.
2498
+ if (next?.status === "terminal") {
2499
+ const restarts = current.restarts ?? 0;
2500
+ const reason = next.outcome.status === "failed" ? interruptionOf(next.outcome.error) : undefined;
2501
+ if (reason !== undefined && next.outcome.status === "failed") {
2502
+ interrupted = { message: next.outcome.error.message, reason, restarts };
2503
+ }
2504
+ else {
2505
+ // A task that ended for crashes with another outcome says so itself, as a generation whose run
2506
+ // answered ends `completed` without its after-run tasks.
2507
+ const message = endedByCrashesStates.get(next);
2508
+ if (message !== undefined)
2509
+ interrupted = { message, reason: "restart_limit", restarts };
2510
+ }
2511
+ }
2512
+ }, context, options);
2513
+ if (interrupted !== undefined) {
2514
+ this.#report(new Error(interrupted.message), {
2515
+ kind: "task_interrupted",
2516
+ ...this.#aboutTask(invocation),
2517
+ reason: interrupted.reason,
2518
+ restarts: interrupted.restarts,
2519
+ }, invocation.context);
2520
+ }
2521
+ if (checkpoint !== undefined) {
2522
+ invocation.context.telemetry.addEvent("pi.durable.checkpoint", {
2523
+ phase: checkpoint,
2524
+ });
2525
+ }
2526
+ },
1539
2527
  memo: ((name, ...rest) => {
1540
2528
  if (rest.length === 1) {
1541
2529
  return this.#read(invocation, async () => memoOf(this.#live.get(invocation.taskId), name));
@@ -1549,11 +2537,11 @@ export class TaskScheduler {
1549
2537
  return candidate;
1550
2538
  }, rest[1]);
1551
2539
  }),
1552
- sleep: (until, context) => this.#sleep(invocation, until, context),
2540
+ sleepUntil: (until, checkpoint, context) => this.#gated(invocation, (tx, current) => this.#commitState(tx, invocation, current, { status: "sleeping", checkpoint, until }), context),
1553
2541
  watchDoc: ((...args) => this.#watchDoc(invocation, args)),
1554
2542
  snapshot: ((...args) => this.#read(invocation, () => sessionMethod(this.#session, "snapshot")(...args))),
1555
2543
  snapshotAsOf: ((...args) => this.#read(invocation, () => sessionMethod(this.#session, "snapshotAsOf")(...args))),
1556
- getTask: ((id, context) => this.#read(invocation, () => this.#session.readOnLine(() => this.#storage.task(id, context)))),
2544
+ getTask: ((id, context) => this.#read(invocation, () => this.#session.readOnLine((storage) => storage.task(id, context)))),
1557
2545
  waitForTask: ((id, context) => this.#read(invocation, () => this.waitForTask(id, withAbortSignal(invocation.controller.signal, context)))),
1558
2546
  ownedTasks: (context) => this.#read(invocation, () => this.#session.readOnLine(async () => {
1559
2547
  // The step that ends the invocation may have run on the line before this.
@@ -1561,12 +2549,16 @@ export class TaskScheduler {
1561
2549
  throw endedError(invocation);
1562
2550
  context.abortSignal?.throwIfAborted();
1563
2551
  // Every live task is listed under its parent node, its owner task when it has one.
1564
- const owned = [];
1565
- for (const node of this.#below.get(invocation.taskId) ?? []) {
1566
- const record = typeof node === "string" ? undefined : this.#live.get(node);
1567
- if (record?.owner === invocation.taskId)
1568
- owned.push(record);
1569
- }
2552
+ const owned = await this.#region(() => {
2553
+ this.#record(ownedKey(invocation.taskId));
2554
+ const records = [];
2555
+ for (const node of this.#below.get(invocation.taskId) ?? []) {
2556
+ const record = typeof node === "string" ? undefined : this.#liveRecord(node);
2557
+ if (record?.owner === invocation.taskId)
2558
+ records.push(record);
2559
+ }
2560
+ return records;
2561
+ });
1570
2562
  // Copies, as from storage: the scheduler reads its own.
1571
2563
  return owned
1572
2564
  .sort((a, b) => a.id - b.id)
@@ -1574,7 +2566,7 @@ export class TaskScheduler {
1574
2566
  })),
1575
2567
  abortOwned: (id, context) => this.#read(invocation, async () => {
1576
2568
  const bound = withAbortSignal(invocation.controller.signal, context);
1577
- const record = await this.#session.readOnLine(() => this.#storage.task(id, bound));
2569
+ const record = await this.#session.readOnLine((storage) => storage.task(id, bound));
1578
2570
  if (record?.owner !== invocation.taskId) {
1579
2571
  throw new Error(`Task ${id} is not owned by task ${invocation.taskId}`);
1580
2572
  }
@@ -1583,10 +2575,10 @@ export class TaskScheduler {
1583
2575
  await this.abort(id, bound, true);
1584
2576
  await this.waitForTask(id, bound);
1585
2577
  }),
1586
- outcomes: ((ids, context) => this.#read(invocation, () => this.#session.readOnLine(async () => {
2578
+ outcomes: ((ids, context) => this.#read(invocation, () => this.#session.readOnLine(async (storage) => {
1587
2579
  const outcomes = [];
1588
2580
  for (const id of ids) {
1589
- const state = (await this.#storage.task(id, context))?.state;
2581
+ const state = (await storage.task(id, context))?.state;
1590
2582
  if (state?.status !== "terminal")
1591
2583
  throw new Error(`Task ${id} is not terminal`);
1592
2584
  outcomes.push(state.outcome);
@@ -1605,13 +2597,24 @@ export class TaskScheduler {
1605
2597
  ? [undefined, args[0], args[1]]
1606
2598
  : [args[0], args[1], args[2]];
1607
2599
  return this.#read(invocation, async () => {
1608
- const found = await this.#session.readOnLine(() => this.#storage.entry(invocation.conversationId, id, context));
1609
- const entry = found?.entry;
2600
+ // An entry of the conversation's kept context range is committed and visible: no Storage read.
2601
+ const kept = this.#contexts.get(invocation.conversationId);
2602
+ const entry = (kept === undefined ? undefined : rangeEntry(kept.range, id)) ??
2603
+ (await this.#session.readOnLine((storage) => storage.entry(invocation.conversationId, id, context)))
2604
+ ?.entry;
1610
2605
  return token === undefined || entry?.kind === token.kind ? entry : undefined;
1611
2606
  });
1612
2607
  }),
2608
+ entries: (query, limit, cursor, context) => this.#read(invocation, () => this.#session.readOnLine((storage) => storage.scanEntries(conversationEntries(invocation.conversationId, query), limit, cursor, context))),
1613
2609
  context: (conversationId, context, options) => this.#read(invocation, async () => {
1614
- const { view, range } = await readContextFrom(this.#session, this.#storage, conversationId, context, options?.at, this.#contexts.get(conversationId)?.range);
2610
+ const read = await readContextFrom(this.#session, this.#storage, conversationId, context, options?.at, this.#contexts.get(conversationId)?.range);
2611
+ const { view } = read;
2612
+ // Checked again here, synchronously up to the install below: a commit published since the read returned
2613
+ // was not seen by `#advanceContexts`, so a range that missed it must not be advanced.
2614
+ const range = read.range?.current === true &&
2615
+ this.#session.knownLatestEntryId(conversationId) !== read.range.bounds.tail
2616
+ ? { ...read.range, current: false }
2617
+ : read.range;
1615
2618
  // Keep it unless the invocation ended or a concurrent read already kept a newer range.
1616
2619
  const kept = this.#contexts.get(conversationId);
1617
2620
  if (range !== undefined &&
@@ -1621,9 +2624,17 @@ export class TaskScheduler {
1621
2624
  // A read of another, idle conversation starts or continues its retention period.
1622
2625
  if (!this.#idle(conversationId)) {
1623
2626
  this.#contexts.set(conversationId, { range, idleSince: undefined });
2627
+ this.#idleOrder.delete(conversationId);
1624
2628
  }
1625
2629
  else if (this.#contextRetentionMs() > 0) {
1626
- this.#contexts.set(conversationId, { range, idleSince: kept?.idleSince ?? this.#now() });
2630
+ // An idle period that already started keeps its expiry and its place; a new one starts now.
2631
+ const idleSince = kept?.idleSince ?? this.#now();
2632
+ const expiresAt = kept?.idleSince !== undefined && kept.expiresAt !== undefined
2633
+ ? kept.expiresAt
2634
+ : idleSince + this.#contextRetentionMs();
2635
+ this.#contexts.set(conversationId, { range, idleSince, expiresAt });
2636
+ if (kept?.idleSince === undefined)
2637
+ this.#markIdle(conversationId, idleSince);
1627
2638
  this.#scheduleExpiry();
1628
2639
  }
1629
2640
  }
@@ -1634,13 +2645,43 @@ export class TaskScheduler {
1634
2645
  throw endedError(invocation);
1635
2646
  return this.#now();
1636
2647
  },
1637
- report: (error) => {
2648
+ setTimer: (callback, delayMs) => {
2649
+ // An ended invocation's timers never fire.
2650
+ if (invocation.ended)
2651
+ return () => { };
2652
+ const disarm = this.#timers.foreground(() => {
2653
+ invocation.timers.delete(disarm);
2654
+ if (!invocation.ended)
2655
+ callback();
2656
+ }, delayMs);
2657
+ invocation.timers.add(disarm);
2658
+ return () => {
2659
+ invocation.timers.delete(disarm);
2660
+ disarm();
2661
+ };
2662
+ },
2663
+ report: (error, from) => {
1638
2664
  if (invocation.ended)
1639
2665
  throw endedError(invocation);
1640
- this.#report(error);
2666
+ // The task's own failure names the task's extension; any other names whose code failed, if anyone's.
2667
+ const task = reportTask(invocation);
2668
+ this.#report(error, from === undefined
2669
+ ? { kind: "task_error", ...this.#aboutTask(invocation) }
2670
+ : from.kind === "extension"
2671
+ ? { kind: "extension_error", task, ...optionalExtension(from.extension) }
2672
+ : { kind: "host_error", callback: from.callback, task }, invocation.context);
1641
2673
  },
1642
2674
  };
1643
2675
  }
2676
+ /**
2677
+ * On the line: wait until every commit admitted before is published, for a decision that reads what cannot be keyed,
2678
+ * every live task (open, inspection) or every task a conversation abort reaches.
2679
+ */
2680
+ async #published() {
2681
+ const gate = this.#session.touch(ALL_KEYS);
2682
+ if (gate !== undefined)
2683
+ await gate;
2684
+ }
1644
2685
  /** Run a committed-state read unless the invocation has ended. */
1645
2686
  async #read(invocation, read) {
1646
2687
  if (invocation.ended)
@@ -1648,7 +2689,7 @@ export class TaskScheduler {
1648
2689
  return read();
1649
2690
  }
1650
2691
  /** Commit after rereading the task on the line and gating the invocation. */
1651
- #gated(invocation, change, context) {
2692
+ #gated(invocation, change, context, options) {
1652
2693
  if (invocation.ended)
1653
2694
  return Promise.reject(endedError(invocation));
1654
2695
  return this.#session.commitWith(async (tx) => {
@@ -1656,33 +2697,22 @@ export class TaskScheduler {
1656
2697
  throw endedError(invocation);
1657
2698
  if (this.#closing)
1658
2699
  throw closedError(this.#session);
1659
- const found = this.#live.get(invocation.taskId);
1660
- if (found === undefined)
1661
- throw new Error(`Task ${invocation.taskId} is terminal`);
1662
- if (found.state.status !== "running")
1663
- throw new Error(`Task ${invocation.taskId} is ${found.state.status}`);
1664
- const current = found;
1665
- if (invocation.mode === "run" && current.abortRequested) {
1666
- throw new Error(`Task ${invocation.taskId} has a durable abort mark`);
1667
- }
2700
+ const current = await this.#region(() => {
2701
+ const found = this.#liveRecord(invocation.taskId);
2702
+ if (found === undefined)
2703
+ throw new Error(`Task ${invocation.taskId} is terminal`);
2704
+ if (found.state.status !== "running")
2705
+ throw new Error(`Task ${invocation.taskId} is ${found.state.status}`);
2706
+ if (invocation.mode === "run" && found.abortRequested) {
2707
+ throw new Error(`Task ${invocation.taskId} has a durable abort mark`);
2708
+ }
2709
+ return found;
2710
+ });
2711
+ // The step that ends it may have run while the region waited.
2712
+ if (invocation.ended)
2713
+ throw endedError(invocation);
1668
2714
  return change(tx, current);
1669
- }, context, { conversationId: invocation.conversationId, taskId: invocation.taskId });
1670
- }
1671
- /** Wait until the Harness clock reaches `until`, rechecking it after every timer. */
1672
- async #sleep(invocation, until, context) {
1673
- if (invocation.ended)
1674
- throw endedError(invocation);
1675
- const signals = [invocation.controller.signal];
1676
- if (context.abortSignal !== undefined)
1677
- signals.push(context.abortSignal);
1678
- const signal = AbortSignal.any(signals);
1679
- for (;;) {
1680
- signal.throwIfAborted();
1681
- const remaining = until - this.#now();
1682
- if (remaining <= 0)
1683
- return;
1684
- await delay(Math.min(remaining, MAX_TIMER_DELAY), signal);
1685
- }
2715
+ }, context, { conversationId: invocation.conversationId, taskId: invocation.taskId }, options);
1686
2716
  }
1687
2717
  async #watchDoc(invocation, args) {
1688
2718
  if (invocation.ended)
@@ -1699,15 +2729,64 @@ export class TaskScheduler {
1699
2729
  return watch;
1700
2730
  }
1701
2731
  }
1702
- /** The record with an abort mark: `restart` for an abandonment; without a reason, a request, which replaces `restart`. */
2732
+ const REGISTRY = { kind: "host_error", callback: "registry" };
2733
+ const SETTINGS = { kind: "host_error", callback: "settings" };
2734
+ const ON_WAKE = { kind: "host_error", callback: "onWake" };
2735
+ const TELEMETRY = { kind: "host_error", callback: "telemetry" };
2736
+ /** The extension that installed `task` in `snapshot`, the last one that lists it; none for a built-in task. */
2737
+ function installerOf(snapshot, task) {
2738
+ if (task === undefined)
2739
+ return undefined;
2740
+ return snapshot.installed().findLast((installed) => installed.tasks?.includes(task) === true)?.name;
2741
+ }
2742
+ /** The report info of a failure of a task. */
2743
+ function reportTask(task) {
2744
+ return { id: task.taskId, kind: task.kind, conversationId: task.conversationId };
2745
+ }
2746
+ /** A stored task's identity, as an invocation names it. */
2747
+ function recordTask(record) {
2748
+ return { taskId: record.id, kind: record.kind, conversationId: record.conversationId };
2749
+ }
2750
+ /**
2751
+ * The record with an abort mark: `restart` for an abandonment; without a reason, a request, which replaces `restart`.
2752
+ * The restart count starts over: it now counts cut-offs of the abort handler.
2753
+ */
2754
+ /** Why a failed outcome's error says crashes ended the task (`detail.reason`), if it does. */
2755
+ /**
2756
+ * Terminal states a built-in task commits because it was cut off `settings.restartLimit` times, when their outcome does
2757
+ * not say so (a failure with detail `{ reason: "restart_limit" }` does), with the message the scheduler reports them
2758
+ * `task_interrupted` with. Keyed by the state object the task returns, so nothing is added to the stored state.
2759
+ */
2760
+ const endedByCrashesStates = new WeakMap();
2761
+ /** `state`, marked as ending its task for crashes (`task_interrupted` with `message`), whatever its outcome. */
2762
+ export function endedByCrashes(state, message) {
2763
+ endedByCrashesStates.set(state, message);
2764
+ return state;
2765
+ }
2766
+ function interruptionOf(error) {
2767
+ const detail = error.detail;
2768
+ if (typeof detail !== "object" || detail === null || !("reason" in detail))
2769
+ return undefined;
2770
+ return detail.reason === "restart_limit" || detail.reason === "interrupted" ? detail.reason : undefined;
2771
+ }
1703
2772
  function withAbortMark(record, reason) {
1704
- const { abortReason: _reason, ...rest } = record;
2773
+ const { abortReason: _reason, restarts: _restarts, ...rest } = record;
1705
2774
  return { ...rest, abortRequested: true, ...(reason === undefined ? {} : { abortReason: reason }) };
1706
2775
  }
1707
2776
  /** A live owner's durable cancellation intent: its abort mark, or a held outcome other than `completed`. */
1708
2777
  function cancellationIntent(record) {
1709
2778
  return record.state.status !== "terminal" && (record.abortRequested || failedOutcome(record));
1710
2779
  }
2780
+ /**
2781
+ * Whether replacing `previous` (absent: a new task) with `next` changes the cancellation intent set or what it means: a
2782
+ * mark set, upgraded, or cleared, or a held or ended outcome other than `completed`.
2783
+ */
2784
+ function intentChanges(previous, next) {
2785
+ return ((previous?.abortRequested ?? false) !== next.abortRequested ||
2786
+ previous?.abortReason !== next.abortReason ||
2787
+ (previous !== undefined && failedOutcome(previous)) !== failedOutcome(next) ||
2788
+ (previous !== undefined && cancellationIntent(previous)) !== cancellationIntent(next));
2789
+ }
1711
2790
  /** Whether the record holds or ends with an outcome other than `completed`. */
1712
2791
  function failedOutcome(record) {
1713
2792
  const state = record.state;
@@ -1761,30 +2840,41 @@ function erased(task) {
1761
2840
  function endedError(invocation) {
1762
2841
  return new Error(`Task ${invocation.taskId} invocation has ended`);
1763
2842
  }
1764
- /** Replace a live record's state; memos disappear once an outcome is decided. */
2843
+ /** The record without its restart count, as after a state the task committed. */
2844
+ function progressed(record) {
2845
+ if (record.restarts === undefined)
2846
+ return record;
2847
+ const { restarts: _restarts, ...rest } = record;
2848
+ return rest;
2849
+ }
2850
+ /** Replace a live record's state; memos and the restart count disappear once an outcome is decided. */
1765
2851
  function withState(record, state) {
1766
2852
  if (state.status !== "terminal" && state.status !== "completing")
1767
2853
  return { ...record, state };
1768
- const { memos: _memos, ...rest } = record;
2854
+ const { memos: _memos, restarts: _restarts, ...rest } = record;
1769
2855
  return { ...rest, state };
1770
2856
  }
1771
2857
  /** Whether a definition can take the task at reservation: same version, or newer with a migration. */
2858
+ /**
2859
+ * Why a replacement definition cannot take a running task, in `#fit()`'s words: none is registered, it is older than
2860
+ * the task, or it is newer without a `migrate`.
2861
+ */
2862
+ function outdatedReason(task, record) {
2863
+ if (task === undefined)
2864
+ return "missing_task";
2865
+ return task.definition.version < record.version ? "task_too_old" : "migration_failed";
2866
+ }
1772
2867
  function canReserve(task, record) {
1773
2868
  const definition = task.definition;
1774
2869
  return (definition.version === record.version || (definition.version > record.version && definition.migrate !== undefined));
1775
2870
  }
1776
- function delay(ms, signal) {
1777
- return new Promise((resolve, reject) => {
1778
- const onAbort = () => {
1779
- clearTimeout(timer);
1780
- reject(signal.reason);
1781
- };
1782
- const timer = setTimeout(() => {
1783
- signal.removeEventListener("abort", onAbort);
1784
- resolve();
1785
- }, ms);
1786
- signal.addEventListener("abort", onAbort, { once: true });
1787
- });
2871
+ /** A sleep's `until` is a time on the Harness clock; an abort handler cannot sleep. */
2872
+ function validateSleep(invocation, task, until) {
2873
+ if (invocation.mode === "abort")
2874
+ throw new Error(`Abort handler of task ${task} cannot sleep`);
2875
+ if (typeof until !== "number" || !Number.isFinite(until) || until < 0) {
2876
+ throw new Error(`Task ${task} cannot sleep until ${until}: not a time in milliseconds`);
2877
+ }
1788
2878
  }
1789
2879
  /** Structural equality of two JSON values; object key order is ignored. */
1790
2880
  function jsonEqual(left, right) {