@earendil-works/pi-durable 1.4.0 → 1.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (270) hide show
  1. package/CHANGELOG.md +204 -0
  2. package/README.md +195 -85
  3. package/dist/codemode/codemode.d.ts +6 -5
  4. package/dist/codemode/codemode.d.ts.map +1 -1
  5. package/dist/codemode/codemode.js +10 -49
  6. package/dist/codemode/codemode.js.map +1 -1
  7. package/dist/codemode/models.d.ts +2 -1
  8. package/dist/codemode/models.d.ts.map +1 -1
  9. package/dist/codemode/models.js +4 -4
  10. package/dist/codemode/models.js.map +1 -1
  11. package/dist/documents.d.ts +1 -1
  12. package/dist/documents.d.ts.map +1 -1
  13. package/dist/documents.js +1 -1
  14. package/dist/documents.js.map +1 -1
  15. package/dist/entries.d.ts +1 -1
  16. package/dist/entries.d.ts.map +1 -1
  17. package/dist/entries.js.map +1 -1
  18. package/dist/env/index.d.ts +13 -11
  19. package/dist/env/index.d.ts.map +1 -1
  20. package/dist/env/index.js.map +1 -1
  21. package/dist/env/node-watch.d.ts +4 -4
  22. package/dist/env/node-watch.d.ts.map +1 -1
  23. package/dist/env/node-watch.js +18 -16
  24. package/dist/env/node-watch.js.map +1 -1
  25. package/dist/env/node.d.ts +3 -3
  26. package/dist/env/node.d.ts.map +1 -1
  27. package/dist/env/node.js +1 -1
  28. package/dist/env/node.js.map +1 -1
  29. package/dist/errors.d.ts +18 -2
  30. package/dist/errors.d.ts.map +1 -1
  31. package/dist/errors.js +26 -3
  32. package/dist/errors.js.map +1 -1
  33. package/dist/harness/agent.d.ts.map +1 -1
  34. package/dist/harness/agent.js +37 -3
  35. package/dist/harness/agent.js.map +1 -1
  36. package/dist/harness/compaction.d.ts +8 -3
  37. package/dist/harness/compaction.d.ts.map +1 -1
  38. package/dist/harness/compaction.js +40 -11
  39. package/dist/harness/compaction.js.map +1 -1
  40. package/dist/harness/context.d.ts +24 -8
  41. package/dist/harness/context.d.ts.map +1 -1
  42. package/dist/harness/context.js +71 -25
  43. package/dist/harness/context.js.map +1 -1
  44. package/dist/harness/define.d.ts +8 -4
  45. package/dist/harness/define.d.ts.map +1 -1
  46. package/dist/harness/define.js +4 -0
  47. package/dist/harness/define.js.map +1 -1
  48. package/dist/harness/generation.d.ts +39 -6
  49. package/dist/harness/generation.d.ts.map +1 -1
  50. package/dist/harness/generation.js +451 -100
  51. package/dist/harness/generation.js.map +1 -1
  52. package/dist/harness/harness.d.ts +3 -1
  53. package/dist/harness/harness.d.ts.map +1 -1
  54. package/dist/harness/harness.js +87 -62
  55. package/dist/harness/harness.js.map +1 -1
  56. package/dist/harness/inbox.d.ts +7 -2
  57. package/dist/harness/inbox.d.ts.map +1 -1
  58. package/dist/harness/inbox.js +5 -2
  59. package/dist/harness/inbox.js.map +1 -1
  60. package/dist/harness/json.d.ts +21 -5
  61. package/dist/harness/json.d.ts.map +1 -1
  62. package/dist/harness/json.js +23 -5
  63. package/dist/harness/json.js.map +1 -1
  64. package/dist/harness/live.d.ts +124 -17
  65. package/dist/harness/live.d.ts.map +1 -1
  66. package/dist/harness/live.js +93 -12
  67. package/dist/harness/live.js.map +1 -1
  68. package/dist/harness/partial-changes.d.ts +54 -0
  69. package/dist/harness/partial-changes.d.ts.map +1 -0
  70. package/dist/harness/partial-changes.js +163 -0
  71. package/dist/harness/partial-changes.js.map +1 -0
  72. package/dist/harness/prompt.d.ts +1 -1
  73. package/dist/harness/prompt.d.ts.map +1 -1
  74. package/dist/harness/prompt.js.map +1 -1
  75. package/dist/harness/provider.d.ts +1 -1
  76. package/dist/harness/provider.d.ts.map +1 -1
  77. package/dist/harness/provider.js.map +1 -1
  78. package/dist/harness/registry-reader.d.ts +11 -0
  79. package/dist/harness/registry-reader.d.ts.map +1 -0
  80. package/dist/harness/registry-reader.js +25 -0
  81. package/dist/harness/registry-reader.js.map +1 -0
  82. package/dist/harness/scheduler.d.ts +48 -20
  83. package/dist/harness/scheduler.d.ts.map +1 -1
  84. package/dist/harness/scheduler.js +1006 -380
  85. package/dist/harness/scheduler.js.map +1 -1
  86. package/dist/harness/submissions.d.ts +18 -11
  87. package/dist/harness/submissions.d.ts.map +1 -1
  88. package/dist/harness/submissions.js +63 -26
  89. package/dist/harness/submissions.js.map +1 -1
  90. package/dist/harness/task-graph.d.ts +14 -9
  91. package/dist/harness/task-graph.d.ts.map +1 -1
  92. package/dist/harness/task-graph.js +38 -28
  93. package/dist/harness/task-graph.js.map +1 -1
  94. package/dist/harness/telemetry.d.ts +11 -11
  95. package/dist/harness/telemetry.d.ts.map +1 -1
  96. package/dist/harness/telemetry.js +7 -7
  97. package/dist/harness/telemetry.js.map +1 -1
  98. package/dist/harness/tool.d.ts +3 -1
  99. package/dist/harness/tool.d.ts.map +1 -1
  100. package/dist/harness/tool.js +172 -56
  101. package/dist/harness/tool.js.map +1 -1
  102. package/dist/harness/transcript.d.ts +72 -0
  103. package/dist/harness/transcript.d.ts.map +1 -0
  104. package/dist/harness/transcript.js +264 -0
  105. package/dist/harness/transcript.js.map +1 -0
  106. package/dist/harness/types.d.ts +237 -61
  107. package/dist/harness/types.d.ts.map +1 -1
  108. package/dist/harness/types.js.map +1 -1
  109. package/dist/harness/usage.d.ts +2 -1
  110. package/dist/harness/usage.d.ts.map +1 -1
  111. package/dist/harness/usage.js +1 -1
  112. package/dist/harness/usage.js.map +1 -1
  113. package/dist/harness/util.d.ts +4 -2
  114. package/dist/harness/util.d.ts.map +1 -1
  115. package/dist/harness/util.js +9 -0
  116. package/dist/harness/util.js.map +1 -1
  117. package/dist/harness/view.d.ts +44 -22
  118. package/dist/harness/view.d.ts.map +1 -1
  119. package/dist/harness/view.js +130 -71
  120. package/dist/harness/view.js.map +1 -1
  121. package/dist/index.d.ts +9 -7
  122. package/dist/index.d.ts.map +1 -1
  123. package/dist/index.js +4 -3
  124. package/dist/index.js.map +1 -1
  125. package/dist/search.d.ts +19 -3
  126. package/dist/search.d.ts.map +1 -1
  127. package/dist/search.js +107 -44
  128. package/dist/search.js.map +1 -1
  129. package/dist/session/cached-storage.d.ts +72 -0
  130. package/dist/session/cached-storage.d.ts.map +1 -0
  131. package/dist/session/cached-storage.js +269 -0
  132. package/dist/session/cached-storage.js.map +1 -0
  133. package/dist/session/conversation-tails.d.ts +33 -0
  134. package/dist/session/conversation-tails.d.ts.map +1 -0
  135. package/dist/session/conversation-tails.js +101 -0
  136. package/dist/session/conversation-tails.js.map +1 -0
  137. package/dist/session/forks.d.ts +1 -1
  138. package/dist/session/forks.d.ts.map +1 -1
  139. package/dist/session/forks.js.map +1 -1
  140. package/dist/session/keys.d.ts +69 -0
  141. package/dist/session/keys.d.ts.map +1 -0
  142. package/dist/session/keys.js +161 -0
  143. package/dist/session/keys.js.map +1 -0
  144. package/dist/session/session.d.ts +169 -26
  145. package/dist/session/session.d.ts.map +1 -1
  146. package/dist/session/session.js +1029 -171
  147. package/dist/session/session.js.map +1 -1
  148. package/dist/session/transaction.d.ts +84 -5
  149. package/dist/session/transaction.d.ts.map +1 -1
  150. package/dist/session/transaction.js +227 -40
  151. package/dist/session/transaction.js.map +1 -1
  152. package/dist/session/writer.d.ts +74 -0
  153. package/dist/session/writer.d.ts.map +1 -0
  154. package/dist/session/writer.js +294 -0
  155. package/dist/session/writer.js.map +1 -0
  156. package/dist/storage/indexeddb.d.ts +12 -3
  157. package/dist/storage/indexeddb.d.ts.map +1 -1
  158. package/dist/storage/indexeddb.js +91 -41
  159. package/dist/storage/indexeddb.js.map +1 -1
  160. package/dist/storage/jsonl/index.d.ts +1 -1
  161. package/dist/storage/jsonl/index.d.ts.map +1 -1
  162. package/dist/storage/jsonl/index.js +1 -1
  163. package/dist/storage/jsonl/index.js.map +1 -1
  164. package/dist/storage/jsonl/node.d.ts +3 -3
  165. package/dist/storage/jsonl/node.d.ts.map +1 -1
  166. package/dist/storage/jsonl/node.js +2 -2
  167. package/dist/storage/jsonl/node.js.map +1 -1
  168. package/dist/storage/jsonl/storage.d.ts +46 -5
  169. package/dist/storage/jsonl/storage.d.ts.map +1 -1
  170. package/dist/storage/jsonl/storage.js +158 -43
  171. package/dist/storage/jsonl/storage.js.map +1 -1
  172. package/dist/storage/memory.d.ts +9 -3
  173. package/dist/storage/memory.d.ts.map +1 -1
  174. package/dist/storage/memory.js +28 -2
  175. package/dist/storage/memory.js.map +1 -1
  176. package/dist/storage/sqlite/commit.d.ts +52 -0
  177. package/dist/storage/sqlite/commit.d.ts.map +1 -0
  178. package/dist/storage/sqlite/commit.js +371 -0
  179. package/dist/storage/sqlite/commit.js.map +1 -0
  180. package/dist/storage/sqlite/rows.d.ts +33 -0
  181. package/dist/storage/sqlite/rows.d.ts.map +1 -0
  182. package/dist/storage/sqlite/rows.js +53 -0
  183. package/dist/storage/sqlite/rows.js.map +1 -0
  184. package/dist/storage/sqlite/statements.d.ts +13 -0
  185. package/dist/storage/sqlite/statements.d.ts.map +1 -0
  186. package/dist/storage/sqlite/statements.js +88 -0
  187. package/dist/storage/sqlite/statements.js.map +1 -0
  188. package/dist/storage/sqlite/storage.d.ts +23 -10
  189. package/dist/storage/sqlite/storage.d.ts.map +1 -1
  190. package/dist/storage/sqlite/storage.js +120 -261
  191. package/dist/storage/sqlite/storage.js.map +1 -1
  192. package/dist/storage/store/commit.d.ts +33 -0
  193. package/dist/storage/store/commit.d.ts.map +1 -0
  194. package/dist/storage/store/commit.js +286 -0
  195. package/dist/storage/store/commit.js.map +1 -0
  196. package/dist/storage/store/documents.d.ts +43 -0
  197. package/dist/storage/store/documents.d.ts.map +1 -0
  198. package/dist/storage/store/documents.js +102 -0
  199. package/dist/storage/store/documents.js.map +1 -0
  200. package/dist/storage/store/index.d.ts +2 -0
  201. package/dist/storage/store/index.d.ts.map +1 -0
  202. package/dist/storage/store/index.js +2 -0
  203. package/dist/storage/store/index.js.map +1 -0
  204. package/dist/storage/store/queries.d.ts +49 -0
  205. package/dist/storage/store/queries.d.ts.map +1 -0
  206. package/dist/storage/store/queries.js +93 -0
  207. package/dist/storage/store/queries.js.map +1 -0
  208. package/dist/storage/store/storage.d.ts +60 -0
  209. package/dist/storage/store/storage.d.ts.map +1 -0
  210. package/dist/storage/store/storage.js +438 -0
  211. package/dist/storage/store/storage.js.map +1 -0
  212. package/dist/storage/store/tables.d.ts +54 -0
  213. package/dist/storage/store/tables.d.ts.map +1 -0
  214. package/dist/storage/store/tables.js +108 -0
  215. package/dist/storage/store/tables.js.map +1 -0
  216. package/dist/testing/commit-writes.d.ts +5 -0
  217. package/dist/testing/commit-writes.d.ts.map +1 -0
  218. package/dist/testing/commit-writes.js +12 -0
  219. package/dist/testing/commit-writes.js.map +1 -0
  220. package/dist/testing/env-conformance.js +7 -7
  221. package/dist/testing/env-conformance.js.map +1 -1
  222. package/dist/testing/index.d.ts +1 -0
  223. package/dist/testing/index.d.ts.map +1 -1
  224. package/dist/testing/index.js +1 -0
  225. package/dist/testing/index.js.map +1 -1
  226. package/dist/testing/storage-benchmark.d.ts.map +1 -1
  227. package/dist/testing/storage-benchmark.js +18 -17
  228. package/dist/testing/storage-benchmark.js.map +1 -1
  229. package/dist/testing/storage-conformance.d.ts.map +1 -1
  230. package/dist/testing/storage-conformance.js +354 -69
  231. package/dist/testing/storage-conformance.js.map +1 -1
  232. package/dist/tool-search/tool-search.d.ts +8 -14
  233. package/dist/tool-search/tool-search.d.ts.map +1 -1
  234. package/dist/tool-search/tool-search.js +25 -13
  235. package/dist/tool-search/tool-search.js.map +1 -1
  236. package/dist/tools/bash.d.ts +13 -13
  237. package/dist/tools/bash.d.ts.map +1 -1
  238. package/dist/tools/bash.js +16 -15
  239. package/dist/tools/bash.js.map +1 -1
  240. package/dist/tools/edit.d.ts +7 -7
  241. package/dist/tools/edit.d.ts.map +1 -1
  242. package/dist/tools/edit.js +9 -9
  243. package/dist/tools/edit.js.map +1 -1
  244. package/dist/tools/file-mutation-queue.d.ts +1 -1
  245. package/dist/tools/file-mutation-queue.d.ts.map +1 -1
  246. package/dist/tools/file-mutation-queue.js.map +1 -1
  247. package/dist/tools/index.d.ts +32 -32
  248. package/dist/tools/path-utils.d.ts +1 -1
  249. package/dist/tools/path-utils.d.ts.map +1 -1
  250. package/dist/tools/path-utils.js.map +1 -1
  251. package/dist/tools/read.d.ts +6 -6
  252. package/dist/tools/read.d.ts.map +1 -1
  253. package/dist/tools/read.js +19 -8
  254. package/dist/tools/read.js.map +1 -1
  255. package/dist/tools/write.d.ts +4 -4
  256. package/dist/tools/write.d.ts.map +1 -1
  257. package/dist/tools/write.js +5 -5
  258. package/dist/tools/write.js.map +1 -1
  259. package/dist/types.d.ts +228 -78
  260. package/dist/types.d.ts.map +1 -1
  261. package/dist/types.js.map +1 -1
  262. package/package.json +19 -12
  263. package/dist/harness/events.d.ts +0 -186
  264. package/dist/harness/events.d.ts.map +0 -1
  265. package/dist/harness/events.js +0 -373
  266. package/dist/harness/events.js.map +0 -1
  267. package/dist/session/observation.d.ts +0 -42
  268. package/dist/session/observation.d.ts.map +0 -1
  269. package/dist/session/observation.js +0 -281
  270. package/dist/session/observation.js.map +0 -1
@@ -1,12 +1,15 @@
1
- import { copyJson } from "@earendil-works/chord";
2
- import { awaitWithContext, withAbortSignal, withoutAbortSignal, withTelemetry } from "@earendil-works/chord/context";
1
+ import { awaitWithContext, withAbortSignal, withoutAbortSignal, withTelemetry, } from "@earendil-works/chord/context";
2
+ import { ownedSlice } from "@earendil-works/chord/delta";
3
+ import { copyJson } from "@earendil-works/chord/json";
3
4
  import { createTypedMetrics, createTypedSpanStarter, NOOP_TELEMETRY_CONTEXT, NOOP_TELEMETRY_METRICS, } from "@earendil-works/pi-telemetry";
4
- import { errorMessage } from "../errors.js";
5
+ import { errorMessage, ReadOnlySession } from "../errors.js";
6
+ import { ALL_KEYS, conversationKey, INTENT_KEY, ownedKey, ROOTS_KEY, taskKey } from "../session/keys.js";
5
7
  import { agentHooks, DEFAULT_RESTART_LIMIT } from "./agent.js";
6
- import { readContextFrom } from "./context.js";
8
+ import { advanceRange, rangeEntry, readContextFrom } from "./context.js";
7
9
  import { DURABLE_TELEMETRY_SCHEMA } from "./telemetry.js";
8
- import { closedError, optionalExtension, scanAll, Waiters } from "./util.js";
10
+ import { closedError, conversationEntries, optionalExtension, scanAll, Waiters } from "./util.js";
9
11
  const SCAN_PAGE_SIZE = 256;
12
+ const NO_KEYS = [];
10
13
  const LIVE_STATUSES = ["pending", "running", "waiting", "sleeping", "completing"];
11
14
  /** The part of a record `blockedReason()` reads. */
12
15
  export function blockedSubject(record) {
@@ -16,10 +19,17 @@ export function blockedSubject(record) {
16
19
  version: record.version,
17
20
  abortRequested: record.abortRequested,
18
21
  ...(record.abortReason === undefined ? {} : { abortReason: record.abortReason }),
19
- ...(record.restarts === undefined ? {} : { restarts: record.restarts }),
20
22
  status: record.state.status,
21
23
  };
22
24
  }
25
+ /** Continue with the next phase, the first one. */
26
+ const CONTINUE = { kind: "continue" };
27
+ /** Rule 5: the phase made durable progress; `#progress()` decides once the decision stands. */
28
+ const PROGRESS = { kind: "progress" };
29
+ /** End the invocation without a write. */
30
+ const END = { kind: "end" };
31
+ /** End it: an owner's cancellation intent reaches it before its cascade did; schedule the cascade. */
32
+ const CASCADE = { kind: "cascade" };
23
33
  function conversationNode(id) {
24
34
  return `c${id}`;
25
35
  }
@@ -48,9 +58,16 @@ function parentNode(node) {
48
58
  * later below a dropped node finds it unknown and loads its chain from storage again.
49
59
  *
50
60
  * Invariant: every task transition is decided and written by one callback serialized on the Session line. That covers
51
- * reservation, marks, runtime commits, finalization, and the synchronous step before each phase, which applies the
52
- * precedence rules and writes a fault or handover. Handlers and joins run off the line. An invocation ends inside the
53
- * step that decides its end, so a runtime commit it queued either lands before that decision or is rejected.
61
+ * reservation, marks, runtime commits, finalization, and the step before each phase, which applies the precedence rules
62
+ * and writes a fault or handover. Handlers and joins run off the line. An invocation ends inside the step that decides
63
+ * its end, so a runtime commit it queued either lands before that decision or is rejected.
64
+ *
65
+ * Decisions stay serialized; their storage overlaps (group commit). The Session stores the commits of several jobs in
66
+ * one call while the line runs on, and this memory changes only as each commit is published. So every decision reads
67
+ * it in a decision region (`#region()`): synchronous, recording the keys it reads (a task's record, the live work below
68
+ * a node, the cancellation intent set), staging nothing; when one of them has an unpublished writer, the region waits
69
+ * for its publication and runs again, and only a region that read nothing unpublished is applied. Every commit writes
70
+ * the memberships its task changes alter (`membershipKeys()`), so a walk that a pending commit would change waits.
54
71
  */
55
72
  export class TaskScheduler {
56
73
  #telemetry;
@@ -70,6 +87,7 @@ export class TaskScheduler {
70
87
  #writeAtClose;
71
88
  #report;
72
89
  #settleOutcome;
90
+ #limitsRestarts;
73
91
  #withdrawInputs;
74
92
  #conversation;
75
93
  #context;
@@ -80,6 +98,20 @@ export class TaskScheduler {
80
98
  * first idle check after `settings.contextRetentionMs` of idleness, and at close.
81
99
  */
82
100
  #contexts = new Map();
101
+ /**
102
+ * The idle kept contexts by `idleSince`, oldest first: the expiry order, so neither an expiry nor finding the next one
103
+ * scans `#contexts`. Kept sorted also when the Harness clock goes back (see `#markIdle`).
104
+ */
105
+ #idleOrder = new Map();
106
+ /** At least the newest `idleSince` in `#idleOrder` (drops do not lower it): an older one added means the clock went back. */
107
+ #idleNewest = Number.NEGATIVE_INFINITY;
108
+ /**
109
+ * A chain load resolved what `#idle()` treats as unknown for a live task whose chain is not loaded, which can make a
110
+ * scope idle without a task change below it; the next idle check then covers every scope.
111
+ */
112
+ #unknownsChanged = false;
113
+ /** Idle checks that covered every scope (`idleState`, a test diagnostic). */
114
+ #fullIdleChecks = 0;
83
115
  /** Background timer for the earliest expiry of an idle context, unless the host declined it (see `#scheduleExpiry`). */
84
116
  #expiry;
85
117
  #invocations = new Map();
@@ -107,9 +139,15 @@ export class TaskScheduler {
107
139
  #unloaded = new Set();
108
140
  /** Live tasks with cancellation intent, the sources of abort cascades (spec §5.4). */
109
141
  #intent = new Set();
110
- /** Live tasks that are not `completing`, the candidates of a reservation pass; ones found waiting leave it. */
142
+ /**
143
+ * Live tasks that are not `completing`, the candidates of a reservation pass; ones found waiting leave it. A pass
144
+ * takes the set while it plans, so a task made a candidate meanwhile is not dropped by its stale decision.
145
+ */
111
146
  #runnable = new Set();
112
- /** `completing` tasks the next finalize pass checks: those that started holding, and holders of ended work. */
147
+ /**
148
+ * `completing` tasks the next finalize pass checks: those that started holding, and holders of ended work. A pass
149
+ * takes them before it plans, so a check asked for meanwhile stays for the next pass.
150
+ */
113
151
  #finalizeChecks = new Set();
114
152
  /** Live waiting tasks by each task in their `on`. */
115
153
  #waiters = new Map();
@@ -133,15 +171,21 @@ export class TaskScheduler {
133
171
  #blockedListeners = new Set();
134
172
  /** A registry snapshot that threw for the task graph was reported; it is read per record, so only once. */
135
173
  #snapshotReported = false;
136
- /** Tasks a pass left out as parked; each pass checks them again, as `settings.restartLimit` may have risen. */
137
- #parkedTasks = new Set();
138
174
  #reconcileScheduled = false;
139
175
  /** Reconcile passes underway with work (they may overlap): their commits finalize held outcomes and mark cascades. */
140
176
  #reconcilingWork = 0;
141
177
  #sweepScheduled = false;
178
+ /**
179
+ * Nodes chain loads installed for jobs whose commits are not published yet, with how many such jobs: the sweep keeps
180
+ * them, as work a pending commit creates below them needs them.
181
+ */
182
+ #pins = new Map();
183
+ /** Keys the decision region underway reads (`#region()`); set only while one runs. */
184
+ #reading;
142
185
  #cascadePending = false;
143
186
  #unsubscribeRegistry = () => { };
144
187
  #enabled = false;
188
+ #readOnly;
145
189
  #closing = false;
146
190
  #dirty = false;
147
191
  #draining = false;
@@ -163,24 +207,26 @@ export class TaskScheduler {
163
207
  this.#writeAtClose = options.writeAtClose;
164
208
  this.#report = options.report;
165
209
  this.#settleOutcome = options.settleOutcome;
210
+ this.#limitsRestarts = options.limitsRestarts;
166
211
  this.#withdrawInputs = options.withdrawInputs;
167
212
  this.#conversation = options.conversation;
168
213
  this.#context = options.context;
214
+ this.#readOnly = options.readOnly === true;
169
215
  }
170
216
  /**
171
- * Load live tasks and change surviving `running` tasks back to `pending`, counting the restart. Dispatches nothing. A
172
- * task that reaches the restart limit this way is parked and reported.
217
+ * Load live tasks and change surviving `running` tasks back to `pending`, counting the restart. Dispatches nothing;
218
+ * a task cut off `settings.restartLimit` times ends when it is next reserved. Read-only, it only loads them: a
219
+ * `running` task stays as committed, running in another process.
173
220
  */
174
221
  async open(context) {
175
- this.#session.observeCommits((publication) => this.#observe(publication));
222
+ this.#session.followCommits((publication) => this.#observe(publication));
176
223
  this.#session.subscribeClose(() => this.#seal());
177
224
  this.#unsubscribeRegistry = this.#registry.subscribe(() => {
178
225
  this.#notifyBlocked();
179
226
  this.#kick();
180
227
  });
181
- const limit = this.#restartLimit();
182
- const parked = [];
183
228
  await this.#session.commitWith(async (tx) => {
229
+ await this.#published();
184
230
  // Every table read before the first write.
185
231
  const scans = [];
186
232
  for (const status of LIVE_STATUSES) {
@@ -190,14 +236,14 @@ export class TaskScheduler {
190
236
  for (const record of records) {
191
237
  this.#track(undefined, record);
192
238
  this.#resumed.add(record.id);
239
+ if (this.#readOnly)
240
+ continue;
193
241
  if (record.abandonOnRestart === true)
194
242
  this.#abandoned.add(record.id);
195
243
  if (record.state.status === "running") {
196
244
  const restarts = (record.restarts ?? 0) + 1;
197
245
  const pending = withState(record, { status: "pending", checkpoint: record.state.checkpoint });
198
246
  tx.setTask({ ...pending, restarts });
199
- if (restarts >= limit && !record.abortRequested)
200
- parked.push(record);
201
247
  }
202
248
  if (record.state.status === "waiting" && record.state.policy === "failFast") {
203
249
  this.#failFastChecks.add(record.id);
@@ -205,11 +251,8 @@ export class TaskScheduler {
205
251
  }
206
252
  }
207
253
  }, context);
208
- for (const record of parked) {
209
- const restarts = (record.restarts ?? 0) + 1;
210
- const error = new Error(`Task ${record.id} (${record.kind}) was cut off ${restarts} times without progress; it is parked until retryTask()`);
211
- this.#report(error, { type: "task_parked", ...this.#aboutTask(recordTask(record)) });
212
- }
254
+ if (this.#readOnly)
255
+ return;
213
256
  // Derive abort marks a crash left unapplied below cancelled owners, and finalize held outcomes.
214
257
  this.#cascadePending = true;
215
258
  this.#scheduleReconcile();
@@ -220,29 +263,15 @@ export class TaskScheduler {
220
263
  * Idempotent; the kick does nothing once closing.
221
264
  */
222
265
  resume() {
266
+ if (this.#readOnly)
267
+ throw new ReadOnlySession("running tasks");
223
268
  this.#enabled = true;
224
269
  this.#wakeDue();
225
270
  this.#kick();
226
271
  }
227
272
  /**
228
- * Run a task parked by the restart limit again: clears its restart count and enables scheduling. Does nothing for a
229
- * task that is not parked; rejects for an unknown one.
230
- */
231
- async retry(id, context) {
232
- await this.#session.commitWith(async (tx) => {
233
- const current = await tx.task(id);
234
- if (current === undefined)
235
- throw new Error(`Task ${id} does not exist`);
236
- const status = current.state.status;
237
- if (status === "terminal" || status === "completing" || !this.#parked(current))
238
- return;
239
- tx.setTask(progressed(current));
240
- }, context);
241
- this.resume();
242
- }
243
- /**
244
- * Why a live task cannot run here, if it cannot: its definition (missing, too old, failed migration) or the restart
245
- * limit. A running task, an abort-marked one, and one holding its outcome are never blocked.
273
+ * Why a live task cannot run here, if it cannot: its definition is missing, too old, or failed to migrate. A running
274
+ * task, an abort-marked one, and one holding its outcome are never blocked.
246
275
  */
247
276
  blockedReason(record) {
248
277
  const status = record.status;
@@ -251,8 +280,6 @@ export class TaskScheduler {
251
280
  // An abort request runs its handler or orphans the task; a `restart` mark waits for the definition, blocked.
252
281
  if (record.abortRequested && record.abortReason === undefined)
253
282
  return undefined;
254
- if (this.#parked(record))
255
- return "restart_limit";
256
283
  let snapshot;
257
284
  try {
258
285
  snapshot = this.#registry.snapshot();
@@ -278,11 +305,26 @@ export class TaskScheduler {
278
305
  * sleep not yet due. Synchronous, from memory; a task whose chain is not loaded yet counts as it would alone.
279
306
  */
280
307
  wake() {
308
+ // A read-only Harness runs nothing, so it never needs a process.
309
+ if (this.#readOnly)
310
+ return { active: false };
281
311
  const now = this.#now();
282
312
  // A reconcile pass with work to do is work too: it finalizes held outcomes and marks cascades, at open as
283
313
  // recovery. A process that dies before it lands must be woken again. A pass with nothing to do is not.
284
- let active = this.#invocations.size > 0 || this.#reconcilingWork > 0 || this.#reconcileWork();
314
+ const active = this.#invocations.size > 0 || this.#reconcilingWork > 0 || this.#reconcileWork() || this.#runsNow(now);
315
+ // The earliest sleep not yet due: only `#sleeping` holds sleeping tasks, so no pass visits every live task.
285
316
  let at;
317
+ for (const id of this.#sleeping) {
318
+ const record = this.#live.get(id);
319
+ const state = record?.state;
320
+ if (state?.status === "sleeping" && !record.abortRequested && state.until > now) {
321
+ at = at === undefined ? state.until : Math.min(at, state.until);
322
+ }
323
+ }
324
+ return at === undefined ? { active } : { active, at };
325
+ }
326
+ /** Whether a live task can run now: the scan `wake()` needs only while no invocation runs. */
327
+ #runsNow(now) {
286
328
  let snapshot;
287
329
  let snapshotFailed = false;
288
330
  for (const record of this.#live.values()) {
@@ -291,13 +333,7 @@ export class TaskScheduler {
291
333
  continue;
292
334
  if (state.status === "waiting" && !record.abortRequested && state.on.some((id) => this.#live.has(id)))
293
335
  continue;
294
- if (state.status === "sleeping" && !record.abortRequested && state.until > now) {
295
- at = at === undefined ? state.until : Math.min(at, state.until);
296
- continue;
297
- }
298
- if (active)
299
- continue;
300
- if (this.#parked(record))
336
+ if (state.status === "sleeping" && !record.abortRequested && state.until > now)
301
337
  continue;
302
338
  // An abort invocation waits for the live work below it, which counts itself.
303
339
  if (record.abortRequested && this.#waitingOn(record, true).length > 0)
@@ -316,9 +352,9 @@ export class TaskScheduler {
316
352
  if (snapshot !== undefined && "reason" in this.#fit(record, snapshot.task(record.kind)))
317
353
  continue;
318
354
  }
319
- active = true;
355
+ return true;
320
356
  }
321
- return at === undefined ? { active } : { active, at };
357
+ return false;
322
358
  }
323
359
  /**
324
360
  * Signal every invocation and wait for them, then write the tasks they ran back as `pending` (`writeAtClose`), as a
@@ -363,12 +399,14 @@ export class TaskScheduler {
363
399
  return { result: "marked" };
364
400
  if (invocation === undefined && current.state.status !== "completing") {
365
401
  await this.#loadScopes(false);
366
- if (!this.#hasOwnedLive(id)) {
367
- const resolution = this.#resolve(current, this.#registry.snapshot());
368
- if (resolution.kind === "blocked") {
369
- await this.#terminate(tx, current, { status: "orphaned", reason: resolution.reason });
370
- return { result: "marked" };
371
- }
402
+ const owns = await this.#region(() => this.#hasOwnedLive(id), this.#reloadScopes);
403
+ // Resolved once that decision stands, as it may run `migrate`.
404
+ const resolution = owns
405
+ ? undefined
406
+ : this.#resolve(current, this.#registry.snapshot());
407
+ if (resolution?.kind === "blocked") {
408
+ await this.#terminate(tx, current, { status: "orphaned", reason: resolution.reason });
409
+ return { result: "marked" };
372
410
  }
373
411
  }
374
412
  if (!current.abortRequested || current.abortReason !== undefined)
@@ -384,12 +422,13 @@ export class TaskScheduler {
384
422
  }
385
423
  async waitForTask(id, context) {
386
424
  // Check and register on the line so no terminal publication falls between them.
387
- const found = await this.#session.readOnLine(async () => {
425
+ const found = await this.#session.readOnLine(async (storage) => {
388
426
  if (this.#closing)
389
427
  throw closedError(this.#session);
390
- if (this.#live.has(id))
428
+ if (await this.#region(() => this.#liveRecord(id) !== undefined)) {
391
429
  return { promise: this.#taskWaiters.add(id, context) };
392
- const record = await this.#storage.task(id, context);
430
+ }
431
+ const record = await storage.task(id, context);
393
432
  if (record === undefined)
394
433
  throw new Error(`Task ${id} does not exist`);
395
434
  return { promise: Promise.resolve(record) };
@@ -415,6 +454,7 @@ export class TaskScheduler {
415
454
  */
416
455
  async abortConversation(conversationId, background, context) {
417
456
  const reached = await this.#session.commitWith(async (tx) => {
457
+ await this.#published();
418
458
  const queued = await this.#loadScopes(true);
419
459
  const scope = { conversation: conversationId };
420
460
  const reached = [];
@@ -437,11 +477,100 @@ export class TaskScheduler {
437
477
  await this.waitForTask(id, context);
438
478
  await this.waitForIdle(conversationId, context);
439
479
  }
480
+ /**
481
+ * The ownership memberships a commit's task and conversation changes write (group commit §4.2), at its admission on
482
+ * the line: for each task created or made terminal, every node from its parent up to and including the nearest live
483
+ * task, or the root conversation, and then the roots too, conservatively, as a pending end may unlist the root a
484
+ * creation finds listed; for each conversation created under a task, that task; and the intent set when
485
+ * a change sets, upgrades, or clears a mark or a held or ended outcome other than `completed`. Loads the chains it
486
+ * walks; the nodes it installs stay until the commit is published.
487
+ */
488
+ membershipKeys(tx) {
489
+ const tasks = tx.stagedTasks();
490
+ const conversations = tx.stagedConversations();
491
+ if (tasks.length === 0 && conversations.every((record) => record.owner === undefined))
492
+ return NO_KEYS;
493
+ return this.#membershipKeys(tx, tasks, conversations);
494
+ }
495
+ async #membershipKeys(tx, tasks, conversations) {
496
+ const keys = new Set();
497
+ const overlay = overlayOf(tx);
498
+ const created = new Set(tx.createdTaskIds());
499
+ const reached = new Map();
500
+ for (const record of tasks) {
501
+ if (intentChanges(this.#live.get(record.id), record))
502
+ keys.add(INTENT_KEY);
503
+ if (!created.has(record.id) && record.state.status !== "terminal")
504
+ continue;
505
+ await this.#loadChain(parentOf(record), overlay);
506
+ if ((await this.#chainKeys(parentNode(record), overlay, keys, reached)) !== undefined)
507
+ keys.add(ROOTS_KEY);
508
+ }
509
+ for (const conversation of conversations) {
510
+ const owner = conversation.owner?.taskId;
511
+ if (owner === undefined)
512
+ continue;
513
+ await this.#loadChain({ task: owner }, overlay);
514
+ await this.#chainKeys(owner, overlay, keys, reached);
515
+ }
516
+ return [...keys];
517
+ }
518
+ /**
519
+ * Add the memberships from `start` up to and including the nearest live task, or the root conversation, which it
520
+ * returns. `reached` keeps, per node walked, where its walk ended, so a chain several tasks of a commit share is
521
+ * walked once. Whether an ancestor is live can change under a pending commit, so it is read after a touch of the
522
+ * ancestor's record: a pending end must not stop the chain early. Ownership fields are immutable, so not keyed.
523
+ */
524
+ async #chainKeys(start, overlay, keys, reached) {
525
+ const walked = [];
526
+ let root;
527
+ let node = start;
528
+ while (node !== undefined && node !== null) {
529
+ const key = ownedKey(node);
530
+ // Walked up from here already, for another task of the commit.
531
+ if (reached.has(key)) {
532
+ root = reached.get(key);
533
+ break;
534
+ }
535
+ keys.add(key);
536
+ walked.push(key);
537
+ if (typeof node === "string") {
538
+ const conversation = conversationOf(node);
539
+ node = this.#edge(conversation, overlay);
540
+ if (node === null)
541
+ root = conversation;
542
+ continue;
543
+ }
544
+ let record = overlay.tasks.get(node);
545
+ if (record === undefined) {
546
+ const gate = this.#session.touch(taskKey(node));
547
+ if (gate !== undefined) {
548
+ await gate;
549
+ // It may have ended meanwhile, and its node been swept: load the chain above it again.
550
+ await this.#loadChain({ task: node }, overlay);
551
+ }
552
+ record = this.#live.get(node);
553
+ }
554
+ if (record !== undefined && record.state.status !== "terminal")
555
+ break;
556
+ const above = record ?? this.#settled.get(node);
557
+ node = above === undefined ? undefined : parentNode(above);
558
+ }
559
+ for (const key of walked)
560
+ reached.set(key, root);
561
+ return root;
562
+ }
440
563
  // ─── Scheduling ────────────────────────────────────────────────────────
441
564
  #observe(publication) {
565
+ // A commit held in memory changes documents only.
566
+ if (publication.kind === "memory")
567
+ return;
568
+ this.#advanceContexts(publication);
442
569
  const updated = [];
443
570
  const failed = [];
444
571
  let changed = false;
572
+ // The conversations whose idleness the commit's task changes can change; undefined when a chain is not known.
573
+ let dirty = new Set();
445
574
  // Edges first, so the tasks a commit creates in a new conversation hang from it.
446
575
  for (const change of publication.changes) {
447
576
  if (change.type === "conversation" && !this.#edges.has(change.value.id)) {
@@ -457,6 +586,11 @@ export class TaskScheduler {
457
586
  if (failedOutcome(record) && (previous === undefined || !failedOutcome(previous)))
458
587
  failed.push(record.id);
459
588
  this.#track(previous, record);
589
+ // Idleness counts live tasks, so only a task that became live or ended can change it. Collected before `#sweep()`,
590
+ // which may forget the chain of a task that ended.
591
+ const liveness = previous === undefined || record.state.status === "terminal";
592
+ if (liveness && dirty !== undefined && !this.#addScopesAbove(record, dirty))
593
+ dirty = undefined;
460
594
  if (record.state.status === "terminal") {
461
595
  this.#failedMigrations.delete(record.id);
462
596
  this.#failFastChecks.delete(record.id);
@@ -467,10 +601,11 @@ export class TaskScheduler {
467
601
  }
468
602
  if (record.abortRequested && previous?.abortRequested !== true) {
469
603
  this.#cascadePending = true;
470
- // Signal a run invocation of the newly marked task; its next step ends it.
604
+ // Signal a run invocation of the newly marked task; its next step ends it. Abort listeners are task code, which
605
+ // never runs inside a publication: the signal follows in a microtask, after host commit listeners.
471
606
  const invocation = this.#invocations.get(record.id);
472
607
  if (invocation?.mode === "run")
473
- invocation.controller.abort();
608
+ queueMicrotask(() => invocation.controller.abort());
474
609
  }
475
610
  // A request replacing a `restart` mark upgrades the marks of the work below.
476
611
  if (previous?.abortReason !== undefined && record.abortReason === undefined)
@@ -524,9 +659,60 @@ export class TaskScheduler {
524
659
  this.#sweep();
525
660
  if (!changed)
526
661
  return;
527
- this.#settleIdle();
662
+ this.#settleIdle(dirty);
528
663
  this.#kick();
529
664
  }
665
+ /**
666
+ * Add the conversations above `record` to `scopes`: those whose idleness a change of it can change. False when a node
667
+ * on the way up is not known. A conversation already in `scopes` has everything above it there too.
668
+ */
669
+ #addScopesAbove(record, scopes) {
670
+ for (const step of this.#above(parentOf(record))) {
671
+ if ("unknown" in step)
672
+ return false;
673
+ if (!("conversation" in step))
674
+ continue;
675
+ if (scopes.has(step.conversation))
676
+ return true;
677
+ scopes.add(step.conversation);
678
+ }
679
+ return true;
680
+ }
681
+ /**
682
+ * Extend each kept context range with the entries a stored commit appended to its conversation, so a later read
683
+ * needs no Storage scan. Entries are published in ID order.
684
+ */
685
+ #advanceContexts(publication) {
686
+ if (this.#contexts.size === 0)
687
+ return;
688
+ let appended;
689
+ for (const change of publication.changes) {
690
+ if (change.type !== "entry" || !this.#contexts.has(change.value.conversationId))
691
+ continue;
692
+ appended ??= new Map();
693
+ const entries = appended.get(change.value.conversationId);
694
+ if (entries === undefined)
695
+ appended.set(change.value.conversationId, [change.value]);
696
+ else
697
+ entries.push(change.value);
698
+ }
699
+ if (appended === undefined)
700
+ return;
701
+ for (const [conversationId, entries] of appended) {
702
+ const kept = this.#contexts.get(conversationId);
703
+ // An idle range past its retention is dropped here too: entry-only commits do not run `#settleIdle()`, and a
704
+ // host may decline the expiry timer. `expiresAt` was fixed when it went idle, so no settings are read here.
705
+ if (kept.idleSince !== undefined && kept.expiresAt !== undefined && kept.expiresAt <= this.#now()) {
706
+ this.#dropContext(conversationId);
707
+ continue;
708
+ }
709
+ const range = advanceRange(kept.range, entries);
710
+ if (range === undefined)
711
+ this.#dropContext(conversationId);
712
+ else
713
+ kept.range = range;
714
+ }
715
+ }
530
716
  /**
531
717
  * `settings.contextRetentionMs`, or 0 when the host's settings throw. Kept contexts are only a cache, so dropping them
532
718
  * is safe, while a throw here would escape commit listeners, reconciliation, and the expiry timer.
@@ -540,43 +726,94 @@ export class TaskScheduler {
540
726
  return 0;
541
727
  }
542
728
  }
543
- /** Resolve idle waiters, and drop each kept context whose conversation has been idle for the retention period. */
544
- #settleIdle() {
545
- for (const conversationId of this.#idleWaiters.keys()) {
546
- if (this.#idle(conversationId))
547
- this.#idleWaiters.resolve(conversationId);
548
- }
729
+ /**
730
+ * After task changes below the conversations in `scopes` (every conversation when undefined): resolve their idle
731
+ * waiters and the Harness-wide ones, drop the kept contexts whose idle period passed the retention, then start or end
732
+ * the idle periods of the kept contexts in `scopes`. A task change can change the idleness only of the conversations
733
+ * above it, so no check visits every conversation.
734
+ */
735
+ #settleIdle(scopes) {
736
+ const all = scopes === undefined || this.#unknownsChanged;
737
+ this.#unknownsChanged = false;
738
+ if (all)
739
+ this.#fullIdleChecks++;
740
+ this.#resolveIdleWaiters(all ? undefined : scopes);
549
741
  const now = this.#now();
550
742
  const retention = this.#contextRetentionMs();
551
- for (const [conversationId, kept] of this.#contexts) {
552
- if (kept.idleSince !== undefined && now - kept.idleSince >= retention) {
553
- this.#contexts.delete(conversationId);
554
- }
555
- else if (!this.#idle(conversationId)) {
743
+ // Expired first: a context past its retention is dropped even when the same commit makes it busy again.
744
+ this.#expireContexts(now, retention);
745
+ for (const conversationId of all ? [...this.#contexts.keys()] : scopes) {
746
+ const kept = this.#contexts.get(conversationId);
747
+ if (kept === undefined)
748
+ continue;
749
+ if (!this.#idle(conversationId)) {
750
+ if (kept.idleSince === undefined)
751
+ continue;
556
752
  kept.idleSince = undefined;
753
+ this.#idleOrder.delete(conversationId);
557
754
  }
558
- else if (retention > 0) {
559
- kept.idleSince ??= now;
755
+ else if (retention <= 0) {
756
+ this.#dropContext(conversationId);
560
757
  }
561
- else {
562
- this.#contexts.delete(conversationId);
758
+ else if (kept.idleSince === undefined) {
759
+ kept.idleSince = now;
760
+ kept.expiresAt = now + retention;
761
+ this.#markIdle(conversationId, now);
563
762
  }
564
763
  }
565
764
  this.#scheduleExpiry();
566
765
  }
766
+ /** Resolve the idle waiters of the conversations in `scopes` (all when undefined) and the Harness-wide ones. */
767
+ #resolveIdleWaiters(scopes) {
768
+ for (const conversationId of this.#idleWaiters.keys()) {
769
+ if (conversationId !== undefined && scopes !== undefined && !scopes.has(conversationId))
770
+ continue;
771
+ if (this.#idle(conversationId))
772
+ this.#idleWaiters.resolve(conversationId);
773
+ }
774
+ }
567
775
  /**
568
- * Run `#settleIdle()` when the earliest idle context expires, on a background timer, which never keeps the process
569
- * alive. Where the host declines it, as the default does in Cloudflare Workers, none is kept: a pending timer could
570
- * keep a Durable Object from being evicted, and eviction frees the contexts. There, task changes alone check expiry.
776
+ * Drop the kept contexts idle for at least `retention`, oldest first, with the retention setting of now; with no
777
+ * retention, every idle one, also those whose idle period a clock that went back makes start after now.
571
778
  */
572
- #scheduleExpiry() {
573
- const retention = this.#contextRetentionMs();
574
- let at;
575
- for (const kept of this.#contexts.values()) {
576
- if (kept.idleSince !== undefined && (at === undefined || kept.idleSince + retention < at)) {
577
- at = kept.idleSince + retention;
578
- }
779
+ #expireContexts(now, retention) {
780
+ for (const [conversationId, idleSince] of this.#idleOrder) {
781
+ if (retention > 0 && now - idleSince < retention)
782
+ return;
783
+ this.#dropContext(conversationId);
784
+ }
785
+ }
786
+ /**
787
+ * Start the idle period of a kept context in the expiry order. An idle period older than the newest one kept, which
788
+ * a clock that went back starts, sorts the order again, and the newest is taken from what is kept then.
789
+ */
790
+ #markIdle(conversationId, idleSince) {
791
+ this.#idleOrder.delete(conversationId);
792
+ this.#idleOrder.set(conversationId, idleSince);
793
+ if (this.#idleOrder.size === 1 || idleSince >= this.#idleNewest) {
794
+ this.#idleNewest = idleSince;
795
+ return;
579
796
  }
797
+ const sorted = [...this.#idleOrder].sort((a, b) => a[1] - b[1]);
798
+ this.#idleOrder.clear();
799
+ for (const [id, since] of sorted)
800
+ this.#idleOrder.set(id, since);
801
+ this.#idleNewest = sorted.at(-1)[1];
802
+ }
803
+ /** Forget a kept context, idle or not, and its place in the expiry order. */
804
+ #dropContext(conversationId) {
805
+ this.#contexts.delete(conversationId);
806
+ this.#idleOrder.delete(conversationId);
807
+ }
808
+ /**
809
+ * Drop the expired contexts when the earliest idle context expires, on a background timer, which never keeps the
810
+ * process alive. Where the host declines it, as the default does in Cloudflare Workers, none is kept: a pending timer
811
+ * could keep a Durable Object from being evicted, and eviction frees the contexts. There, task changes alone check
812
+ * expiry.
813
+ */
814
+ #scheduleExpiry() {
815
+ const oldest = this.#idleOrder.values().next();
816
+ const at = oldest.done === true ? undefined : oldest.value + this.#contextRetentionMs();
580
817
  if (this.#expiry !== undefined && this.#expiry.at === at)
581
818
  return;
582
819
  this.#expiry?.disarm();
@@ -585,14 +822,16 @@ export class TaskScheduler {
585
822
  return;
586
823
  const disarm = this.#timers.background(() => {
587
824
  this.#expiry = undefined;
588
- this.#settleIdle();
825
+ this.#expireContexts(this.#now(), this.#contextRetentionMs());
826
+ this.#scheduleExpiry();
589
827
  }, at - this.#now());
590
828
  if (disarm !== undefined)
591
829
  this.#expiry = { at, disarm };
592
830
  }
593
831
  // ─── Ownership ───────────────────────────────────────────────────────────
594
832
  #scheduleReconcile() {
595
- if (this.#reconcileScheduled || this.#closing)
833
+ // A read-only reconcile could only fail: what it writes is refused, and its own commits fail the Session.
834
+ if (this.#reconcileScheduled || this.#closing || this.#readOnly)
596
835
  return;
597
836
  this.#reconcileScheduled = true;
598
837
  queueMicrotask(() => void this.#reconcile());
@@ -608,54 +847,22 @@ export class TaskScheduler {
608
847
  const work = this.#reconcileWork();
609
848
  if (work)
610
849
  this.#reconcilingWork++;
611
- const cascade = this.#cascadePending;
612
- this.#cascadePending = false;
613
- const checks = [...this.#failFastChecks];
614
- this.#failFastChecks.clear();
615
850
  try {
616
- await this.#session.commitWith(async (tx) => {
851
+ await this.#session.commitInternal(async (tx) => {
617
852
  if (this.#closing)
618
853
  return;
854
+ // A cascade scans the queued inputs, which waits for every unpublished commit: a request made before the scan
855
+ // is covered by it, and none is published while the job holds the line after it.
856
+ const cascade = this.#cascadePending;
619
857
  const queued = await this.#loadScopes(cascade);
620
- // A cascade from an abandoned owner passes its `restart` reason on; any other intent marks, or upgrades a
621
- // `restart` mark, as a request. Collected first, so a request wins over a `restart` mark of the same pass.
622
- const marks = new Map();
623
- const mark = (record, reason) => {
624
- const staged = marks.get(record.id);
625
- if (staged !== undefined && staged.reason === undefined)
626
- return;
627
- if (record.abortRequested && (record.abortReason === undefined || reason !== undefined))
628
- return;
629
- marks.set(record.id, { record, reason });
630
- };
631
- // Every live foreground task below a live owner with cancellation intent, which is its nearest one: a walk down
632
- // from each owner with intent does not enter another, whose own walk covers what is below it. Derived again on
633
- // every pass, as loading edges can reveal a cancelled owner.
634
- for (const id of this.#intent) {
635
- const owner = this.#live.get(id);
636
- // Only an owner abandoned after a restart, and nothing else, passes its reason on.
637
- const restart = owner.abortReason === "restart" && !failedOutcome(owner);
638
- for (const { node, live } of this.#walkDown(id, { enter: (below) => !this.#intent.has(below) })) {
639
- if (live !== undefined && !node.background)
640
- mark(live, restart ? "restart" : undefined);
641
- }
642
- }
643
- for (const id of checks) {
644
- const waiter = this.#live.get(id);
645
- if (waiter?.state.status !== "waiting" || !(await this.#anyFailed(waiter.state.on)))
646
- continue;
647
- // Every other live task: the failed one keeps its own outcome.
648
- for (const member of waiter.state.on) {
649
- const record = this.#live.get(member);
650
- if (record !== undefined && !failedOutcome(record))
651
- mark(record, undefined);
652
- }
653
- }
858
+ if (cascade)
859
+ this.#cascadePending = false;
860
+ const marks = await this.#marks();
654
861
  for (const { record, reason } of marks.values())
655
862
  tx.setTask(withAbortMark(record, reason));
656
- for (const id of queued)
657
- if (this.#belowCancelled({ conversation: id }))
658
- await this.#withdrawInputs(tx, id);
863
+ const withdrawn = await this.#region(() => queued.filter((id) => this.#belowCancelled({ conversation: id })), this.#reloadScopes);
864
+ for (const id of withdrawn)
865
+ await this.#withdrawInputs(tx, id);
659
866
  await this.#finalize(tx);
660
867
  }, this.#context);
661
868
  }
@@ -667,63 +874,168 @@ export class TaskScheduler {
667
874
  if (work)
668
875
  this.#reconcilingWork--;
669
876
  this.#scheduleWake();
670
- this.#settleIdle();
877
+ // Its commit's task changes were settled at their publication; a waiter that asked for this pass checks again, and
878
+ // what its chain loads resolved can make any scope idle.
879
+ if (this.#unknownsChanged) {
880
+ this.#settleIdle(undefined);
881
+ return;
882
+ }
883
+ this.#resolveIdleWaiters(undefined);
884
+ // A pass checks expiry too, for hosts that decline the expiry timer.
885
+ this.#expireContexts(this.#now(), this.#contextRetentionMs());
886
+ this.#scheduleExpiry();
887
+ }
888
+ /**
889
+ * The abort marks committed records imply (spec §5.4, §5.5): every live foreground task below a live owner with
890
+ * cancellation intent, which is its nearest one (a walk down from each owner with intent does not enter another,
891
+ * whose own walk covers what is below it), and every other live task in the `on` of a `failFast` waiter one of whose
892
+ * tasks failed. A cascade from an abandoned owner passes its `restart` reason on; any other intent marks, or upgrades
893
+ * a `restart` mark, as a request. Collected first, so a request wins over a `restart` mark of the same pass. Derived
894
+ * again on every pass, as loading edges can reveal a cancelled owner. Takes the `failFast` checks once its decision
895
+ * stands.
896
+ */
897
+ async #marks() {
898
+ // Terminal tasks in a checked `on`, read from Storage: each one's outcome is final.
899
+ const ended = new Map();
900
+ for (;;) {
901
+ const plan = await this.#region(() => this.#planMarks(ended), this.#reloadScopes);
902
+ if ("missing" in plan) {
903
+ for (const id of plan.missing)
904
+ ended.set(id, (await this.#session.lineStorage.task(id, this.#context)) ?? null);
905
+ continue;
906
+ }
907
+ for (const id of plan.checked)
908
+ this.#failFastChecks.delete(id);
909
+ return plan.marks;
910
+ }
911
+ }
912
+ /** `#marks()`' decision region; the `failFast` members it needs from Storage first, when `ended` lacks them. */
913
+ #planMarks(ended) {
914
+ const marks = new Map();
915
+ // A mark rewrites the whole record, so the record it copies is read as a task record, keyed: a commit of the task
916
+ // that is not published yet makes the region wait and run again, rather than stage a mark over a stale checkpoint.
917
+ const mark = (found, reason) => {
918
+ const record = this.#liveRecord(found.id);
919
+ const staged = marks.get(record.id);
920
+ if (staged !== undefined && staged.reason === undefined)
921
+ return;
922
+ if (record.abortRequested && (record.abortReason === undefined || reason !== undefined))
923
+ return;
924
+ marks.set(record.id, { record, reason });
925
+ };
926
+ this.#record(INTENT_KEY);
927
+ for (const id of this.#intent) {
928
+ const owner = this.#liveRecord(id);
929
+ // Only an owner abandoned after a restart, and nothing else, passes its reason on.
930
+ const restart = owner.abortReason === "restart" && !failedOutcome(owner);
931
+ for (const { node, live } of this.#walkDown(id, { enter: (below) => !this.#intent.has(below) })) {
932
+ if (live !== undefined && !node.background)
933
+ mark(live, restart ? "restart" : undefined);
934
+ }
935
+ }
936
+ const checked = [...this.#failFastChecks];
937
+ const missing = [];
938
+ for (const id of checked) {
939
+ const waiter = this.#liveRecord(id);
940
+ if (waiter?.state.status !== "waiting")
941
+ continue;
942
+ let failed = false;
943
+ for (const member of waiter.state.on) {
944
+ const record = this.#liveRecord(member) ?? ended.get(member);
945
+ if (record === undefined)
946
+ missing.push(member);
947
+ else if (record !== null && failedOutcome(record))
948
+ failed = true;
949
+ }
950
+ if (!failed)
951
+ continue;
952
+ // Every other live task: the failed one keeps its own outcome.
953
+ for (const member of waiter.state.on) {
954
+ const record = this.#liveRecord(member);
955
+ if (record !== undefined && !failedOutcome(record))
956
+ mark(record, undefined);
957
+ }
958
+ }
959
+ return missing.length > 0 ? { missing } : { marks, checked };
671
960
  }
672
961
  /** Whether a reconcile pass would have work: a cascade to derive, held outcomes to check, or `failFast` waits. */
673
962
  #reconcileWork() {
674
963
  return this.#cascadePending || this.#finalizeChecks.size > 0 || this.#failFastChecks.size > 0;
675
964
  }
676
- /** Whether any of `ids` holds or ended with an outcome other than `completed`. */
677
- async #anyFailed(ids) {
678
- for (const id of ids) {
679
- const record = this.#live.get(id) ?? (await this.#storage.task(id, this.#context));
680
- if (record !== undefined && failedOutcome(record))
681
- return true;
682
- }
683
- return false;
684
- }
685
965
  /**
686
966
  * Write the terminal record of every `completing` task without live ordinary owned work. Finalizing one can free its
687
967
  * owner, so this repeats over the commit's candidates until nothing changes. A held scheduler outcome gets its
688
- * Harness cleanup here.
968
+ * Harness cleanup here. In rounds, one decision region each: the holders the tasks of one round free are checked once,
969
+ * in the next, so a holder of many tasks that end together is walked once, not once per task. The first round takes
970
+ * the `completing` tasks to check once its decision stands.
689
971
  */
690
972
  async #finalize(tx) {
691
- // A worklist: the tasks that started holding, and the holders of work that ended. Finalizing one can free the
692
- // nearest live task above it, which goes on the list. Tasks finalized here have no live work below them, so walks
693
- // down skip them.
694
- // In rounds: the holders the tasks of one round free are checked once, in the next, so a holder of many tasks that
695
- // end together is walked once, not once per task.
696
- let round = [...this.#finalizeChecks];
697
- this.#finalizeChecks.clear();
698
973
  const done = new Set();
974
+ // The commit's candidates, kept in step with the terminal records each round stages.
699
975
  const overlay = overlayOf(tx);
700
- while (round.length > 0) {
701
- const next = new Set();
702
- for (const id of round) {
703
- const record = overlay.tasks.get(id) ?? this.#live.get(id);
704
- if (record?.state.status !== "completing" || done.has(id))
705
- continue;
706
- if (this.#hasOwnedLive(id, overlay, done))
707
- continue;
708
- const outcome = record.state.outcome;
709
- const terminal = withState(record, { status: "terminal", outcome });
976
+ let round = [...this.#finalizeChecks];
977
+ this.#finalizeChecks.clear();
978
+ for (;;) {
979
+ const ids = round;
980
+ const plan = await this.#region(() => this.#planRound(ids, overlay, done), this.#reloadScopes);
981
+ for (const { record, terminal } of plan.finalized) {
710
982
  tx.setTask(terminal);
711
- overlay.tasks.set(id, terminal);
712
- done.add(id);
983
+ overlay.tasks.set(record.id, terminal);
984
+ done.add(record.id);
985
+ const outcome = record.state.outcome;
713
986
  // REMINDER: only the scheduler writes `faulted` and `orphaned` (spec §5.4); their cleanup waits for this commit.
714
987
  if (outcome.status === "faulted" || outcome.status === "orphaned") {
715
988
  await this.#settleOutcome(tx, record, outcome);
716
989
  }
990
+ }
991
+ if (plan.next.length === 0)
992
+ return;
993
+ round = plan.next;
994
+ }
995
+ }
996
+ /**
997
+ * One finalize round's decision: which of `ids` are `completing` without live ordinary owned work below them, in the
998
+ * commit's candidates with the round's own finalizations applied in order, and the holders above them to check next.
999
+ * Tasks finalized have no live work below them, so walks down skip them. It applies the round's terminal records to
1000
+ * `overlay` while it decides and takes them back before it returns, so a run again starts from the same candidates.
1001
+ */
1002
+ #planRound(ids, overlay, finalized) {
1003
+ const tasks = overlay.tasks;
1004
+ // The tasks finalized so far, earlier rounds' and this one's, without copying the former.
1005
+ const round = new Set();
1006
+ const done = { has: (id) => round.has(id) || finalized.has(id) };
1007
+ const ended = [];
1008
+ const next = new Set();
1009
+ try {
1010
+ for (const id of ids) {
1011
+ const staged = tasks.get(id);
1012
+ const record = staged ?? this.#liveRecord(id);
1013
+ if (record?.state.status !== "completing" || done.has(id))
1014
+ continue;
1015
+ if (this.#hasOwnedLive(id, overlay, done))
1016
+ continue;
1017
+ const terminal = withState(record, { status: "terminal", outcome: record.state.outcome });
1018
+ tasks.set(id, terminal);
1019
+ round.add(id);
1020
+ ended.push({ record, terminal, staged: staged !== undefined });
717
1021
  const above = this.#nearestLiveAbove(record, overlay);
718
1022
  if (above?.state.status === "completing")
719
1023
  next.add(above.id);
720
1024
  }
721
- round = [...next];
722
1025
  }
1026
+ finally {
1027
+ for (const { record, staged } of ended) {
1028
+ if (staged)
1029
+ tasks.set(record.id, record);
1030
+ else
1031
+ tasks.delete(record.id);
1032
+ }
1033
+ }
1034
+ return { finalized: ended, next: [...next] };
723
1035
  }
724
1036
  /**
725
1037
  * Load the owner chains of the tasks in `#unloaded` and, with `queued`, of every conversation with queued inputs,
726
- * on the Session line; returns the latter. Reads committed Storage directly, so it may run inside a commit callback.
1038
+ * on the Session line; returns the latter. Reads through the line's Storage, so it may run inside a commit callback.
727
1039
  */
728
1040
  async #loadScopes(queued) {
729
1041
  for (const id of [...this.#unloaded]) {
@@ -733,7 +1045,8 @@ export class TaskScheduler {
733
1045
  }
734
1046
  if (!queued)
735
1047
  return [];
736
- const submissions = await scanAll((cursor) => this.#storage.scanSubmissions({ status: "queued" }, SCAN_PAGE_SIZE, cursor, this.#context));
1048
+ // A scan, which cannot be keyed: it waits for every unpublished commit.
1049
+ const submissions = await scanAll((cursor) => this.#session.lineStorage.scanSubmissions({ status: "queued" }, SCAN_PAGE_SIZE, cursor, this.#context));
737
1050
  const inputs = submissions.filter((submission) => submission.type === "input");
738
1051
  const conversations = [...new Set(inputs.map((submission) => submission.conversationId))];
739
1052
  for (const id of conversations)
@@ -743,48 +1056,76 @@ export class TaskScheduler {
743
1056
  /**
744
1057
  * Load the owner edges and task nodes from `start` up to its ownerless root. A known task outside `#unloaded` ends the
745
1058
  * walk, as its chain is loaded; the `#unloaded` tasks passed have a loaded chain afterwards. Loading a node can reveal
746
- * a cancelled owner, so it makes the next reconcile derive cascade marks.
1059
+ * a cancelled owner, so it makes the next reconcile derive cascade marks. The nodes it installs stay until the job's
1060
+ * commit is published (`#pin()`).
747
1061
  */
748
1062
  async #loadChain(start, overlay) {
749
1063
  const passed = [];
1064
+ let installed = false;
750
1065
  let at = start;
751
- while (at !== undefined) {
752
- if ("task" in at) {
753
- const id = at.task;
754
- let node = this.#live.get(id) ?? this.#settled.get(id);
755
- // Created in this commit: the load from its own record covers what is above it.
756
- if (node === undefined && overlay?.tasks.has(id) === true)
757
- break;
758
- if (node !== undefined && !this.#unloaded.has(id))
759
- break;
760
- if (node === undefined) {
761
- const record = await this.#storage.task(id, this.#context);
762
- if (record === undefined)
763
- return;
764
- node = nodeOf(record);
765
- if (record.state.status === "terminal")
766
- this.#settle(id, node);
767
- // A loaded node can reveal a cancelling owner: the next reconcile derives marks again.
768
- this.#scheduleReconcile();
1066
+ try {
1067
+ while (at !== undefined) {
1068
+ if ("task" in at) {
1069
+ const id = at.task;
1070
+ let node = this.#live.get(id) ?? this.#settled.get(id);
1071
+ // Created in this commit: the load from its own record covers what is above it.
1072
+ if (node === undefined && overlay?.tasks.has(id) === true)
1073
+ break;
1074
+ if (node !== undefined && !this.#unloaded.has(id))
1075
+ break;
1076
+ if (node === undefined) {
1077
+ // Through the line's Storage: a task whose creation or end is not published yet waits for it.
1078
+ const record = await this.#session.lineStorage.task(id, this.#context);
1079
+ if (record === undefined)
1080
+ return;
1081
+ node = nodeOf(record);
1082
+ if (record.state.status === "terminal") {
1083
+ this.#settle(id, node);
1084
+ this.#pin(id);
1085
+ installed = true;
1086
+ }
1087
+ // A loaded node can reveal a cancelling owner: the next reconcile derives marks again.
1088
+ this.#scheduleReconcile();
1089
+ }
1090
+ passed.push(id);
1091
+ at = parentOf(node);
769
1092
  }
770
- passed.push(id);
771
- at = parentOf(node);
772
- }
773
- else {
774
- let edge = this.#edge(at.conversation, overlay);
775
- if (edge === undefined) {
776
- const record = await this.#storage.conversation(at.conversation, this.#context);
777
- edge = record?.owner?.taskId ?? null;
778
- this.#setEdge(at.conversation, edge);
779
- // A loaded node can reveal a cancelling owner: the next reconcile derives marks again.
780
- this.#scheduleReconcile();
1093
+ else {
1094
+ let edge = this.#edge(at.conversation, overlay);
1095
+ if (edge === undefined) {
1096
+ // Through the line's Storage: a conversation whose creation is not published yet waits for it, so a missing
1097
+ // record is never mistaken for an ownerless one.
1098
+ const record = await this.#session.lineStorage.conversation(at.conversation, this.#context);
1099
+ edge = record?.owner?.taskId ?? null;
1100
+ this.#setEdge(at.conversation, edge);
1101
+ this.#pin(conversationNode(at.conversation));
1102
+ installed = true;
1103
+ // A loaded node can reveal a cancelling owner: the next reconcile derives marks again.
1104
+ this.#scheduleReconcile();
1105
+ }
1106
+ at = edge === null ? undefined : { task: edge };
781
1107
  }
782
- at = edge === null ? undefined : { task: edge };
1108
+ }
1109
+ }
1110
+ finally {
1111
+ // `#idle()` counts a live task whose chain is not loaded as inside every scope it may be in: what this load
1112
+ // resolved, also when it stopped early, can make such a scope idle without a task change below it.
1113
+ if (passed.some((id) => this.#live.has(id)) || (installed && this.#liveUnloaded())) {
1114
+ this.#unknownsChanged = true;
1115
+ // A waiter such a task held is checked again at the pass's end, also when this load was not a reconcile's.
1116
+ this.#scheduleReconcile();
783
1117
  }
784
1118
  }
785
1119
  for (const id of passed)
786
1120
  this.#unloaded.delete(id);
787
1121
  }
1122
+ /** Whether a live task's chain is not loaded yet: one `#idle()` counts as inside every scope it may be in. */
1123
+ #liveUnloaded() {
1124
+ for (const id of this.#unloaded)
1125
+ if (this.#live.has(id))
1126
+ return true;
1127
+ return false;
1128
+ }
788
1129
  #setEdge(conversationId, owner) {
789
1130
  this.#edges.set(conversationId, owner);
790
1131
  // The conversation now hangs from its owner, or is a root; with nothing below it, it waits for the sweep.
@@ -831,7 +1172,8 @@ export class TaskScheduler {
831
1172
  */
832
1173
  #sweep() {
833
1174
  for (const node of this.#dropQueue) {
834
- if (this.#below.has(node))
1175
+ // Queued again once its pins go.
1176
+ if (this.#below.has(node) || this.#pins.has(node))
835
1177
  continue;
836
1178
  if (typeof node === "string") {
837
1179
  this.#edges.delete(conversationOf(node));
@@ -844,6 +1186,69 @@ export class TaskScheduler {
844
1186
  }
845
1187
  this.#dropQueue.clear();
846
1188
  }
1189
+ /**
1190
+ * Run `decide`, a decision region (group commit §6): synchronous, reading scheduler memory and recording the keys it
1191
+ * reads, staging nothing and changing no scheduler state. When a key it read has an unpublished writer, its result,
1192
+ * or its error, is dropped: once that writer is published, `refresh` reloads what the region needs and it runs
1193
+ * again. A waiting job holds the line, so unpublished writes only shrink, and the runs converge. What a region read
1194
+ * that had no unpublished writer gets none for the rest of the job, so its result stands while the job applies it.
1195
+ */
1196
+ async #region(decide, refresh) {
1197
+ for (;;) {
1198
+ const keys = new Set();
1199
+ this.#reading = keys;
1200
+ let result;
1201
+ try {
1202
+ result = { value: decide() };
1203
+ }
1204
+ catch (error) {
1205
+ result = { error };
1206
+ }
1207
+ finally {
1208
+ this.#reading = undefined;
1209
+ }
1210
+ const gates = [];
1211
+ for (const key of keys) {
1212
+ const gate = this.#session.touch(key);
1213
+ if (gate !== undefined)
1214
+ gates.push(gate);
1215
+ }
1216
+ if (gates.length === 0) {
1217
+ if ("error" in result)
1218
+ throw result.error;
1219
+ return result.value;
1220
+ }
1221
+ await Promise.all(gates);
1222
+ await refresh?.();
1223
+ }
1224
+ }
1225
+ /** Load the chains a region walks, again after it waited: what was published meanwhile may list new work. */
1226
+ #reloadScopes = () => this.#loadScopes(false);
1227
+ /** In a decision region, note that it reads `key`; anywhere else, nothing. */
1228
+ #record(key) {
1229
+ this.#reading?.add(key);
1230
+ }
1231
+ /** A live task's committed record, read by a decision region. */
1232
+ #liveRecord(id) {
1233
+ this.#record(taskKey(id));
1234
+ return this.#live.get(id);
1235
+ }
1236
+ /**
1237
+ * Keep `node`, which a chain load of the job on the line installed, until that job's commit is published or fails:
1238
+ * work it creates below the node needs it, and the sweep would drop it as unneeded before then.
1239
+ */
1240
+ #pin(node) {
1241
+ this.#pins.set(node, (this.#pins.get(node) ?? 0) + 1);
1242
+ this.#session.onJobEnd(() => {
1243
+ const count = this.#pins.get(node) - 1;
1244
+ if (count > 0) {
1245
+ this.#pins.set(node, count);
1246
+ return;
1247
+ }
1248
+ this.#pins.delete(node);
1249
+ this.#dropIfUnneeded(node);
1250
+ });
1251
+ }
847
1252
  /**
848
1253
  * Bring the scheduler's sets in line with a task's committed record. A task that ended may free the nearest live task
849
1254
  * above it: one holding its outcome finalizes, an abort-marked one runs its abort handler; its waiters may run.
@@ -855,11 +1260,11 @@ export class TaskScheduler {
855
1260
  invocation.record = record;
856
1261
  if (record.state.status === "terminal") {
857
1262
  if (previous !== undefined && !this.#closing)
858
- this.#activeMetric.add(-1, { task: record.kind });
1263
+ this.#countActive(-1, record.kind);
859
1264
  this.#resumed.delete(id);
860
1265
  }
861
1266
  else if (previous === undefined && !this.#closing) {
862
- this.#activeMetric.add(1, { task: record.kind });
1267
+ this.#countActive(1, record.kind);
863
1268
  }
864
1269
  if (previous?.state.status === "waiting")
865
1270
  for (const on of previous.state.on)
@@ -967,11 +1372,12 @@ export class TaskScheduler {
967
1372
  * at or below a background task holds nothing above it.
968
1373
  */
969
1374
  #nearestLiveAbove(node, overlay) {
1375
+ this.#record(INTENT_KEY);
970
1376
  for (let current = node; !current.background;) {
971
1377
  const owner = current.owner ?? this.#edge(current.conversationId, overlay);
972
1378
  if (owner === undefined || owner === null)
973
1379
  return undefined;
974
- const record = overlay?.tasks.get(owner) ?? this.#live.get(owner);
1380
+ const record = overlay?.tasks.get(owner) ?? this.#liveRecord(owner);
975
1381
  if (record !== undefined && record.state.status !== "terminal")
976
1382
  return record;
977
1383
  const above = record ?? this.#settled.get(owner);
@@ -991,7 +1397,10 @@ export class TaskScheduler {
991
1397
  }
992
1398
  /** Owner task of a conversation, `null` when ownerless, `undefined` while not loaded. */
993
1399
  #edge(id, overlay) {
994
- return overlay?.edges.has(id) === true ? overlay.edges.get(id) : this.#edges.get(id);
1400
+ if (overlay?.edges.has(id) === true)
1401
+ return overlay.edges.get(id);
1402
+ this.#record(conversationKey(id));
1403
+ return this.#edges.get(id);
995
1404
  }
996
1405
  #node(id, overlay) {
997
1406
  return overlay?.tasks.get(id) ?? this.#live.get(id) ?? this.#settled.get(id);
@@ -1030,6 +1439,7 @@ export class TaskScheduler {
1030
1439
  const links = overlay === undefined ? undefined : this.#links(overlay);
1031
1440
  const pending = [start];
1032
1441
  for (let at = pending.pop(); at !== undefined; at = pending.pop()) {
1442
+ this.#record(ownedKey(at));
1033
1443
  for (const children of [this.#below.get(at), links?.get(at)]) {
1034
1444
  for (const child of children ?? []) {
1035
1445
  if (typeof child === "string") {
@@ -1117,6 +1527,7 @@ export class TaskScheduler {
1117
1527
  * edge is not loaded.
1118
1528
  */
1119
1529
  #inScope(start, scope, crossBackground = false) {
1530
+ this.#record(INTENT_KEY);
1120
1531
  for (const step of this.#above(start)) {
1121
1532
  if ("unknown" in step)
1122
1533
  return undefined;
@@ -1139,6 +1550,7 @@ export class TaskScheduler {
1139
1550
  }
1140
1551
  /** The nearest live owner above `start` whose cancellation intent reaches it, if any; see `#belowCancelled`. */
1141
1552
  #cancellingOwner(start) {
1553
+ this.#record(INTENT_KEY);
1142
1554
  // Nothing cancels, the common case: no walk.
1143
1555
  if (this.#intent.size === 0)
1144
1556
  return undefined;
@@ -1155,6 +1567,18 @@ export class TaskScheduler {
1155
1567
  }
1156
1568
  return undefined;
1157
1569
  }
1570
+ /**
1571
+ * Count a task in or out of `pi.durable.tasks.active`. Commit observers count at a commit's publication, which must not
1572
+ * throw, so a throwing adapter is reported.
1573
+ */
1574
+ #countActive(delta, kind) {
1575
+ try {
1576
+ this.#activeMetric.add(delta, { task: kind });
1577
+ }
1578
+ catch (error) {
1579
+ this.#report(error, TELEMETRY);
1580
+ }
1581
+ }
1158
1582
  /** Fail the Session, which reports it, for a throw in one of the scheduler's own commits; ignored once closing. */
1159
1583
  #failSession(error, context) {
1160
1584
  if (!this.#closing)
@@ -1163,7 +1587,7 @@ export class TaskScheduler {
1163
1587
  /** Close listener: runs synchronously once admission is sealed, before `join()`, or when the Session fails. */
1164
1588
  #seal() {
1165
1589
  for (const record of this.#live.values())
1166
- this.#activeMetric.add(-1, { task: record.kind });
1590
+ this.#countActive(-1, record.kind);
1167
1591
  this.#resumed.clear();
1168
1592
  this.#closing = true;
1169
1593
  try {
@@ -1176,14 +1600,18 @@ export class TaskScheduler {
1176
1600
  this.#taskWaiters.rejectAll(error);
1177
1601
  this.#idleWaiters.rejectAll(error);
1178
1602
  this.#contexts.clear();
1603
+ this.#idleOrder.clear();
1179
1604
  this.#expiry?.disarm();
1180
1605
  this.#expiry = undefined;
1181
1606
  this.#timer?.disarm();
1182
1607
  this.#timer = undefined;
1183
1608
  }
1184
- /** Recompute the wake in a microtask, off the Session line: arm the built-in timer, or tell `onWake` of a change. */
1609
+ /**
1610
+ * Recompute the wake in a microtask the line never waits for, with no order relative to host commit listeners (one
1611
+ * already queued serves later changes too): arm the built-in timer, or tell `onWake` of a change.
1612
+ */
1185
1613
  #scheduleWake() {
1186
- if (this.#wakeScheduled || this.#closing)
1614
+ if (this.#wakeScheduled || this.#closing || this.#readOnly)
1187
1615
  return;
1188
1616
  this.#wakeScheduled = true;
1189
1617
  queueMicrotask(() => {
@@ -1235,10 +1663,6 @@ export class TaskScheduler {
1235
1663
  this.#runnable.add(id);
1236
1664
  }
1237
1665
  }
1238
- /** A task cut off `restartLimit` times in a row; an abort mark still runs its abort handler. */
1239
- #parked(record) {
1240
- return !record.abortRequested && record.restarts !== undefined && record.restarts >= this.#restartLimit();
1241
- }
1242
1666
  /**
1243
1667
  * The report info of a task's failure, with the extension that installed its definition: the code that failed is
1244
1668
  * that extension's. Built-in tasks have none.
@@ -1259,22 +1683,6 @@ export class TaskScheduler {
1259
1683
  }
1260
1684
  return { task: reportTask(task), ...optionalExtension(extension) };
1261
1685
  }
1262
- /** Make parked tasks that a raised `settings.restartLimit` no longer parks candidates again, and tell the graph. */
1263
- #unparkRaised() {
1264
- let unparked = false;
1265
- for (const id of this.#parkedTasks) {
1266
- const record = this.#live.get(id);
1267
- if (record !== undefined && this.#parked(record))
1268
- continue;
1269
- this.#parkedTasks.delete(id);
1270
- if (record === undefined)
1271
- continue;
1272
- this.#runnable.add(id);
1273
- unparked = true;
1274
- }
1275
- if (unparked)
1276
- queueMicrotask(() => this.#notifyBlocked());
1277
- }
1278
1686
  /** `settings.restartLimit`, or the default when the host's settings throw. */
1279
1687
  #restartLimit() {
1280
1688
  try {
@@ -1291,7 +1699,7 @@ export class TaskScheduler {
1291
1699
  listener();
1292
1700
  }
1293
1701
  catch (error) {
1294
- this.#report(error, { type: "listener_error" });
1702
+ this.#report(error, { kind: "listener_error" });
1295
1703
  }
1296
1704
  }
1297
1705
  }
@@ -1337,62 +1745,43 @@ export class TaskScheduler {
1337
1745
  const faulted = [];
1338
1746
  let abandoned = [];
1339
1747
  try {
1340
- await this.#session.commitWith(async (tx) => {
1748
+ await this.#session.commitInternal(async (tx) => {
1341
1749
  if (!this.#enabled || this.#closing)
1342
1750
  return;
1343
- for (const id of this.#abandoned) {
1344
- const record = this.#live.get(id);
1345
- if (record !== undefined && !record.abortRequested)
1346
- tx.setTask(withAbortMark(record, "restart"));
1347
- }
1348
1751
  if (this.#abandoned.size > 0) {
1349
- abandoned = [...this.#abandoned];
1752
+ const ids = [...this.#abandoned];
1753
+ const unmarked = await this.#region(() => ids
1754
+ .map((id) => this.#liveRecord(id))
1755
+ .filter((record) => record !== undefined && !record.abortRequested));
1756
+ for (const record of unmarked)
1757
+ tx.setTask(withAbortMark(record, "restart"));
1758
+ abandoned = ids;
1350
1759
  return;
1351
1760
  }
1352
1761
  await this.#loadScopes(false);
1353
1762
  // Every pass picks up sleeps whose time has come: a pass started by any change may be the first to run after
1354
1763
  // a deadline, and the wake it schedules no longer keeps a timer for a due sleep.
1355
1764
  this.#wakeDue();
1356
- this.#unparkRaised();
1765
+ const plan = await this.#planCandidates();
1766
+ // Work below an owner with cancellation intent waits for its cascade mark instead of running a phase; schedule
1767
+ // the cascade, which may not have run yet.
1768
+ if (plan.cascade) {
1769
+ this.#cascadePending = true;
1770
+ this.#scheduleReconcile();
1771
+ }
1357
1772
  // Taken once per pass, and only when some task is a candidate.
1358
1773
  let snapshot;
1359
- for (const id of [...this.#runnable]) {
1360
- const record = this.#live.get(id);
1361
- if (record === undefined || record.state.status === "completing" || this.#invocations.has(id)) {
1362
- this.#runnable.delete(id);
1363
- continue;
1364
- }
1365
- // It returns once what it waits for ends, its record changes, or the work below it ends.
1366
- if (this.#waitingOn(record, true).length > 0) {
1367
- this.#runnable.delete(id);
1368
- continue;
1369
- }
1370
- // It returns at the first pass after its time comes, which the timer, `resume()`, or any change starts.
1371
- if (this.#notDue(record)) {
1372
- this.#runnable.delete(id);
1373
- continue;
1374
- }
1375
- // It returns when `retry()` clears its restart count, or at a pass after `settings.restartLimit` rose.
1376
- if (this.#parked(record)) {
1377
- this.#runnable.delete(id);
1378
- // Newly parked without a commit, as after a lowered limit: the graph must hear it.
1379
- if (!this.#parkedTasks.has(id))
1380
- queueMicrotask(() => this.#notifyBlocked());
1381
- this.#parkedTasks.add(id);
1382
- continue;
1383
- }
1384
- const runnable = record;
1774
+ for (const record of plan.ready) {
1775
+ const id = record.id;
1385
1776
  const mode = record.abortRequested ? "abort" : "run";
1386
- // Work below an owner with cancellation intent waits for its cascade mark instead of running a phase; schedule
1387
- // the cascade, which may not have run yet.
1388
- if (mode === "run" && !record.background && this.#belowCancelled(parentOf(record))) {
1389
- this.#cascadePending = true;
1390
- this.#scheduleReconcile();
1391
- continue;
1392
- }
1393
- // An abort handler cut off `restartLimit` times in a row would run forever; nothing is left to park it for.
1394
- if (mode === "abort" && (record.restarts ?? 0) >= this.#restartLimit()) {
1395
- const message = `Abort handler of task ${id} (${record.kind}) was cut off ${record.restarts} times`;
1777
+ // Work cut off `restartLimit` times in a row would run forever: an abort handler, or a phase of a kind that
1778
+ // does not end itself at the limit, ends the task `faulted` instead.
1779
+ // The settings are read only for a task that was cut off at all.
1780
+ const cutOff = record.restarts !== undefined && record.restarts >= this.#restartLimit();
1781
+ if (cutOff && (mode === "abort" || !this.#limitsRestarts(record.kind))) {
1782
+ const message = mode === "abort"
1783
+ ? `Abort handler of task ${id} (${record.kind}) was cut off ${record.restarts} times`
1784
+ : `Task ${id} (${record.kind}) was cut off ${record.restarts} times without progress`;
1396
1785
  await this.#terminate(tx, record, {
1397
1786
  status: "faulted",
1398
1787
  error: { message, detail: { reason: "restart_limit" } },
@@ -1401,7 +1790,7 @@ export class TaskScheduler {
1401
1790
  continue;
1402
1791
  }
1403
1792
  snapshot ??= this.#registry.snapshot();
1404
- const resolution = this.#resolve(runnable, snapshot);
1793
+ const resolution = this.#resolve(record, snapshot);
1405
1794
  if (resolution.kind === "blocked") {
1406
1795
  // An abandoned task waits for its definition, so its abort handler can clean up.
1407
1796
  if (mode === "abort" && record.abortReason === undefined) {
@@ -1409,7 +1798,7 @@ export class TaskScheduler {
1409
1798
  }
1410
1799
  continue;
1411
1800
  }
1412
- if (resolution.record !== runnable || record.state.status !== "running") {
1801
+ if (resolution.record !== record || record.state.status !== "running") {
1413
1802
  tx.setTask(withState(resolution.record, {
1414
1803
  status: "running",
1415
1804
  checkpoint: resolution.record.state.checkpoint,
@@ -1418,7 +1807,7 @@ export class TaskScheduler {
1418
1807
  // Registered on the line, so marks and later reservations see it and close joins it; `#end` makes it a
1419
1808
  // candidate again.
1420
1809
  this.#runnable.delete(id);
1421
- const invocation = this.#createInvocation(runnable, mode, installerOf(snapshot, resolution.task));
1810
+ const invocation = this.#createInvocation(record, mode, installerOf(snapshot, resolution.task));
1422
1811
  reservations.push({ invocation, task: resolution.task, snapshot });
1423
1812
  }
1424
1813
  }, this.#context);
@@ -1433,7 +1822,7 @@ export class TaskScheduler {
1433
1822
  }
1434
1823
  for (const { record, message } of faulted) {
1435
1824
  this.#report(new Error(message), {
1436
- type: "task_faulted",
1825
+ kind: "task_faulted",
1437
1826
  ...this.#aboutTask(recordTask(record)),
1438
1827
  reason: "restart_limit",
1439
1828
  });
@@ -1445,6 +1834,65 @@ export class TaskScheduler {
1445
1834
  this.#dirty = true;
1446
1835
  return reservations;
1447
1836
  }
1837
+ /**
1838
+ * Plan a reservation pass over the candidates, taken out of `#runnable` while it plans: a task made a candidate
1839
+ * meanwhile, such as a waiter whose last task ended, lands in the fresh set, so the pass's decision to drop it cannot
1840
+ * erase that. The candidates not dropped go back in front of the fresh set, unless they ended or hold meanwhile.
1841
+ */
1842
+ async #planCandidates() {
1843
+ const candidates = this.#runnable;
1844
+ this.#runnable = new Set();
1845
+ let dropped = new Set();
1846
+ try {
1847
+ const plan = await this.#region(() => this.#planReservation(candidates), this.#reloadScopes);
1848
+ dropped = new Set(plan.dropped);
1849
+ return plan;
1850
+ }
1851
+ finally {
1852
+ const merged = new Set();
1853
+ for (const id of candidates) {
1854
+ const status = this.#live.get(id)?.state.status;
1855
+ if (!dropped.has(id) && status !== undefined && status !== "completing")
1856
+ merged.add(id);
1857
+ }
1858
+ for (const id of this.#runnable)
1859
+ merged.add(id);
1860
+ this.#runnable = merged;
1861
+ }
1862
+ }
1863
+ /**
1864
+ * A reservation pass's decision region: the candidates it drops (gone, holding, waiting, asleep, or with an
1865
+ * invocation), whether one waits for a cascade, and the runnable records to resolve. Iterating the candidates is not
1866
+ * keyed: a pass that misses a task a pending commit makes runnable only delays it, as does skipping one with an
1867
+ * invocation without reading its record, or a waiter's check whether what it waits on is still live.
1868
+ */
1869
+ #planReservation(candidates) {
1870
+ const dropped = [];
1871
+ const ready = [];
1872
+ let cascade = false;
1873
+ for (const id of candidates) {
1874
+ if (this.#invocations.has(id)) {
1875
+ dropped.push(id);
1876
+ continue;
1877
+ }
1878
+ const record = this.#liveRecord(id);
1879
+ // It returns once what it waits for ends, its record changes, or the work below it ends; a sleep at the first pass
1880
+ // after its time comes, which the timer, `resume()`, or any change starts.
1881
+ if (record === undefined ||
1882
+ record.state.status === "completing" ||
1883
+ this.#waitingOn(record, true).length > 0 ||
1884
+ this.#notDue(record)) {
1885
+ dropped.push(id);
1886
+ continue;
1887
+ }
1888
+ if (!record.abortRequested && !record.background && this.#belowCancelled(parentOf(record))) {
1889
+ cascade = true;
1890
+ continue;
1891
+ }
1892
+ ready.push(record);
1893
+ }
1894
+ return { dropped, cascade, ready };
1895
+ }
1448
1896
  /**
1449
1897
  * Live tasks a task waits for before its next invocation: its live ordinary owned work when abort-marked, since
1450
1898
  * abort runs bottom-up, otherwise the live part of the `on` of a wait.
@@ -1481,7 +1929,7 @@ export class TaskScheduler {
1481
1929
  queueMicrotask(() => this.#notifyBlocked());
1482
1930
  this.#scheduleWake();
1483
1931
  const extension = installerOf(snapshot, fit.task);
1484
- this.#report(error, { type: "migration_failed", ...this.#aboutTask({ ...recordTask(record), extension }) });
1932
+ this.#report(error, { kind: "migration_failed", ...this.#aboutTask({ ...recordTask(record), extension }) });
1485
1933
  return { kind: "blocked", reason: "migration_failed" };
1486
1934
  }
1487
1935
  }
@@ -1511,6 +1959,36 @@ export class TaskScheduler {
1511
1959
  roots: this.#roots.size,
1512
1960
  unloaded: this.#unloaded.size,
1513
1961
  dropQueue: this.#dropQueue.size,
1962
+ pins: this.#pins.size,
1963
+ };
1964
+ }
1965
+ /**
1966
+ * The kept contexts' idle state, for tests (a diagnostic, computed on demand): how many are kept and how many are
1967
+ * idle, how many disagree with a full recompute (an idle period without `#idle()`, or `#idle()` without one; an
1968
+ * expiry order that differs from the idle periods, or is not sorted), and how many idle checks covered every scope.
1969
+ */
1970
+ get idleState() {
1971
+ let mismatches = 0;
1972
+ let idle = 0;
1973
+ for (const [conversationId, kept] of this.#contexts) {
1974
+ if (kept.idleSince !== undefined)
1975
+ idle++;
1976
+ if ((kept.idleSince !== undefined) !== this.#idle(conversationId))
1977
+ mismatches++;
1978
+ if (this.#idleOrder.get(conversationId) !== kept.idleSince)
1979
+ mismatches++;
1980
+ }
1981
+ let previous = Number.NEGATIVE_INFINITY;
1982
+ for (const [conversationId, idleSince] of this.#idleOrder) {
1983
+ if (idleSince < previous || this.#contexts.get(conversationId)?.idleSince !== idleSince)
1984
+ mismatches++;
1985
+ previous = idleSince;
1986
+ }
1987
+ return {
1988
+ contexts: this.#contexts.size,
1989
+ idleContexts: idle,
1990
+ idleMismatches: mismatches,
1991
+ fullIdleChecks: this.#fullIdleChecks,
1514
1992
  };
1515
1993
  }
1516
1994
  /**
@@ -1519,6 +1997,7 @@ export class TaskScheduler {
1519
1997
  * cannot exist, shows as failed.
1520
1998
  */
1521
1999
  async inspect(snapshot) {
2000
+ await this.#published();
1522
2001
  await this.#loadScopes(false);
1523
2002
  const tasks = [];
1524
2003
  for (const record of this.#live.values())
@@ -1536,15 +2015,13 @@ export class TaskScheduler {
1536
2015
  return { kind: "waiting", on };
1537
2016
  if (this.#notDue(record))
1538
2017
  return { kind: "sleeping", until: record.state.until };
1539
- if (this.#parked(record))
1540
- return { kind: "blocked", reason: "restart_limit" };
1541
2018
  const fit = this.#fit(record, snapshot.task(record.kind));
1542
2019
  if ("reason" in fit)
1543
2020
  return { kind: "blocked", ...fit };
1544
2021
  if (fit.migrates && fit.task.definition.migrate === undefined) {
1545
2022
  return { kind: "blocked", reason: "migration_failed", error: missingMigration(record, erased(fit.task)) };
1546
2023
  }
1547
- return { kind: "ready", migrates: fit.migrates };
2024
+ return { kind: "pending", migrates: fit.migrates };
1548
2025
  }
1549
2026
  #createInvocation(record, mode, extension) {
1550
2027
  const controller = new AbortController();
@@ -1578,7 +2055,7 @@ export class TaskScheduler {
1578
2055
  await this.#runAbort(reservation);
1579
2056
  }
1580
2057
  catch (error) {
1581
- this.#report(error, { type: "task_error", ...this.#aboutTask(invocation) }, invocation.context);
2058
+ this.#report(error, { kind: "task_error", ...this.#aboutTask(invocation) }, invocation.context);
1582
2059
  }
1583
2060
  finally {
1584
2061
  this.#end(invocation);
@@ -1596,7 +2073,8 @@ export class TaskScheduler {
1596
2073
  const state = { task: reservation.task, snapshot: reservation.snapshot, reported: undefined };
1597
2074
  const phase = { snapshot: () => state.snapshot, task: () => state.task, agent: undefined };
1598
2075
  const runtime = this.#runtime(invocation, phase);
1599
- let current = await this.#step(invocation, (tx, next) => this.#decide(tx, next, undefined, state, invocation.extension, invocation.context));
2076
+ const progress = (tx, current) => this.#progress(tx, current, state, invocation.extension, invocation.context);
2077
+ let current = await this.#step(invocation, (task) => this.#rules(task, undefined), progress);
1600
2078
  while (current !== undefined && !this.#closing) {
1601
2079
  const task = current;
1602
2080
  current = await this.#phase(invocation, task, async () => {
@@ -1611,7 +2089,7 @@ export class TaskScheduler {
1611
2089
  catch (error) {
1612
2090
  previous = { checkpoint, failure: { error } };
1613
2091
  }
1614
- return this.#step(invocation, (tx, next) => this.#decide(tx, next, previous, state, invocation.extension, invocation.context));
2092
+ return this.#step(invocation, (next) => this.#rules(next, previous), progress);
1615
2093
  });
1616
2094
  }
1617
2095
  }
@@ -1662,7 +2140,7 @@ export class TaskScheduler {
1662
2140
  const message = terminal.status === "orphaned" ? terminal.reason : terminal.error.message;
1663
2141
  span.setStatus({
1664
2142
  status: "error",
1665
- error: { name: `task_${terminal.status}`, message: message.slice(0, 500) },
2143
+ error: { name: `task_${terminal.status}`, message: ownedSlice(message, 0, 500) },
1666
2144
  });
1667
2145
  }
1668
2146
  else if (outcome === "aborted") {
@@ -1673,45 +2151,50 @@ export class TaskScheduler {
1673
2151
  });
1674
2152
  }
1675
2153
  /**
1676
- * Precedence rules for a run invocation, on the line. Rules 1 (terminal, `completing`, `waiting`, or `sleeping`) and 2
1677
- * (closing) are applied by `#step`. Returns whether the invocation continues with the next phase.
2154
+ * Precedence rules for a run invocation, a decision region of its step. Rules 1 (terminal, `completing`, `waiting`, or
2155
+ * `sleeping`) and 2 (closing) are applied by `#step`. Progress (rule 5) is decided by `#progress()` once the decision
2156
+ * stands, as it takes a registry snapshot.
1678
2157
  */
1679
- #decide(tx, current, previous, state, extension, context) {
2158
+ #rules(current, previous) {
1680
2159
  // 3. abort mark: end; a fresh abort invocation starts once the task's ordinary owned work is gone.
1681
2160
  if (current.abortRequested)
1682
- return false;
2161
+ return END;
1683
2162
  // An owner's cancellation intent ends it too, before its cascade marks it; reservation then holds it back.
1684
- if (!current.background && this.#belowCancelled(parentOf(current))) {
1685
- this.#cascadePending = true;
1686
- this.#scheduleReconcile();
1687
- return false;
1688
- }
2163
+ if (!current.background && this.#belowCancelled(parentOf(current)))
2164
+ return CASCADE;
1689
2165
  if (previous === undefined)
1690
- return true;
2166
+ return CONTINUE;
1691
2167
  // 4. uncaught error.
1692
2168
  if (previous.failure !== undefined)
1693
- return { fault: previous.failure.error };
2169
+ return { kind: "fault", error: previous.failure.error };
1694
2170
  // 6. no durable progress.
1695
2171
  if (jsonEqual(current.state.checkpoint, previous.checkpoint)) {
1696
2172
  const message = `Task ${current.kind} phase ${previous.checkpoint.phase} returned without durable progress`;
1697
- return { fault: new Error(message) };
2173
+ return { kind: "fault", error: new Error(message) };
1698
2174
  }
1699
- // 5. progress: refresh the snapshot; hand over to a replacement definition that can take the task.
2175
+ return PROGRESS;
2176
+ }
2177
+ /**
2178
+ * Rule 5, progress: refresh the snapshot; hand over to a replacement definition that can take the task, writing it
2179
+ * back as `pending`, or keep running under the old one, reported once per resolved definition. Returns whether the
2180
+ * invocation continues.
2181
+ */
2182
+ #progress(tx, current, state, extension, context) {
1700
2183
  state.snapshot = this.#registry.snapshot();
1701
2184
  const next = state.snapshot.task(current.kind);
1702
- if (next !== state.task) {
1703
- if (next !== undefined && canReserve(next, current)) {
1704
- tx.setTask(withState(current, { status: "pending", checkpoint: current.state.checkpoint }));
1705
- return false;
1706
- }
1707
- if (state.reported === undefined || state.reported.task !== next) {
1708
- state.reported = { task: next };
1709
- this.#report(new Error(`Task ${current.id} keeps running under its old ${current.kind} definition`), {
1710
- type: "task_outdated",
1711
- ...this.#aboutTask({ ...recordTask(current), extension }),
1712
- reason: next === undefined ? "missing_task" : "incompatible_task",
1713
- }, context);
1714
- }
2185
+ if (next === state.task)
2186
+ return true;
2187
+ if (next !== undefined && canReserve(next, current)) {
2188
+ tx.setTask(withState(current, { status: "pending", checkpoint: current.state.checkpoint }));
2189
+ return false;
2190
+ }
2191
+ if (state.reported === undefined || state.reported.task !== next) {
2192
+ state.reported = { task: next };
2193
+ this.#report(new Error(`Task ${current.id} keeps running under its old ${current.kind} definition`), {
2194
+ kind: "task_outdated",
2195
+ ...this.#aboutTask({ ...recordTask(current), extension }),
2196
+ reason: outdatedReason(next, current),
2197
+ }, context);
1715
2198
  }
1716
2199
  return true;
1717
2200
  }
@@ -1736,28 +2219,42 @@ export class TaskScheduler {
1736
2219
  failure = { error };
1737
2220
  }
1738
2221
  const message = `Abort handler of task ${invocation.taskId} returned without a terminal outcome`;
1739
- await this.#step(invocation, () => ({ fault: failure?.error ?? new Error(message) }));
2222
+ await this.#step(invocation, () => ({ kind: "fault", error: failure?.error ?? new Error(message) }));
1740
2223
  });
1741
2224
  }
1742
2225
  /**
1743
- * One synchronous decision on the Session line. A task that is no longer running (rule 1: terminal, `completing`,
1744
- * `waiting`, or `sleeping`) or a closing Harness (rule 2) ends the invocation without a write; otherwise `decide` may
1745
- * stage a write and returns whether the invocation continues. Ending happens inside the callback, before a fault's
1746
- * Harness cleanup. A rejected step, such as admission after close, also ends the invocation.
2226
+ * One decision on the Session line, before a phase: a decision region (`#region()`) that reads the task's record. A
2227
+ * task that is no longer running (rule 1: terminal, `completing`, `waiting`, or `sleeping`) or a closing Harness (rule
2228
+ * 2) ends the invocation without a write; otherwise `plan` decides, and a decision that stands is applied: the
2229
+ * invocation continues, with `progress` for rule 5, or ends, with a fault written. Ending happens inside the callback,
2230
+ * before a fault's Harness cleanup. A rejected step, such as admission after close, also ends the invocation. Its
2231
+ * outcome goes to the invocation only, so it is a scheduler commit (`commitInternal()`).
1747
2232
  */
1748
- async #step(invocation, decide) {
2233
+ async #step(invocation, plan, progress) {
1749
2234
  let fault;
1750
2235
  try {
1751
- const continued = await this.#session.commitWith(async (tx) => {
1752
- const found = this.#live.get(invocation.taskId);
1753
- const current = found?.state.status === "running" ? found : undefined;
1754
- const decision = current !== undefined && !this.#closing ? decide(tx, current) : false;
1755
- if (decision === true)
2236
+ const continued = await this.#session.commitInternal(async (tx) => {
2237
+ const decided = await this.#region(() => {
2238
+ const found = this.#liveRecord(invocation.taskId);
2239
+ if (found?.state.status !== "running" || this.#closing)
2240
+ return { plan: END };
2241
+ const current = found;
2242
+ return { current, plan: plan(current) };
2243
+ });
2244
+ const { current } = decided;
2245
+ const step = decided.plan;
2246
+ if (step.kind === "continue")
2247
+ return current;
2248
+ if (step.kind === "progress" && progress(tx, current))
1756
2249
  return current;
1757
2250
  this.#end(invocation);
1758
- if (decision !== false) {
1759
- fault = { error: decision.fault };
1760
- const message = errorMessage(decision.fault);
2251
+ if (step.kind === "cascade") {
2252
+ this.#cascadePending = true;
2253
+ this.#scheduleReconcile();
2254
+ }
2255
+ if (step.kind === "fault") {
2256
+ fault = { error: step.error };
2257
+ const message = errorMessage(step.error);
1761
2258
  invocation.fault = { status: "faulted", error: { message } };
1762
2259
  await this.#terminate(tx, current, invocation.fault);
1763
2260
  }
@@ -1765,7 +2262,7 @@ export class TaskScheduler {
1765
2262
  }, withoutAbortSignal(invocation.context));
1766
2263
  // Reported once the fault is durable: a host may count it, for example toward a breaker.
1767
2264
  if (fault !== undefined)
1768
- this.#report(fault.error, { type: "task_faulted", ...this.#aboutTask(invocation), reason: "threw" }, invocation.context);
2265
+ this.#report(fault.error, { kind: "task_faulted", ...this.#aboutTask(invocation), reason: "threw" }, invocation.context);
1769
2266
  return continued;
1770
2267
  }
1771
2268
  catch (error) {
@@ -1782,7 +2279,7 @@ export class TaskScheduler {
1782
2279
  */
1783
2280
  async #terminate(tx, record, outcome) {
1784
2281
  await this.#loadScopes(false);
1785
- if (this.#hasOwnedLive(record.id, overlayOf(tx))) {
2282
+ if (await this.#region(() => this.#hasOwnedLive(record.id, overlayOf(tx)), this.#reloadScopes)) {
1786
2283
  tx.setTask(withState(record, { status: "completing", outcome }));
1787
2284
  return;
1788
2285
  }
@@ -1804,7 +2301,7 @@ export class TaskScheduler {
1804
2301
  await this.#loadScopes(false);
1805
2302
  for (const record of overlay.tasks.values())
1806
2303
  await this.#loadChain(parentOf(record), overlay);
1807
- if (this.#hasOwnedLive(current.id, overlay)) {
2304
+ if (await this.#region(() => this.#hasOwnedLive(current.id, overlayOf(tx)), this.#reloadScopes)) {
1808
2305
  tx.setTask(withState(progressed(current), { status: "completing", outcome: next.outcome }));
1809
2306
  return;
1810
2307
  }
@@ -1825,37 +2322,64 @@ export class TaskScheduler {
1825
2322
  async #validateWait(tx, invocation, current, on, policy) {
1826
2323
  if (invocation.mode === "abort")
1827
2324
  throw new Error(`Abort handler of task ${current.id} cannot wait`);
1828
- const overlay = overlayOf(tx);
2325
+ // Owned work is found by walks down, which need the chains of every live task loaded.
2326
+ await this.#loadScopes(false);
2327
+ // Tasks in `on` that are not live, read from Storage: terminal, or missing.
2328
+ const ended = new Map();
2329
+ for (;;) {
2330
+ const checked = await this.#region(() => this.#checkWait(current, on, policy, overlayOf(tx), ended), this.#reloadScopes);
2331
+ if (checked === undefined)
2332
+ return;
2333
+ if ("missing" in checked) {
2334
+ for (const id of checked.missing)
2335
+ ended.set(id, (await this.#session.lineStorage.task(id, this.#context)) ?? null);
2336
+ continue;
2337
+ }
2338
+ throw checked.error;
2339
+ }
2340
+ }
2341
+ /**
2342
+ * `#validateWait()`'s decision region: why the wait is refused, `undefined` when it may be stored, or the tasks in
2343
+ * `on` it needs from Storage first, when `ended` lacks them.
2344
+ */
2345
+ #checkWait(current, on, policy, overlay, ended) {
1829
2346
  const targets = [];
2347
+ const missing = [];
1830
2348
  for (const id of on) {
1831
- const member = overlay.tasks.get(id) ?? this.#live.get(id) ?? (await this.#storage.task(id, this.#context));
1832
- if (member === undefined)
1833
- throw new Error(`Task ${id} does not exist`);
2349
+ const member = overlay.tasks.get(id) ?? this.#liveRecord(id) ?? ended.get(id);
2350
+ if (member === undefined) {
2351
+ missing.push(id);
2352
+ continue;
2353
+ }
2354
+ if (member === null)
2355
+ return { error: new Error(`Task ${id} does not exist`) };
1834
2356
  if (policy === "failFast" && member.owner !== current.id) {
1835
- throw new Error(`Task ${current.id} can wait failFast only on tasks it owns; ${id} is not one`);
2357
+ return { error: new Error(`Task ${current.id} can wait failFast only on tasks it owns; ${id} is not one`) };
1836
2358
  }
1837
2359
  // A live child already holds the waiter (task-owned tasks are never background): waiting on it adds no edge. A
1838
2360
  // terminal task has no edges.
1839
2361
  if (member.state.status !== "terminal" && member.owner !== current.id)
1840
2362
  targets.push(id);
1841
2363
  }
2364
+ if (missing.length > 0)
2365
+ return { missing };
1842
2366
  if (targets.length === 0)
1843
- return;
1844
- const cycle = await this.#waitCycle(current.id, targets, overlay);
1845
- if (cycle !== undefined) {
1846
- throw new Error(`Task ${current.id} cannot wait on ${cycle[0]}: ${[current.id, ...cycle].join(" -> ")} is a wait cycle`);
1847
- }
2367
+ return undefined;
2368
+ const cycle = this.#waitCycle(current.id, targets, overlay);
2369
+ if (cycle === undefined)
2370
+ return undefined;
2371
+ return {
2372
+ error: new Error(`Task ${current.id} cannot wait on ${cycle[0]}: ${[current.id, ...cycle].join(" -> ")} is a wait cycle`),
2373
+ };
1848
2374
  }
1849
2375
  /**
1850
2376
  * The path from a member of `on` back to `task` in the wait-for graph, if the wait would close a cycle (spec §5.1).
1851
2377
  * An edge X -> Y means X cannot settle before Y: Y is in the `on` of waiting X that is not abort-marked, or Y is live
1852
- * ordinary owned work of X,
1853
- * which holds X as `completing`. Each task is visited once and each subtree walked once, so a check is bounded by the
1854
- * live work it reaches.
2378
+ * ordinary owned work of X, which holds X as `completing`. Each task is visited once and each subtree walked once,
2379
+ * so a check is bounded by the live work it reaches. Part of a decision region: it records every record it reads.
2380
+ * Owned work is found by walks down, which need the chains of every live task loaded.
1855
2381
  */
1856
- async #waitCycle(task, on, overlay) {
1857
- // Owned work is found by walks down, which need the chains of every live task loaded.
1858
- await this.#loadScopes(false);
2382
+ #waitCycle(task, on, overlay) {
1859
2383
  const from = new Map();
1860
2384
  const walked = new Set();
1861
2385
  const pending = [];
@@ -1876,7 +2400,7 @@ export class TaskScheduler {
1876
2400
  for (let id = pending.pop(); id !== undefined; id = pending.pop()) {
1877
2401
  if (id === task)
1878
2402
  return path(id);
1879
- const record = overlay.tasks.get(id) ?? this.#live.get(id);
2403
+ const record = overlay.tasks.get(id) ?? this.#liveRecord(id);
1880
2404
  if (record === undefined || record.state.status === "terminal")
1881
2405
  continue;
1882
2406
  // An abort-marked task waits only for its owned work, as abort runs bottom-up (`#waitingOn`).
@@ -1922,6 +2446,9 @@ export class TaskScheduler {
1922
2446
  /** No live non-background task in the scope; a task whose owner edges are not loaded yet counts as inside. */
1923
2447
  #idle(conversationId) {
1924
2448
  const scope = conversationId === undefined ? { roots: true } : { conversation: conversationId };
2449
+ // In a decision region, global idle reads the roots it starts from.
2450
+ if (conversationId === undefined)
2451
+ this.#record(ROOTS_KEY);
1925
2452
  // Tasks whose chain is not loaded are not listed below their scope yet; one counts as inside unless its walk up
1926
2453
  // decides otherwise.
1927
2454
  for (const id of this.#unloaded) {
@@ -1963,7 +2490,7 @@ export class TaskScheduler {
1963
2490
  catch (error) {
1964
2491
  if (invocation.controller.signal.aborted)
1965
2492
  throw error;
1966
- this.#report(error, { type: "extension_error", extension, task: reportTask(invocation) }, invocation.context);
2493
+ this.#report(error, { kind: "extension_error", extension, task: reportTask(invocation) }, invocation.context);
1967
2494
  }
1968
2495
  }
1969
2496
  },
@@ -1986,8 +2513,9 @@ export class TaskScheduler {
1986
2513
  get registry() {
1987
2514
  return phase.snapshot();
1988
2515
  },
1989
- commit: async (change, context) => {
2516
+ commit: async (change, context, options) => {
1990
2517
  let checkpoint;
2518
+ let interrupted;
1991
2519
  await this.#gated(invocation, async (tx, current) => {
1992
2520
  const next = await change(tx, current);
1993
2521
  if (next !== undefined)
@@ -1995,7 +2523,30 @@ export class TaskScheduler {
1995
2523
  if (next?.status === "running" && current.state.checkpoint.phase !== next.checkpoint.phase) {
1996
2524
  checkpoint = next.checkpoint.phase;
1997
2525
  }
1998
- }, context);
2526
+ // A task that ends for crashes says so (`task_interrupted`), so a host keeps a crash-loop signal.
2527
+ if (next?.status === "terminal") {
2528
+ const restarts = current.restarts ?? 0;
2529
+ const reason = next.outcome.status === "failed" ? interruptionOf(next.outcome.error) : undefined;
2530
+ if (reason !== undefined && next.outcome.status === "failed") {
2531
+ interrupted = { message: next.outcome.error.message, reason, restarts };
2532
+ }
2533
+ else {
2534
+ // A task that ended for crashes with another outcome says so itself, as a generation whose run
2535
+ // answered ends `completed` without its after-run tasks.
2536
+ const message = endedByCrashesStates.get(next);
2537
+ if (message !== undefined)
2538
+ interrupted = { message, reason: "restart_limit", restarts };
2539
+ }
2540
+ }
2541
+ }, context, options);
2542
+ if (interrupted !== undefined) {
2543
+ this.#report(new Error(interrupted.message), {
2544
+ kind: "task_interrupted",
2545
+ ...this.#aboutTask(invocation),
2546
+ reason: interrupted.reason,
2547
+ restarts: interrupted.restarts,
2548
+ }, invocation.context);
2549
+ }
1999
2550
  if (checkpoint !== undefined) {
2000
2551
  invocation.context.telemetry.addEvent("pi.durable.checkpoint", {
2001
2552
  phase: checkpoint,
@@ -2019,7 +2570,7 @@ export class TaskScheduler {
2019
2570
  watchDoc: ((...args) => this.#watchDoc(invocation, args)),
2020
2571
  snapshot: ((...args) => this.#read(invocation, () => sessionMethod(this.#session, "snapshot")(...args))),
2021
2572
  snapshotAsOf: ((...args) => this.#read(invocation, () => sessionMethod(this.#session, "snapshotAsOf")(...args))),
2022
- getTask: ((id, context) => this.#read(invocation, () => this.#session.readOnLine(() => this.#storage.task(id, context)))),
2573
+ getTask: ((id, context) => this.#read(invocation, () => this.#session.readOnLine((storage) => storage.task(id, context)))),
2023
2574
  waitForTask: ((id, context) => this.#read(invocation, () => this.waitForTask(id, withAbortSignal(invocation.controller.signal, context)))),
2024
2575
  ownedTasks: (context) => this.#read(invocation, () => this.#session.readOnLine(async () => {
2025
2576
  // The step that ends the invocation may have run on the line before this.
@@ -2027,12 +2578,16 @@ export class TaskScheduler {
2027
2578
  throw endedError(invocation);
2028
2579
  context.abortSignal?.throwIfAborted();
2029
2580
  // Every live task is listed under its parent node, its owner task when it has one.
2030
- const owned = [];
2031
- for (const node of this.#below.get(invocation.taskId) ?? []) {
2032
- const record = typeof node === "string" ? undefined : this.#live.get(node);
2033
- if (record?.owner === invocation.taskId)
2034
- owned.push(record);
2035
- }
2581
+ const owned = await this.#region(() => {
2582
+ this.#record(ownedKey(invocation.taskId));
2583
+ const records = [];
2584
+ for (const node of this.#below.get(invocation.taskId) ?? []) {
2585
+ const record = typeof node === "string" ? undefined : this.#liveRecord(node);
2586
+ if (record?.owner === invocation.taskId)
2587
+ records.push(record);
2588
+ }
2589
+ return records;
2590
+ });
2036
2591
  // Copies, as from storage: the scheduler reads its own.
2037
2592
  return owned
2038
2593
  .sort((a, b) => a.id - b.id)
@@ -2040,7 +2595,7 @@ export class TaskScheduler {
2040
2595
  })),
2041
2596
  abortOwned: (id, context) => this.#read(invocation, async () => {
2042
2597
  const bound = withAbortSignal(invocation.controller.signal, context);
2043
- const record = await this.#session.readOnLine(() => this.#storage.task(id, bound));
2598
+ const record = await this.#session.readOnLine((storage) => storage.task(id, bound));
2044
2599
  if (record?.owner !== invocation.taskId) {
2045
2600
  throw new Error(`Task ${id} is not owned by task ${invocation.taskId}`);
2046
2601
  }
@@ -2049,10 +2604,10 @@ export class TaskScheduler {
2049
2604
  await this.abort(id, bound, true);
2050
2605
  await this.waitForTask(id, bound);
2051
2606
  }),
2052
- outcomes: ((ids, context) => this.#read(invocation, () => this.#session.readOnLine(async () => {
2607
+ outcomes: ((ids, context) => this.#read(invocation, () => this.#session.readOnLine(async (storage) => {
2053
2608
  const outcomes = [];
2054
2609
  for (const id of ids) {
2055
- const state = (await this.#storage.task(id, context))?.state;
2610
+ const state = (await storage.task(id, context))?.state;
2056
2611
  if (state?.status !== "terminal")
2057
2612
  throw new Error(`Task ${id} is not terminal`);
2058
2613
  outcomes.push(state.outcome);
@@ -2071,13 +2626,24 @@ export class TaskScheduler {
2071
2626
  ? [undefined, args[0], args[1]]
2072
2627
  : [args[0], args[1], args[2]];
2073
2628
  return this.#read(invocation, async () => {
2074
- const found = await this.#session.readOnLine(() => this.#storage.entry(invocation.conversationId, id, context));
2075
- const entry = found?.entry;
2629
+ // An entry of the conversation's kept context range is committed and visible: no Storage read.
2630
+ const kept = this.#contexts.get(invocation.conversationId);
2631
+ const entry = (kept === undefined ? undefined : rangeEntry(kept.range, id)) ??
2632
+ (await this.#session.readOnLine((storage) => storage.entry(invocation.conversationId, id, context)))
2633
+ ?.entry;
2076
2634
  return token === undefined || entry?.kind === token.kind ? entry : undefined;
2077
2635
  });
2078
2636
  }),
2637
+ entries: (query, limit, cursor, context) => this.#read(invocation, () => this.#session.readOnLine((storage) => storage.scanEntries(conversationEntries(invocation.conversationId, query), limit, cursor, context))),
2079
2638
  context: (conversationId, context, options) => this.#read(invocation, async () => {
2080
- const { view, range } = await readContextFrom(this.#session, this.#storage, conversationId, context, options?.at, this.#contexts.get(conversationId)?.range);
2639
+ const read = await readContextFrom(this.#session, this.#storage, conversationId, context, options?.at, this.#contexts.get(conversationId)?.range);
2640
+ const { view } = read;
2641
+ // Checked again here, synchronously up to the install below: a commit published since the read returned
2642
+ // was not seen by `#advanceContexts`, so a range that missed it must not be advanced.
2643
+ const range = read.range?.current === true &&
2644
+ this.#session.knownLatestEntryId(conversationId) !== read.range.bounds.tail
2645
+ ? { ...read.range, current: false }
2646
+ : read.range;
2081
2647
  // Keep it unless the invocation ended or a concurrent read already kept a newer range.
2082
2648
  const kept = this.#contexts.get(conversationId);
2083
2649
  if (range !== undefined &&
@@ -2087,9 +2653,17 @@ export class TaskScheduler {
2087
2653
  // A read of another, idle conversation starts or continues its retention period.
2088
2654
  if (!this.#idle(conversationId)) {
2089
2655
  this.#contexts.set(conversationId, { range, idleSince: undefined });
2656
+ this.#idleOrder.delete(conversationId);
2090
2657
  }
2091
2658
  else if (this.#contextRetentionMs() > 0) {
2092
- this.#contexts.set(conversationId, { range, idleSince: kept?.idleSince ?? this.#now() });
2659
+ // An idle period that already started keeps its expiry and its place; a new one starts now.
2660
+ const idleSince = kept?.idleSince ?? this.#now();
2661
+ const expiresAt = kept?.idleSince !== undefined && kept.expiresAt !== undefined
2662
+ ? kept.expiresAt
2663
+ : idleSince + this.#contextRetentionMs();
2664
+ this.#contexts.set(conversationId, { range, idleSince, expiresAt });
2665
+ if (kept?.idleSince === undefined)
2666
+ this.#markIdle(conversationId, idleSince);
2093
2667
  this.#scheduleExpiry();
2094
2668
  }
2095
2669
  }
@@ -2121,13 +2695,22 @@ export class TaskScheduler {
2121
2695
  // The task's own failure names the task's extension; any other names whose code failed, if anyone's.
2122
2696
  const task = reportTask(invocation);
2123
2697
  this.#report(error, from === undefined
2124
- ? { type: "task_error", ...this.#aboutTask(invocation) }
2125
- : from.type === "extension"
2126
- ? { type: "extension_error", task, ...optionalExtension(from.extension) }
2127
- : { type: "host_error", callback: from.callback, task }, invocation.context);
2698
+ ? { kind: "task_error", ...this.#aboutTask(invocation) }
2699
+ : from.kind === "extension"
2700
+ ? { kind: "extension_error", task, ...optionalExtension(from.extension) }
2701
+ : { kind: "host_error", callback: from.callback, task }, invocation.context);
2128
2702
  },
2129
2703
  };
2130
2704
  }
2705
+ /**
2706
+ * On the line: wait until every commit admitted before is published, for a decision that reads what cannot be keyed,
2707
+ * every live task (open, inspection) or every task a conversation abort reaches.
2708
+ */
2709
+ async #published() {
2710
+ const gate = this.#session.touch(ALL_KEYS);
2711
+ if (gate !== undefined)
2712
+ await gate;
2713
+ }
2131
2714
  /** Run a committed-state read unless the invocation has ended. */
2132
2715
  async #read(invocation, read) {
2133
2716
  if (invocation.ended)
@@ -2135,7 +2718,7 @@ export class TaskScheduler {
2135
2718
  return read();
2136
2719
  }
2137
2720
  /** Commit after rereading the task on the line and gating the invocation. */
2138
- #gated(invocation, change, context) {
2721
+ #gated(invocation, change, context, options) {
2139
2722
  if (invocation.ended)
2140
2723
  return Promise.reject(endedError(invocation));
2141
2724
  return this.#session.commitWith(async (tx) => {
@@ -2143,17 +2726,22 @@ export class TaskScheduler {
2143
2726
  throw endedError(invocation);
2144
2727
  if (this.#closing)
2145
2728
  throw closedError(this.#session);
2146
- const found = this.#live.get(invocation.taskId);
2147
- if (found === undefined)
2148
- throw new Error(`Task ${invocation.taskId} is terminal`);
2149
- if (found.state.status !== "running")
2150
- throw new Error(`Task ${invocation.taskId} is ${found.state.status}`);
2151
- const current = found;
2152
- if (invocation.mode === "run" && current.abortRequested) {
2153
- throw new Error(`Task ${invocation.taskId} has a durable abort mark`);
2154
- }
2729
+ const current = await this.#region(() => {
2730
+ const found = this.#liveRecord(invocation.taskId);
2731
+ if (found === undefined)
2732
+ throw new Error(`Task ${invocation.taskId} is terminal`);
2733
+ if (found.state.status !== "running")
2734
+ throw new Error(`Task ${invocation.taskId} is ${found.state.status}`);
2735
+ if (invocation.mode === "run" && found.abortRequested) {
2736
+ throw new Error(`Task ${invocation.taskId} has a durable abort mark`);
2737
+ }
2738
+ return found;
2739
+ });
2740
+ // The step that ends it may have run while the region waited.
2741
+ if (invocation.ended)
2742
+ throw endedError(invocation);
2155
2743
  return change(tx, current);
2156
- }, context, { conversationId: invocation.conversationId, taskId: invocation.taskId });
2744
+ }, context, { conversationId: invocation.conversationId, taskId: invocation.taskId }, options);
2157
2745
  }
2158
2746
  async #watchDoc(invocation, args) {
2159
2747
  if (invocation.ended)
@@ -2170,9 +2758,10 @@ export class TaskScheduler {
2170
2758
  return watch;
2171
2759
  }
2172
2760
  }
2173
- const REGISTRY = { type: "host_error", callback: "registry" };
2174
- const SETTINGS = { type: "host_error", callback: "settings" };
2175
- const ON_WAKE = { type: "host_error", callback: "onWake" };
2761
+ const REGISTRY = { kind: "host_error", callback: "registry" };
2762
+ const SETTINGS = { kind: "host_error", callback: "settings" };
2763
+ const ON_WAKE = { kind: "host_error", callback: "onWake" };
2764
+ const TELEMETRY = { kind: "host_error", callback: "telemetry" };
2176
2765
  /** The extension that installed `task` in `snapshot`, the last one that lists it; none for a built-in task. */
2177
2766
  function installerOf(snapshot, task) {
2178
2767
  if (task === undefined)
@@ -2191,6 +2780,24 @@ function recordTask(record) {
2191
2780
  * The record with an abort mark: `restart` for an abandonment; without a reason, a request, which replaces `restart`.
2192
2781
  * The restart count starts over: it now counts cut-offs of the abort handler.
2193
2782
  */
2783
+ /** Why a failed outcome's error says crashes ended the task (`detail.reason`), if it does. */
2784
+ /**
2785
+ * Terminal states a built-in task commits because it was cut off `settings.restartLimit` times, when their outcome does
2786
+ * not say so (a failure with detail `{ reason: "restart_limit" }` does), with the message the scheduler reports them
2787
+ * `task_interrupted` with. Keyed by the state object the task returns, so nothing is added to the stored state.
2788
+ */
2789
+ const endedByCrashesStates = new WeakMap();
2790
+ /** `state`, marked as ending its task for crashes (`task_interrupted` with `message`), whatever its outcome. */
2791
+ export function endedByCrashes(state, message) {
2792
+ endedByCrashesStates.set(state, message);
2793
+ return state;
2794
+ }
2795
+ function interruptionOf(error) {
2796
+ const detail = error.detail;
2797
+ if (typeof detail !== "object" || detail === null || !("reason" in detail))
2798
+ return undefined;
2799
+ return detail.reason === "restart_limit" || detail.reason === "interrupted" ? detail.reason : undefined;
2800
+ }
2194
2801
  function withAbortMark(record, reason) {
2195
2802
  const { abortReason: _reason, restarts: _restarts, ...rest } = record;
2196
2803
  return { ...rest, abortRequested: true, ...(reason === undefined ? {} : { abortReason: reason }) };
@@ -2199,6 +2806,16 @@ function withAbortMark(record, reason) {
2199
2806
  function cancellationIntent(record) {
2200
2807
  return record.state.status !== "terminal" && (record.abortRequested || failedOutcome(record));
2201
2808
  }
2809
+ /**
2810
+ * Whether replacing `previous` (absent: a new task) with `next` changes the cancellation intent set or what it means: a
2811
+ * mark set, upgraded, or cleared, or a held or ended outcome other than `completed`.
2812
+ */
2813
+ function intentChanges(previous, next) {
2814
+ return ((previous?.abortRequested ?? false) !== next.abortRequested ||
2815
+ previous?.abortReason !== next.abortReason ||
2816
+ (previous !== undefined && failedOutcome(previous)) !== failedOutcome(next) ||
2817
+ (previous !== undefined && cancellationIntent(previous)) !== cancellationIntent(next));
2818
+ }
2202
2819
  /** Whether the record holds or ends with an outcome other than `completed`. */
2203
2820
  function failedOutcome(record) {
2204
2821
  const state = record.state;
@@ -2267,6 +2884,15 @@ function withState(record, state) {
2267
2884
  return { ...rest, state };
2268
2885
  }
2269
2886
  /** Whether a definition can take the task at reservation: same version, or newer with a migration. */
2887
+ /**
2888
+ * Why a replacement definition cannot take a running task, in `#fit()`'s words: none is registered, it is older than
2889
+ * the task, or it is newer without a `migrate`.
2890
+ */
2891
+ function outdatedReason(task, record) {
2892
+ if (task === undefined)
2893
+ return "missing_task";
2894
+ return task.definition.version < record.version ? "task_too_old" : "migration_failed";
2895
+ }
2270
2896
  function canReserve(task, record) {
2271
2897
  const definition = task.definition;
2272
2898
  return (definition.version === record.version || (definition.version > record.version && definition.migrate !== undefined));