@earendil-works/pi-durable 1.3.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +227 -0
- package/README.md +287 -82
- package/dist/codemode/codemode.d.ts +4 -4
- package/dist/codemode/codemode.d.ts.map +1 -1
- package/dist/codemode/codemode.js +21 -16
- package/dist/codemode/codemode.js.map +1 -1
- package/dist/codemode/models.d.ts.map +1 -1
- package/dist/codemode/models.js +6 -4
- package/dist/codemode/models.js.map +1 -1
- package/dist/documents.d.ts +1 -1
- package/dist/documents.d.ts.map +1 -1
- package/dist/documents.js.map +1 -1
- package/dist/env/index.d.ts +12 -10
- package/dist/env/index.d.ts.map +1 -1
- package/dist/env/index.js.map +1 -1
- package/dist/env/node-watch.d.ts +3 -3
- package/dist/env/node-watch.d.ts.map +1 -1
- package/dist/env/node-watch.js +18 -16
- package/dist/env/node-watch.js.map +1 -1
- package/dist/env/node.d.ts +2 -2
- package/dist/env/node.d.ts.map +1 -1
- package/dist/env/node.js +12 -4
- package/dist/env/node.js.map +1 -1
- package/dist/errors.d.ts +18 -2
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +26 -3
- package/dist/errors.js.map +1 -1
- package/dist/harness/agent.d.ts +13 -2
- package/dist/harness/agent.d.ts.map +1 -1
- package/dist/harness/agent.js +48 -6
- package/dist/harness/agent.js.map +1 -1
- package/dist/harness/compaction.d.ts +3 -1
- package/dist/harness/compaction.d.ts.map +1 -1
- package/dist/harness/compaction.js +38 -6
- package/dist/harness/compaction.js.map +1 -1
- package/dist/harness/context.d.ts +23 -7
- package/dist/harness/context.d.ts.map +1 -1
- package/dist/harness/context.js +71 -25
- package/dist/harness/context.js.map +1 -1
- package/dist/harness/define.d.ts +7 -3
- package/dist/harness/define.d.ts.map +1 -1
- package/dist/harness/define.js +4 -0
- package/dist/harness/define.js.map +1 -1
- package/dist/harness/generation.d.ts +38 -5
- package/dist/harness/generation.d.ts.map +1 -1
- package/dist/harness/generation.js +497 -108
- package/dist/harness/generation.js.map +1 -1
- package/dist/harness/harness.d.ts +7 -2
- package/dist/harness/harness.d.ts.map +1 -1
- package/dist/harness/harness.js +147 -57
- package/dist/harness/harness.js.map +1 -1
- package/dist/harness/inbox.d.ts +7 -3
- package/dist/harness/inbox.d.ts.map +1 -1
- package/dist/harness/inbox.js +7 -4
- package/dist/harness/inbox.js.map +1 -1
- package/dist/harness/json.d.ts +20 -4
- package/dist/harness/json.d.ts.map +1 -1
- package/dist/harness/json.js +23 -5
- package/dist/harness/json.js.map +1 -1
- package/dist/harness/live.d.ts +122 -16
- package/dist/harness/live.d.ts.map +1 -1
- package/dist/harness/live.js +93 -12
- package/dist/harness/live.js.map +1 -1
- package/dist/harness/partial-changes.d.ts +54 -0
- package/dist/harness/partial-changes.d.ts.map +1 -0
- package/dist/harness/partial-changes.js +162 -0
- package/dist/harness/partial-changes.js.map +1 -0
- package/dist/harness/prompt.d.ts +1 -1
- package/dist/harness/prompt.d.ts.map +1 -1
- package/dist/harness/prompt.js +1 -1
- package/dist/harness/prompt.js.map +1 -1
- package/dist/harness/registry-reader.d.ts +11 -0
- package/dist/harness/registry-reader.d.ts.map +1 -0
- package/dist/harness/registry-reader.js +25 -0
- package/dist/harness/registry-reader.js.map +1 -0
- package/dist/harness/scheduler.d.ts +86 -14
- package/dist/harness/scheduler.d.ts.map +1 -1
- package/dist/harness/scheduler.js +1459 -369
- package/dist/harness/scheduler.js.map +1 -1
- package/dist/harness/submissions.d.ts +23 -11
- package/dist/harness/submissions.d.ts.map +1 -1
- package/dist/harness/submissions.js +104 -36
- package/dist/harness/submissions.js.map +1 -1
- package/dist/harness/task-graph.d.ts +28 -8
- package/dist/harness/task-graph.d.ts.map +1 -1
- package/dist/harness/task-graph.js +85 -36
- package/dist/harness/task-graph.js.map +1 -1
- package/dist/harness/telemetry.d.ts +307 -0
- package/dist/harness/telemetry.d.ts.map +1 -0
- package/dist/harness/telemetry.js +136 -0
- package/dist/harness/telemetry.js.map +1 -0
- package/dist/harness/tool.d.ts +2 -0
- package/dist/harness/tool.d.ts.map +1 -1
- package/dist/harness/tool.js +223 -61
- package/dist/harness/tool.js.map +1 -1
- package/dist/harness/transcript.d.ts +72 -0
- package/dist/harness/transcript.d.ts.map +1 -0
- package/dist/harness/transcript.js +264 -0
- package/dist/harness/transcript.js.map +1 -0
- package/dist/harness/types.d.ts +302 -48
- package/dist/harness/types.d.ts.map +1 -1
- package/dist/harness/types.js.map +1 -1
- package/dist/harness/util.d.ts +7 -1
- package/dist/harness/util.d.ts.map +1 -1
- package/dist/harness/util.js +13 -0
- package/dist/harness/util.js.map +1 -1
- package/dist/harness/view.d.ts +43 -22
- package/dist/harness/view.d.ts.map +1 -1
- package/dist/harness/view.js +130 -71
- package/dist/harness/view.js.map +1 -1
- package/dist/index.d.ts +13 -9
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -5
- package/dist/index.js.map +1 -1
- package/dist/search.d.ts +19 -3
- package/dist/search.d.ts.map +1 -1
- package/dist/search.js +107 -44
- package/dist/search.js.map +1 -1
- package/dist/session/cached-storage.d.ts +71 -0
- package/dist/session/cached-storage.d.ts.map +1 -0
- package/dist/session/cached-storage.js +269 -0
- package/dist/session/cached-storage.js.map +1 -0
- package/dist/session/conversation-tails.d.ts +33 -0
- package/dist/session/conversation-tails.d.ts.map +1 -0
- package/dist/session/conversation-tails.js +101 -0
- package/dist/session/conversation-tails.js.map +1 -0
- package/dist/session/keys.d.ts +68 -0
- package/dist/session/keys.d.ts.map +1 -0
- package/dist/session/keys.js +161 -0
- package/dist/session/keys.js.map +1 -0
- package/dist/session/session.d.ts +181 -29
- package/dist/session/session.d.ts.map +1 -1
- package/dist/session/session.js +1073 -183
- package/dist/session/session.js.map +1 -1
- package/dist/session/transaction.d.ts +86 -4
- package/dist/session/transaction.d.ts.map +1 -1
- package/dist/session/transaction.js +250 -38
- package/dist/session/transaction.js.map +1 -1
- package/dist/session/writer.d.ts +74 -0
- package/dist/session/writer.d.ts.map +1 -0
- package/dist/session/writer.js +294 -0
- package/dist/session/writer.js.map +1 -0
- package/dist/storage/indexeddb.d.ts +68 -0
- package/dist/storage/indexeddb.d.ts.map +1 -0
- package/dist/storage/indexeddb.js +881 -0
- package/dist/storage/indexeddb.js.map +1 -0
- package/dist/storage/jsonl/storage.d.ts +30 -2
- package/dist/storage/jsonl/storage.d.ts.map +1 -1
- package/dist/storage/jsonl/storage.js +88 -24
- package/dist/storage/jsonl/storage.js.map +1 -1
- package/dist/storage/memory.d.ts +7 -2
- package/dist/storage/memory.d.ts.map +1 -1
- package/dist/storage/memory.js +29 -3
- package/dist/storage/memory.js.map +1 -1
- package/dist/storage/sqlite/cloudflare.d.ts +2 -51
- package/dist/storage/sqlite/cloudflare.d.ts.map +1 -1
- package/dist/storage/sqlite/cloudflare.js +2 -133
- package/dist/storage/sqlite/cloudflare.js.map +1 -1
- package/dist/storage/sqlite/database.d.ts +1 -36
- package/dist/storage/sqlite/database.d.ts.map +1 -1
- package/dist/storage/sqlite/database.js.map +1 -1
- package/dist/storage/sqlite/index.d.ts +1 -1
- package/dist/storage/sqlite/index.d.ts.map +1 -1
- package/dist/storage/sqlite/index.js.map +1 -1
- package/dist/storage/sqlite/migrations.d.ts +2 -5
- package/dist/storage/sqlite/migrations.d.ts.map +1 -1
- package/dist/storage/sqlite/migrations.js +28 -28
- package/dist/storage/sqlite/migrations.js.map +1 -1
- package/dist/storage/sqlite/node.d.ts +2 -37
- package/dist/storage/sqlite/node.d.ts.map +1 -1
- package/dist/storage/sqlite/node.js +3 -174
- package/dist/storage/sqlite/node.js.map +1 -1
- package/dist/storage/sqlite/schema.d.ts +1 -3
- package/dist/storage/sqlite/schema.d.ts.map +1 -1
- package/dist/storage/sqlite/schema.js +2 -9
- package/dist/storage/sqlite/schema.js.map +1 -1
- package/dist/storage/sqlite/storage.d.ts +15 -3
- package/dist/storage/sqlite/storage.d.ts.map +1 -1
- package/dist/storage/sqlite/storage.js +110 -36
- package/dist/storage/sqlite/storage.js.map +1 -1
- package/dist/storage/store/commit.d.ts +33 -0
- package/dist/storage/store/commit.d.ts.map +1 -0
- package/dist/storage/store/commit.js +286 -0
- package/dist/storage/store/commit.js.map +1 -0
- package/dist/storage/store/documents.d.ts +43 -0
- package/dist/storage/store/documents.d.ts.map +1 -0
- package/dist/storage/store/documents.js +102 -0
- package/dist/storage/store/documents.js.map +1 -0
- package/dist/storage/store/index.d.ts +2 -0
- package/dist/storage/store/index.d.ts.map +1 -0
- package/dist/storage/store/index.js +2 -0
- package/dist/storage/store/index.js.map +1 -0
- package/dist/storage/store/queries.d.ts +49 -0
- package/dist/storage/store/queries.d.ts.map +1 -0
- package/dist/storage/store/queries.js +93 -0
- package/dist/storage/store/queries.js.map +1 -0
- package/dist/storage/store/storage.d.ts +59 -0
- package/dist/storage/store/storage.d.ts.map +1 -0
- package/dist/storage/store/storage.js +438 -0
- package/dist/storage/store/storage.js.map +1 -0
- package/dist/storage/store/tables.d.ts +54 -0
- package/dist/storage/store/tables.d.ts.map +1 -0
- package/dist/storage/store/tables.js +108 -0
- package/dist/storage/store/tables.js.map +1 -0
- package/dist/testing/commit-writes.d.ts +5 -0
- package/dist/testing/commit-writes.d.ts.map +1 -0
- package/dist/testing/commit-writes.js +12 -0
- package/dist/testing/commit-writes.js.map +1 -0
- package/dist/testing/env-conformance.js +7 -7
- package/dist/testing/env-conformance.js.map +1 -1
- package/dist/testing/index.d.ts +1 -0
- package/dist/testing/index.d.ts.map +1 -1
- package/dist/testing/index.js +1 -0
- package/dist/testing/index.js.map +1 -1
- package/dist/testing/storage-benchmark.d.ts.map +1 -1
- package/dist/testing/storage-benchmark.js +18 -17
- package/dist/testing/storage-benchmark.js.map +1 -1
- package/dist/testing/storage-conformance.d.ts.map +1 -1
- package/dist/testing/storage-conformance.js +369 -67
- package/dist/testing/storage-conformance.js.map +1 -1
- package/dist/timers.d.ts +21 -0
- package/dist/timers.d.ts.map +1 -0
- package/dist/timers.js +72 -0
- package/dist/timers.js.map +1 -0
- package/dist/tool-search/tool-search.d.ts +6 -12
- package/dist/tool-search/tool-search.d.ts.map +1 -1
- package/dist/tool-search/tool-search.js +7 -7
- package/dist/tool-search/tool-search.js.map +1 -1
- package/dist/tools/bash.d.ts +12 -12
- package/dist/tools/bash.d.ts.map +1 -1
- package/dist/tools/bash.js +16 -15
- package/dist/tools/bash.js.map +1 -1
- package/dist/tools/edit.d.ts +7 -7
- package/dist/tools/edit.d.ts.map +1 -1
- package/dist/tools/edit.js +9 -9
- package/dist/tools/edit.js.map +1 -1
- package/dist/tools/index.d.ts +32 -32
- package/dist/tools/read.d.ts +6 -6
- package/dist/tools/read.d.ts.map +1 -1
- package/dist/tools/read.js +19 -8
- package/dist/tools/read.js.map +1 -1
- package/dist/tools/write.d.ts +4 -4
- package/dist/tools/write.d.ts.map +1 -1
- package/dist/tools/write.js +5 -5
- package/dist/tools/write.js.map +1 -1
- package/dist/types.d.ts +331 -64
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/package.json +27 -10
- package/dist/harness/events.d.ts +0 -186
- package/dist/harness/events.d.ts.map +0 -1
- package/dist/harness/events.js +0 -373
- package/dist/harness/events.js.map +0 -1
- package/dist/session/observation.d.ts +0 -42
- package/dist/session/observation.d.ts.map +0 -1
- package/dist/session/observation.js +0 -281
- package/dist/session/observation.js.map +0 -1
|
@@ -1,13 +1,35 @@
|
|
|
1
1
|
import { copyJson } from "@earendil-works/chord";
|
|
2
|
-
import { awaitWithContext, withAbortSignal } from "@earendil-works/chord/context";
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
2
|
+
import { awaitWithContext, withAbortSignal, withoutAbortSignal, withTelemetry } from "@earendil-works/chord/context";
|
|
3
|
+
import { ownedSlice } from "@earendil-works/chord/delta";
|
|
4
|
+
import { createTypedMetrics, createTypedSpanStarter, NOOP_TELEMETRY_CONTEXT, NOOP_TELEMETRY_METRICS, } from "@earendil-works/pi-telemetry";
|
|
5
|
+
import { errorMessage, ReadOnlySession } from "../errors.js";
|
|
6
|
+
import { ALL_KEYS, conversationKey, INTENT_KEY, ownedKey, ROOTS_KEY, taskKey } from "../session/keys.js";
|
|
7
|
+
import { agentHooks, DEFAULT_RESTART_LIMIT } from "./agent.js";
|
|
8
|
+
import { advanceRange, rangeEntry, readContextFrom } from "./context.js";
|
|
9
|
+
import { DURABLE_TELEMETRY_SCHEMA } from "./telemetry.js";
|
|
10
|
+
import { closedError, conversationEntries, optionalExtension, scanAll, Waiters } from "./util.js";
|
|
7
11
|
const SCAN_PAGE_SIZE = 256;
|
|
8
|
-
|
|
9
|
-
const
|
|
10
|
-
|
|
12
|
+
const NO_KEYS = [];
|
|
13
|
+
const LIVE_STATUSES = ["pending", "running", "waiting", "sleeping", "completing"];
|
|
14
|
+
/** The part of a record `blockedReason()` reads. */
|
|
15
|
+
export function blockedSubject(record) {
|
|
16
|
+
return {
|
|
17
|
+
id: record.id,
|
|
18
|
+
kind: record.kind,
|
|
19
|
+
version: record.version,
|
|
20
|
+
abortRequested: record.abortRequested,
|
|
21
|
+
...(record.abortReason === undefined ? {} : { abortReason: record.abortReason }),
|
|
22
|
+
status: record.state.status,
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
/** Continue with the next phase, the first one. */
|
|
26
|
+
const CONTINUE = { kind: "continue" };
|
|
27
|
+
/** Rule 5: the phase made durable progress; `#progress()` decides once the decision stands. */
|
|
28
|
+
const PROGRESS = { kind: "progress" };
|
|
29
|
+
/** End the invocation without a write. */
|
|
30
|
+
const END = { kind: "end" };
|
|
31
|
+
/** End it: an owner's cancellation intent reaches it before its cascade did; schedule the cascade. */
|
|
32
|
+
const CASCADE = { kind: "cascade" };
|
|
11
33
|
function conversationNode(id) {
|
|
12
34
|
return `c${id}`;
|
|
13
35
|
}
|
|
@@ -21,8 +43,9 @@ function parentNode(node) {
|
|
|
21
43
|
/**
|
|
22
44
|
* Durable task scheduler of one Harness.
|
|
23
45
|
*
|
|
24
|
-
* `#live` mirrors every committed non-terminal task record: pending, running, waiting, and completing. The
|
|
25
|
-
* commit listener updates it on the Session line, so code running on the line reads exactly the committed
|
|
46
|
+
* `#live` mirrors every committed non-terminal task record: pending, running, waiting, sleeping, and completing. The
|
|
47
|
+
* synchronous commit listener updates it on the Session line, so code running on the line reads exactly the committed
|
|
48
|
+
* state from it.
|
|
26
49
|
*
|
|
27
50
|
* Tasks and conversations form one ownership tree (spec §5.5): a task's parent is its owner task, or its conversation;
|
|
28
51
|
* a conversation's parent is its owner task, if any. Walks up that tree decide what reaches one task: a cancelling
|
|
@@ -35,11 +58,22 @@ function parentNode(node) {
|
|
|
35
58
|
* later below a dropped node finds it unknown and loads its chain from storage again.
|
|
36
59
|
*
|
|
37
60
|
* Invariant: every task transition is decided and written by one callback serialized on the Session line. That covers
|
|
38
|
-
* reservation, marks, runtime commits, finalization, and the
|
|
39
|
-
*
|
|
40
|
-
*
|
|
61
|
+
* reservation, marks, runtime commits, finalization, and the step before each phase, which applies the precedence rules
|
|
62
|
+
* and writes a fault or handover. Handlers and joins run off the line. An invocation ends inside the step that decides
|
|
63
|
+
* its end, so a runtime commit it queued either lands before that decision or is rejected.
|
|
64
|
+
*
|
|
65
|
+
* Decisions stay serialized; their storage overlaps (group commit). The Session stores the commits of several jobs in
|
|
66
|
+
* one call while the line runs on, and this memory changes only as each commit is published. So every decision reads
|
|
67
|
+
* it in a decision region (`#region()`): synchronous, recording the keys it reads (a task's record, the live work below
|
|
68
|
+
* a node, the cancellation intent set), staging nothing; when one of them has an unpublished writer, the region waits
|
|
69
|
+
* for its publication and runs again, and only a region that read nothing unpublished is applied. Every commit writes
|
|
70
|
+
* the memberships its task changes alter (`membershipKeys()`), so a walk that a pending commit would change waits.
|
|
41
71
|
*/
|
|
42
72
|
export class TaskScheduler {
|
|
73
|
+
#telemetry;
|
|
74
|
+
#activeMetric;
|
|
75
|
+
#durationMetric;
|
|
76
|
+
#resumed = new Set();
|
|
43
77
|
#session;
|
|
44
78
|
#storage;
|
|
45
79
|
#registry;
|
|
@@ -48,8 +82,12 @@ export class TaskScheduler {
|
|
|
48
82
|
#settings;
|
|
49
83
|
#env;
|
|
50
84
|
#now;
|
|
85
|
+
#onWake;
|
|
86
|
+
#timers;
|
|
87
|
+
#writeAtClose;
|
|
51
88
|
#report;
|
|
52
89
|
#settleOutcome;
|
|
90
|
+
#limitsRestarts;
|
|
53
91
|
#withdrawInputs;
|
|
54
92
|
#conversation;
|
|
55
93
|
#context;
|
|
@@ -60,7 +98,21 @@ export class TaskScheduler {
|
|
|
60
98
|
* first idle check after `settings.contextRetentionMs` of idleness, and at close.
|
|
61
99
|
*/
|
|
62
100
|
#contexts = new Map();
|
|
63
|
-
/**
|
|
101
|
+
/**
|
|
102
|
+
* The idle kept contexts by `idleSince`, oldest first: the expiry order, so neither an expiry nor finding the next one
|
|
103
|
+
* scans `#contexts`. Kept sorted also when the Harness clock goes back (see `#markIdle`).
|
|
104
|
+
*/
|
|
105
|
+
#idleOrder = new Map();
|
|
106
|
+
/** At least the newest `idleSince` in `#idleOrder` (drops do not lower it): an older one added means the clock went back. */
|
|
107
|
+
#idleNewest = Number.NEGATIVE_INFINITY;
|
|
108
|
+
/**
|
|
109
|
+
* A chain load resolved what `#idle()` treats as unknown for a live task whose chain is not loaded, which can make a
|
|
110
|
+
* scope idle without a task change below it; the next idle check then covers every scope.
|
|
111
|
+
*/
|
|
112
|
+
#unknownsChanged = false;
|
|
113
|
+
/** Idle checks that covered every scope (`idleState`, a test diagnostic). */
|
|
114
|
+
#fullIdleChecks = 0;
|
|
115
|
+
/** Background timer for the earliest expiry of an idle context, unless the host declined it (see `#scheduleExpiry`). */
|
|
64
116
|
#expiry;
|
|
65
117
|
#invocations = new Map();
|
|
66
118
|
#taskWaiters = new Waiters();
|
|
@@ -103,15 +155,39 @@ export class TaskScheduler {
|
|
|
103
155
|
* them with reason `restart`, so nothing of them or below them runs a phase again.
|
|
104
156
|
*/
|
|
105
157
|
#abandoned = new Set();
|
|
158
|
+
/** Sleeping live tasks, due or not: the candidates of `#wakeDue()`. */
|
|
159
|
+
#sleeping = new Set();
|
|
160
|
+
/** Timer for the earliest sleep, without `onWake` while scheduling runs. */
|
|
161
|
+
#timer;
|
|
162
|
+
/** Last value given to `onWake`, so an unchanged one is not repeated. */
|
|
163
|
+
#lastWake;
|
|
164
|
+
#wakeScheduled = false;
|
|
165
|
+
#blockedListeners = new Set();
|
|
166
|
+
/** A registry snapshot that threw for the task graph was reported; it is read per record, so only once. */
|
|
167
|
+
#snapshotReported = false;
|
|
106
168
|
#reconcileScheduled = false;
|
|
169
|
+
/** Reconcile passes underway with work (they may overlap): their commits finalize held outcomes and mark cascades. */
|
|
170
|
+
#reconcilingWork = 0;
|
|
107
171
|
#sweepScheduled = false;
|
|
172
|
+
/**
|
|
173
|
+
* Nodes chain loads installed for jobs whose commits are not published yet, with how many such jobs: the sweep keeps
|
|
174
|
+
* them, as work a pending commit creates below them needs them.
|
|
175
|
+
*/
|
|
176
|
+
#pins = new Map();
|
|
177
|
+
/** Keys the decision region underway reads (`#region()`); set only while one runs. */
|
|
178
|
+
#reading;
|
|
108
179
|
#cascadePending = false;
|
|
109
180
|
#unsubscribeRegistry = () => { };
|
|
110
181
|
#enabled = false;
|
|
182
|
+
#readOnly;
|
|
111
183
|
#closing = false;
|
|
112
184
|
#dirty = false;
|
|
113
185
|
#draining = false;
|
|
114
186
|
constructor(options) {
|
|
187
|
+
this.#telemetry = options.telemetry ?? NOOP_TELEMETRY_CONTEXT;
|
|
188
|
+
const metrics = createTypedMetrics(this.#telemetry.metrics ?? NOOP_TELEMETRY_METRICS, [DURABLE_TELEMETRY_SCHEMA]);
|
|
189
|
+
this.#activeMetric = metrics.upDownCounter("pi.durable.tasks.active");
|
|
190
|
+
this.#durationMetric = metrics.histogram("pi.durable.phase.duration");
|
|
115
191
|
this.#session = options.session;
|
|
116
192
|
this.#storage = options.storage;
|
|
117
193
|
this.#registry = options.registry;
|
|
@@ -120,18 +196,31 @@ export class TaskScheduler {
|
|
|
120
196
|
this.#settings = options.settings;
|
|
121
197
|
this.#env = options.env;
|
|
122
198
|
this.#now = options.now;
|
|
199
|
+
this.#onWake = options.onWake;
|
|
200
|
+
this.#timers = options.timers;
|
|
201
|
+
this.#writeAtClose = options.writeAtClose;
|
|
123
202
|
this.#report = options.report;
|
|
124
203
|
this.#settleOutcome = options.settleOutcome;
|
|
204
|
+
this.#limitsRestarts = options.limitsRestarts;
|
|
125
205
|
this.#withdrawInputs = options.withdrawInputs;
|
|
126
206
|
this.#conversation = options.conversation;
|
|
127
207
|
this.#context = options.context;
|
|
208
|
+
this.#readOnly = options.readOnly === true;
|
|
128
209
|
}
|
|
129
|
-
/**
|
|
210
|
+
/**
|
|
211
|
+
* Load live tasks and change surviving `running` tasks back to `pending`, counting the restart. Dispatches nothing;
|
|
212
|
+
* a task cut off `settings.restartLimit` times ends when it is next reserved. Read-only, it only loads them: a
|
|
213
|
+
* `running` task stays as committed, running in another process.
|
|
214
|
+
*/
|
|
130
215
|
async open(context) {
|
|
131
|
-
this.#session.
|
|
216
|
+
this.#session.followCommits((publication) => this.#observe(publication));
|
|
132
217
|
this.#session.subscribeClose(() => this.#seal());
|
|
133
|
-
this.#unsubscribeRegistry = this.#registry.subscribe(() =>
|
|
218
|
+
this.#unsubscribeRegistry = this.#registry.subscribe(() => {
|
|
219
|
+
this.#notifyBlocked();
|
|
220
|
+
this.#kick();
|
|
221
|
+
});
|
|
134
222
|
await this.#session.commitWith(async (tx) => {
|
|
223
|
+
await this.#published();
|
|
135
224
|
// Every table read before the first write.
|
|
136
225
|
const scans = [];
|
|
137
226
|
for (const status of LIVE_STATUSES) {
|
|
@@ -140,10 +229,15 @@ export class TaskScheduler {
|
|
|
140
229
|
for (const records of scans) {
|
|
141
230
|
for (const record of records) {
|
|
142
231
|
this.#track(undefined, record);
|
|
232
|
+
this.#resumed.add(record.id);
|
|
233
|
+
if (this.#readOnly)
|
|
234
|
+
continue;
|
|
143
235
|
if (record.abandonOnRestart === true)
|
|
144
236
|
this.#abandoned.add(record.id);
|
|
145
237
|
if (record.state.status === "running") {
|
|
146
|
-
|
|
238
|
+
const restarts = (record.restarts ?? 0) + 1;
|
|
239
|
+
const pending = withState(record, { status: "pending", checkpoint: record.state.checkpoint });
|
|
240
|
+
tx.setTask({ ...pending, restarts });
|
|
147
241
|
}
|
|
148
242
|
if (record.state.status === "waiting" && record.state.policy === "failFast") {
|
|
149
243
|
this.#failFastChecks.add(record.id);
|
|
@@ -151,23 +245,133 @@ export class TaskScheduler {
|
|
|
151
245
|
}
|
|
152
246
|
}
|
|
153
247
|
}, context);
|
|
248
|
+
if (this.#readOnly)
|
|
249
|
+
return;
|
|
154
250
|
// Derive abort marks a crash left unapplied below cancelled owners, and finalize held outcomes.
|
|
155
251
|
this.#cascadePending = true;
|
|
156
252
|
this.#scheduleReconcile();
|
|
253
|
+
this.#scheduleWake();
|
|
157
254
|
}
|
|
158
|
-
/**
|
|
255
|
+
/**
|
|
256
|
+
* Enable scheduling, and make sleeping tasks that are due runnable: a host wakeup, such as an alarm, calls this.
|
|
257
|
+
* Idempotent; the kick does nothing once closing.
|
|
258
|
+
*/
|
|
159
259
|
resume() {
|
|
260
|
+
if (this.#readOnly)
|
|
261
|
+
throw new ReadOnlySession("running tasks");
|
|
160
262
|
this.#enabled = true;
|
|
263
|
+
this.#wakeDue();
|
|
161
264
|
this.#kick();
|
|
162
265
|
}
|
|
163
266
|
/**
|
|
164
|
-
*
|
|
165
|
-
*
|
|
267
|
+
* Why a live task cannot run here, if it cannot: its definition is missing, too old, or failed to migrate. A running
|
|
268
|
+
* task, an abort-marked one, and one holding its outcome are never blocked.
|
|
269
|
+
*/
|
|
270
|
+
blockedReason(record) {
|
|
271
|
+
const status = record.status;
|
|
272
|
+
if (status === "running" || status === "completing" || status === "terminal")
|
|
273
|
+
return undefined;
|
|
274
|
+
// An abort request runs its handler or orphans the task; a `restart` mark waits for the definition, blocked.
|
|
275
|
+
if (record.abortRequested && record.abortReason === undefined)
|
|
276
|
+
return undefined;
|
|
277
|
+
let snapshot;
|
|
278
|
+
try {
|
|
279
|
+
snapshot = this.#registry.snapshot();
|
|
280
|
+
}
|
|
281
|
+
catch (error) {
|
|
282
|
+
if (!this.#snapshotReported)
|
|
283
|
+
this.#report(error, REGISTRY);
|
|
284
|
+
this.#snapshotReported = true;
|
|
285
|
+
return undefined;
|
|
286
|
+
}
|
|
287
|
+
const fit = this.#fit(record, snapshot.task(record.kind));
|
|
288
|
+
if ("reason" in fit)
|
|
289
|
+
return fit.reason;
|
|
290
|
+
return fit.migrates && fit.task.definition.migrate === undefined ? "migration_failed" : undefined;
|
|
291
|
+
}
|
|
292
|
+
/** Call `listener` when blocked reasons may have changed without a commit: a registry change or a failed migration. */
|
|
293
|
+
subscribeBlocked(listener) {
|
|
294
|
+
this.#blockedListeners.add(listener);
|
|
295
|
+
return () => this.#blockedListeners.delete(listener);
|
|
296
|
+
}
|
|
297
|
+
/**
|
|
298
|
+
* When the Session next needs a process: `active` while some task can run now or is running, `at` for the earliest
|
|
299
|
+
* sleep not yet due. Synchronous, from memory; a task whose chain is not loaded yet counts as it would alone.
|
|
300
|
+
*/
|
|
301
|
+
wake() {
|
|
302
|
+
// A read-only Harness runs nothing, so it never needs a process.
|
|
303
|
+
if (this.#readOnly)
|
|
304
|
+
return { active: false };
|
|
305
|
+
const now = this.#now();
|
|
306
|
+
// A reconcile pass with work to do is work too: it finalizes held outcomes and marks cascades, at open as
|
|
307
|
+
// recovery. A process that dies before it lands must be woken again. A pass with nothing to do is not.
|
|
308
|
+
const active = this.#invocations.size > 0 || this.#reconcilingWork > 0 || this.#reconcileWork() || this.#runsNow(now);
|
|
309
|
+
// The earliest sleep not yet due: only `#sleeping` holds sleeping tasks, so no pass visits every live task.
|
|
310
|
+
let at;
|
|
311
|
+
for (const id of this.#sleeping) {
|
|
312
|
+
const record = this.#live.get(id);
|
|
313
|
+
const state = record?.state;
|
|
314
|
+
if (state?.status === "sleeping" && !record.abortRequested && state.until > now) {
|
|
315
|
+
at = at === undefined ? state.until : Math.min(at, state.until);
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
return at === undefined ? { active } : { active, at };
|
|
319
|
+
}
|
|
320
|
+
/** Whether a live task can run now: the scan `wake()` needs only while no invocation runs. */
|
|
321
|
+
#runsNow(now) {
|
|
322
|
+
let snapshot;
|
|
323
|
+
let snapshotFailed = false;
|
|
324
|
+
for (const record of this.#live.values()) {
|
|
325
|
+
const state = record.state;
|
|
326
|
+
if (state.status === "completing")
|
|
327
|
+
continue;
|
|
328
|
+
if (state.status === "waiting" && !record.abortRequested && state.on.some((id) => this.#live.has(id)))
|
|
329
|
+
continue;
|
|
330
|
+
if (state.status === "sleeping" && !record.abortRequested && state.until > now)
|
|
331
|
+
continue;
|
|
332
|
+
// An abort invocation waits for the live work below it, which counts itself.
|
|
333
|
+
if (record.abortRequested && this.#waitingOn(record, true).length > 0)
|
|
334
|
+
continue;
|
|
335
|
+
// A task no definition can take waits for a registry change, except an abort-marked one, which is orphaned.
|
|
336
|
+
if (!record.abortRequested || record.abortReason !== undefined) {
|
|
337
|
+
if (snapshot === undefined && !snapshotFailed) {
|
|
338
|
+
try {
|
|
339
|
+
snapshot = this.#registry.snapshot();
|
|
340
|
+
}
|
|
341
|
+
catch (error) {
|
|
342
|
+
snapshotFailed = true;
|
|
343
|
+
this.#report(error, REGISTRY);
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
if (snapshot !== undefined && "reason" in this.#fit(record, snapshot.task(record.kind)))
|
|
347
|
+
continue;
|
|
348
|
+
}
|
|
349
|
+
return true;
|
|
350
|
+
}
|
|
351
|
+
return false;
|
|
352
|
+
}
|
|
353
|
+
/**
|
|
354
|
+
* Signal every invocation and wait for them, then write the tasks they ran back as `pending` (`writeAtClose`), as a
|
|
355
|
+
* close is not a crash. Close calls this after every close listener has run, so waits, watches, and streams a task
|
|
356
|
+
* holds end with the Session's reason, not as cancelled by its signal.
|
|
166
357
|
*/
|
|
167
358
|
async join() {
|
|
168
|
-
|
|
359
|
+
const stopped = [...this.#invocations.values()];
|
|
360
|
+
for (const invocation of stopped)
|
|
169
361
|
invocation.controller.abort();
|
|
170
|
-
await Promise.allSettled(
|
|
362
|
+
await Promise.allSettled(stopped.map((invocation) => invocation.done));
|
|
363
|
+
if (stopped.length === 0)
|
|
364
|
+
return;
|
|
365
|
+
// A close is not a crash: a task it stopped is `pending` again, so the next open counts no restart for it. Read
|
|
366
|
+
// on the line after commits admitted before the seal, such as an abort mark, so the write keeps them. A failure
|
|
367
|
+
// leaves them `running`, counted as if cut off; the Storage guard fails the Session and reports it.
|
|
368
|
+
await this.#writeAtClose(() => stopped.flatMap((invocation) => {
|
|
369
|
+
const record = this.#live.get(invocation.taskId);
|
|
370
|
+
if (record?.state.status !== "running")
|
|
371
|
+
return [];
|
|
372
|
+
const value = withState(record, { status: "pending", checkpoint: record.state.checkpoint });
|
|
373
|
+
return [{ type: "task", value }];
|
|
374
|
+
})).catch(() => { });
|
|
171
375
|
}
|
|
172
376
|
/**
|
|
173
377
|
* Commit the abort mark, or settle a task that no registered definition can take as `orphaned` when nothing it owns
|
|
@@ -189,12 +393,14 @@ export class TaskScheduler {
|
|
|
189
393
|
return { result: "marked" };
|
|
190
394
|
if (invocation === undefined && current.state.status !== "completing") {
|
|
191
395
|
await this.#loadScopes(false);
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
396
|
+
const owns = await this.#region(() => this.#hasOwnedLive(id), this.#reloadScopes);
|
|
397
|
+
// Resolved once that decision stands, as it may run `migrate`.
|
|
398
|
+
const resolution = owns
|
|
399
|
+
? undefined
|
|
400
|
+
: this.#resolve(current, this.#registry.snapshot());
|
|
401
|
+
if (resolution?.kind === "blocked") {
|
|
402
|
+
await this.#terminate(tx, current, { status: "orphaned", reason: resolution.reason });
|
|
403
|
+
return { result: "marked" };
|
|
198
404
|
}
|
|
199
405
|
}
|
|
200
406
|
if (!current.abortRequested || current.abortReason !== undefined)
|
|
@@ -210,12 +416,13 @@ export class TaskScheduler {
|
|
|
210
416
|
}
|
|
211
417
|
async waitForTask(id, context) {
|
|
212
418
|
// Check and register on the line so no terminal publication falls between them.
|
|
213
|
-
const found = await this.#session.readOnLine(async () => {
|
|
419
|
+
const found = await this.#session.readOnLine(async (storage) => {
|
|
214
420
|
if (this.#closing)
|
|
215
421
|
throw closedError(this.#session);
|
|
216
|
-
if (this.#
|
|
422
|
+
if (await this.#region(() => this.#liveRecord(id) !== undefined)) {
|
|
217
423
|
return { promise: this.#taskWaiters.add(id, context) };
|
|
218
|
-
|
|
424
|
+
}
|
|
425
|
+
const record = await storage.task(id, context);
|
|
219
426
|
if (record === undefined)
|
|
220
427
|
throw new Error(`Task ${id} does not exist`);
|
|
221
428
|
return { promise: Promise.resolve(record) };
|
|
@@ -241,6 +448,7 @@ export class TaskScheduler {
|
|
|
241
448
|
*/
|
|
242
449
|
async abortConversation(conversationId, background, context) {
|
|
243
450
|
const reached = await this.#session.commitWith(async (tx) => {
|
|
451
|
+
await this.#published();
|
|
244
452
|
const queued = await this.#loadScopes(true);
|
|
245
453
|
const scope = { conversation: conversationId };
|
|
246
454
|
const reached = [];
|
|
@@ -263,11 +471,100 @@ export class TaskScheduler {
|
|
|
263
471
|
await this.waitForTask(id, context);
|
|
264
472
|
await this.waitForIdle(conversationId, context);
|
|
265
473
|
}
|
|
474
|
+
/**
|
|
475
|
+
* The ownership memberships a commit's task and conversation changes write (group commit §4.2), at its admission on
|
|
476
|
+
* the line: for each task created or made terminal, every node from its parent up to and including the nearest live
|
|
477
|
+
* task, or the root conversation, and then the roots too, conservatively, as a pending end may unlist the root a
|
|
478
|
+
* creation finds listed; for each conversation created under a task, that task; and the intent set when
|
|
479
|
+
* a change sets, upgrades, or clears a mark or a held or ended outcome other than `completed`. Loads the chains it
|
|
480
|
+
* walks; the nodes it installs stay until the commit is published.
|
|
481
|
+
*/
|
|
482
|
+
membershipKeys(tx) {
|
|
483
|
+
const tasks = tx.stagedTasks();
|
|
484
|
+
const conversations = tx.stagedConversations();
|
|
485
|
+
if (tasks.length === 0 && conversations.every((record) => record.owner === undefined))
|
|
486
|
+
return NO_KEYS;
|
|
487
|
+
return this.#membershipKeys(tx, tasks, conversations);
|
|
488
|
+
}
|
|
489
|
+
async #membershipKeys(tx, tasks, conversations) {
|
|
490
|
+
const keys = new Set();
|
|
491
|
+
const overlay = overlayOf(tx);
|
|
492
|
+
const created = new Set(tx.createdTaskIds());
|
|
493
|
+
const reached = new Map();
|
|
494
|
+
for (const record of tasks) {
|
|
495
|
+
if (intentChanges(this.#live.get(record.id), record))
|
|
496
|
+
keys.add(INTENT_KEY);
|
|
497
|
+
if (!created.has(record.id) && record.state.status !== "terminal")
|
|
498
|
+
continue;
|
|
499
|
+
await this.#loadChain(parentOf(record), overlay);
|
|
500
|
+
if ((await this.#chainKeys(parentNode(record), overlay, keys, reached)) !== undefined)
|
|
501
|
+
keys.add(ROOTS_KEY);
|
|
502
|
+
}
|
|
503
|
+
for (const conversation of conversations) {
|
|
504
|
+
const owner = conversation.owner?.taskId;
|
|
505
|
+
if (owner === undefined)
|
|
506
|
+
continue;
|
|
507
|
+
await this.#loadChain({ task: owner }, overlay);
|
|
508
|
+
await this.#chainKeys(owner, overlay, keys, reached);
|
|
509
|
+
}
|
|
510
|
+
return [...keys];
|
|
511
|
+
}
|
|
512
|
+
/**
|
|
513
|
+
* Add the memberships from `start` up to and including the nearest live task, or the root conversation, which it
|
|
514
|
+
* returns. `reached` keeps, per node walked, where its walk ended, so a chain several tasks of a commit share is
|
|
515
|
+
* walked once. Whether an ancestor is live can change under a pending commit, so it is read after a touch of the
|
|
516
|
+
* ancestor's record: a pending end must not stop the chain early. Ownership fields are immutable, so not keyed.
|
|
517
|
+
*/
|
|
518
|
+
async #chainKeys(start, overlay, keys, reached) {
|
|
519
|
+
const walked = [];
|
|
520
|
+
let root;
|
|
521
|
+
let node = start;
|
|
522
|
+
while (node !== undefined && node !== null) {
|
|
523
|
+
const key = ownedKey(node);
|
|
524
|
+
// Walked up from here already, for another task of the commit.
|
|
525
|
+
if (reached.has(key)) {
|
|
526
|
+
root = reached.get(key);
|
|
527
|
+
break;
|
|
528
|
+
}
|
|
529
|
+
keys.add(key);
|
|
530
|
+
walked.push(key);
|
|
531
|
+
if (typeof node === "string") {
|
|
532
|
+
const conversation = conversationOf(node);
|
|
533
|
+
node = this.#edge(conversation, overlay);
|
|
534
|
+
if (node === null)
|
|
535
|
+
root = conversation;
|
|
536
|
+
continue;
|
|
537
|
+
}
|
|
538
|
+
let record = overlay.tasks.get(node);
|
|
539
|
+
if (record === undefined) {
|
|
540
|
+
const gate = this.#session.touch(taskKey(node));
|
|
541
|
+
if (gate !== undefined) {
|
|
542
|
+
await gate;
|
|
543
|
+
// It may have ended meanwhile, and its node been swept: load the chain above it again.
|
|
544
|
+
await this.#loadChain({ task: node }, overlay);
|
|
545
|
+
}
|
|
546
|
+
record = this.#live.get(node);
|
|
547
|
+
}
|
|
548
|
+
if (record !== undefined && record.state.status !== "terminal")
|
|
549
|
+
break;
|
|
550
|
+
const above = record ?? this.#settled.get(node);
|
|
551
|
+
node = above === undefined ? undefined : parentNode(above);
|
|
552
|
+
}
|
|
553
|
+
for (const key of walked)
|
|
554
|
+
reached.set(key, root);
|
|
555
|
+
return root;
|
|
556
|
+
}
|
|
266
557
|
// ─── Scheduling ────────────────────────────────────────────────────────
|
|
267
558
|
#observe(publication) {
|
|
559
|
+
// A commit held in memory changes documents only.
|
|
560
|
+
if (publication.kind === "memory")
|
|
561
|
+
return;
|
|
562
|
+
this.#advanceContexts(publication);
|
|
268
563
|
const updated = [];
|
|
269
564
|
const failed = [];
|
|
270
565
|
let changed = false;
|
|
566
|
+
// The conversations whose idleness the commit's task changes can change; undefined when a chain is not known.
|
|
567
|
+
let dirty = new Set();
|
|
271
568
|
// Edges first, so the tasks a commit creates in a new conversation hang from it.
|
|
272
569
|
for (const change of publication.changes) {
|
|
273
570
|
if (change.type === "conversation" && !this.#edges.has(change.value.id)) {
|
|
@@ -283,6 +580,11 @@ export class TaskScheduler {
|
|
|
283
580
|
if (failedOutcome(record) && (previous === undefined || !failedOutcome(previous)))
|
|
284
581
|
failed.push(record.id);
|
|
285
582
|
this.#track(previous, record);
|
|
583
|
+
// Idleness counts live tasks, so only a task that became live or ended can change it. Collected before `#sweep()`,
|
|
584
|
+
// which may forget the chain of a task that ended.
|
|
585
|
+
const liveness = previous === undefined || record.state.status === "terminal";
|
|
586
|
+
if (liveness && dirty !== undefined && !this.#addScopesAbove(record, dirty))
|
|
587
|
+
dirty = undefined;
|
|
286
588
|
if (record.state.status === "terminal") {
|
|
287
589
|
this.#failedMigrations.delete(record.id);
|
|
288
590
|
this.#failFastChecks.delete(record.id);
|
|
@@ -293,10 +595,11 @@ export class TaskScheduler {
|
|
|
293
595
|
}
|
|
294
596
|
if (record.abortRequested && previous?.abortRequested !== true) {
|
|
295
597
|
this.#cascadePending = true;
|
|
296
|
-
// Signal a run invocation of the newly marked task; its next step ends it.
|
|
598
|
+
// Signal a run invocation of the newly marked task; its next step ends it. Abort listeners are task code, which
|
|
599
|
+
// never runs inside a publication: the signal follows in a microtask, after host commit listeners.
|
|
297
600
|
const invocation = this.#invocations.get(record.id);
|
|
298
601
|
if (invocation?.mode === "run")
|
|
299
|
-
invocation.controller.abort();
|
|
602
|
+
queueMicrotask(() => invocation.controller.abort());
|
|
300
603
|
}
|
|
301
604
|
// A request replacing a `restart` mark upgrades the marks of the work below.
|
|
302
605
|
if (previous?.abortReason !== undefined && record.abortReason === undefined)
|
|
@@ -350,9 +653,60 @@ export class TaskScheduler {
|
|
|
350
653
|
this.#sweep();
|
|
351
654
|
if (!changed)
|
|
352
655
|
return;
|
|
353
|
-
this.#settleIdle();
|
|
656
|
+
this.#settleIdle(dirty);
|
|
354
657
|
this.#kick();
|
|
355
658
|
}
|
|
659
|
+
/**
|
|
660
|
+
* Add the conversations above `record` to `scopes`: those whose idleness a change of it can change. False when a node
|
|
661
|
+
* on the way up is not known. A conversation already in `scopes` has everything above it there too.
|
|
662
|
+
*/
|
|
663
|
+
#addScopesAbove(record, scopes) {
|
|
664
|
+
for (const step of this.#above(parentOf(record))) {
|
|
665
|
+
if ("unknown" in step)
|
|
666
|
+
return false;
|
|
667
|
+
if (!("conversation" in step))
|
|
668
|
+
continue;
|
|
669
|
+
if (scopes.has(step.conversation))
|
|
670
|
+
return true;
|
|
671
|
+
scopes.add(step.conversation);
|
|
672
|
+
}
|
|
673
|
+
return true;
|
|
674
|
+
}
|
|
675
|
+
/**
|
|
676
|
+
* Extend each kept context range with the entries a stored commit appended to its conversation, so a later read
|
|
677
|
+
* needs no Storage scan. Entries are published in ID order.
|
|
678
|
+
*/
|
|
679
|
+
#advanceContexts(publication) {
|
|
680
|
+
if (this.#contexts.size === 0)
|
|
681
|
+
return;
|
|
682
|
+
let appended;
|
|
683
|
+
for (const change of publication.changes) {
|
|
684
|
+
if (change.type !== "entry" || !this.#contexts.has(change.value.conversationId))
|
|
685
|
+
continue;
|
|
686
|
+
appended ??= new Map();
|
|
687
|
+
const entries = appended.get(change.value.conversationId);
|
|
688
|
+
if (entries === undefined)
|
|
689
|
+
appended.set(change.value.conversationId, [change.value]);
|
|
690
|
+
else
|
|
691
|
+
entries.push(change.value);
|
|
692
|
+
}
|
|
693
|
+
if (appended === undefined)
|
|
694
|
+
return;
|
|
695
|
+
for (const [conversationId, entries] of appended) {
|
|
696
|
+
const kept = this.#contexts.get(conversationId);
|
|
697
|
+
// An idle range past its retention is dropped here too: entry-only commits do not run `#settleIdle()`, and a
|
|
698
|
+
// host may decline the expiry timer. `expiresAt` was fixed when it went idle, so no settings are read here.
|
|
699
|
+
if (kept.idleSince !== undefined && kept.expiresAt !== undefined && kept.expiresAt <= this.#now()) {
|
|
700
|
+
this.#dropContext(conversationId);
|
|
701
|
+
continue;
|
|
702
|
+
}
|
|
703
|
+
const range = advanceRange(kept.range, entries);
|
|
704
|
+
if (range === undefined)
|
|
705
|
+
this.#dropContext(conversationId);
|
|
706
|
+
else
|
|
707
|
+
kept.range = range;
|
|
708
|
+
}
|
|
709
|
+
}
|
|
356
710
|
/**
|
|
357
711
|
* `settings.contextRetentionMs`, or 0 when the host's settings throw. Kept contexts are only a cache, so dropping them
|
|
358
712
|
* is safe, while a throw here would escape commit listeners, reconciliation, and the expiry timer.
|
|
@@ -362,69 +716,116 @@ export class TaskScheduler {
|
|
|
362
716
|
return this.#settings().contextRetentionMs;
|
|
363
717
|
}
|
|
364
718
|
catch (error) {
|
|
365
|
-
this.#report(error);
|
|
719
|
+
this.#report(error, SETTINGS);
|
|
366
720
|
return 0;
|
|
367
721
|
}
|
|
368
722
|
}
|
|
369
|
-
/**
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
723
|
+
/**
|
|
724
|
+
* After task changes below the conversations in `scopes` (every conversation when undefined): resolve their idle
|
|
725
|
+
* waiters and the Harness-wide ones, drop the kept contexts whose idle period passed the retention, then start or end
|
|
726
|
+
* the idle periods of the kept contexts in `scopes`. A task change can change the idleness only of the conversations
|
|
727
|
+
* above it, so no check visits every conversation.
|
|
728
|
+
*/
|
|
729
|
+
#settleIdle(scopes) {
|
|
730
|
+
const all = scopes === undefined || this.#unknownsChanged;
|
|
731
|
+
this.#unknownsChanged = false;
|
|
732
|
+
if (all)
|
|
733
|
+
this.#fullIdleChecks++;
|
|
734
|
+
this.#resolveIdleWaiters(all ? undefined : scopes);
|
|
375
735
|
const now = this.#now();
|
|
376
736
|
const retention = this.#contextRetentionMs();
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
737
|
+
// Expired first: a context past its retention is dropped even when the same commit makes it busy again.
|
|
738
|
+
this.#expireContexts(now, retention);
|
|
739
|
+
for (const conversationId of all ? [...this.#contexts.keys()] : scopes) {
|
|
740
|
+
const kept = this.#contexts.get(conversationId);
|
|
741
|
+
if (kept === undefined)
|
|
742
|
+
continue;
|
|
743
|
+
if (!this.#idle(conversationId)) {
|
|
744
|
+
if (kept.idleSince === undefined)
|
|
745
|
+
continue;
|
|
382
746
|
kept.idleSince = undefined;
|
|
747
|
+
this.#idleOrder.delete(conversationId);
|
|
383
748
|
}
|
|
384
|
-
else if (retention
|
|
385
|
-
|
|
749
|
+
else if (retention <= 0) {
|
|
750
|
+
this.#dropContext(conversationId);
|
|
386
751
|
}
|
|
387
|
-
else {
|
|
388
|
-
|
|
752
|
+
else if (kept.idleSince === undefined) {
|
|
753
|
+
kept.idleSince = now;
|
|
754
|
+
kept.expiresAt = now + retention;
|
|
755
|
+
this.#markIdle(conversationId, now);
|
|
389
756
|
}
|
|
390
757
|
}
|
|
391
758
|
this.#scheduleExpiry();
|
|
392
759
|
}
|
|
760
|
+
/** Resolve the idle waiters of the conversations in `scopes` (all when undefined) and the Harness-wide ones. */
|
|
761
|
+
#resolveIdleWaiters(scopes) {
|
|
762
|
+
for (const conversationId of this.#idleWaiters.keys()) {
|
|
763
|
+
if (conversationId !== undefined && scopes !== undefined && !scopes.has(conversationId))
|
|
764
|
+
continue;
|
|
765
|
+
if (this.#idle(conversationId))
|
|
766
|
+
this.#idleWaiters.resolve(conversationId);
|
|
767
|
+
}
|
|
768
|
+
}
|
|
393
769
|
/**
|
|
394
|
-
*
|
|
395
|
-
*
|
|
396
|
-
* Durable Object from being evicted, and eviction frees the contexts. There, task changes alone check expiry.
|
|
770
|
+
* Drop the kept contexts idle for at least `retention`, oldest first, with the retention setting of now; with no
|
|
771
|
+
* retention, every idle one, also those whose idle period a clock that went back makes start after now.
|
|
397
772
|
*/
|
|
398
|
-
#
|
|
399
|
-
const
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
at = kept.idleSince + retention;
|
|
404
|
-
}
|
|
773
|
+
#expireContexts(now, retention) {
|
|
774
|
+
for (const [conversationId, idleSince] of this.#idleOrder) {
|
|
775
|
+
if (retention > 0 && now - idleSince < retention)
|
|
776
|
+
return;
|
|
777
|
+
this.#dropContext(conversationId);
|
|
405
778
|
}
|
|
779
|
+
}
|
|
780
|
+
/**
|
|
781
|
+
* Start the idle period of a kept context in the expiry order. An idle period older than the newest one kept, which
|
|
782
|
+
* a clock that went back starts, sorts the order again, and the newest is taken from what is kept then.
|
|
783
|
+
*/
|
|
784
|
+
#markIdle(conversationId, idleSince) {
|
|
785
|
+
this.#idleOrder.delete(conversationId);
|
|
786
|
+
this.#idleOrder.set(conversationId, idleSince);
|
|
787
|
+
if (this.#idleOrder.size === 1 || idleSince >= this.#idleNewest) {
|
|
788
|
+
this.#idleNewest = idleSince;
|
|
789
|
+
return;
|
|
790
|
+
}
|
|
791
|
+
const sorted = [...this.#idleOrder].sort((a, b) => a[1] - b[1]);
|
|
792
|
+
this.#idleOrder.clear();
|
|
793
|
+
for (const [id, since] of sorted)
|
|
794
|
+
this.#idleOrder.set(id, since);
|
|
795
|
+
this.#idleNewest = sorted.at(-1)[1];
|
|
796
|
+
}
|
|
797
|
+
/** Forget a kept context, idle or not, and its place in the expiry order. */
|
|
798
|
+
#dropContext(conversationId) {
|
|
799
|
+
this.#contexts.delete(conversationId);
|
|
800
|
+
this.#idleOrder.delete(conversationId);
|
|
801
|
+
}
|
|
802
|
+
/**
|
|
803
|
+
* Drop the expired contexts when the earliest idle context expires, on a background timer, which never keeps the
|
|
804
|
+
* process alive. Where the host declines it, as the default does in Cloudflare Workers, none is kept: a pending timer
|
|
805
|
+
* could keep a Durable Object from being evicted, and eviction frees the contexts. There, task changes alone check
|
|
806
|
+
* expiry.
|
|
807
|
+
*/
|
|
808
|
+
#scheduleExpiry() {
|
|
809
|
+
const oldest = this.#idleOrder.values().next();
|
|
810
|
+
const at = oldest.done === true ? undefined : oldest.value + this.#contextRetentionMs();
|
|
406
811
|
if (this.#expiry !== undefined && this.#expiry.at === at)
|
|
407
812
|
return;
|
|
408
|
-
|
|
409
|
-
clearTimeout(this.#expiry.timer);
|
|
813
|
+
this.#expiry?.disarm();
|
|
410
814
|
this.#expiry = undefined;
|
|
411
815
|
if (at === undefined || this.#closing)
|
|
412
816
|
return;
|
|
413
|
-
const
|
|
817
|
+
const disarm = this.#timers.background(() => {
|
|
414
818
|
this.#expiry = undefined;
|
|
415
|
-
this.#
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
if (
|
|
419
|
-
|
|
420
|
-
return;
|
|
421
|
-
}
|
|
422
|
-
unref.call(timer);
|
|
423
|
-
this.#expiry = { at, timer };
|
|
819
|
+
this.#expireContexts(this.#now(), this.#contextRetentionMs());
|
|
820
|
+
this.#scheduleExpiry();
|
|
821
|
+
}, at - this.#now());
|
|
822
|
+
if (disarm !== undefined)
|
|
823
|
+
this.#expiry = { at, disarm };
|
|
424
824
|
}
|
|
425
825
|
// ─── Ownership ───────────────────────────────────────────────────────────
|
|
426
826
|
#scheduleReconcile() {
|
|
427
|
-
|
|
827
|
+
// A read-only reconcile could only fail: what it writes is refused, and its own commits fail the Session.
|
|
828
|
+
if (this.#reconcileScheduled || this.#closing || this.#readOnly)
|
|
428
829
|
return;
|
|
429
830
|
this.#reconcileScheduled = true;
|
|
430
831
|
queueMicrotask(() => void this.#reconcile());
|
|
@@ -437,54 +838,25 @@ export class TaskScheduler {
|
|
|
437
838
|
*/
|
|
438
839
|
async #reconcile() {
|
|
439
840
|
this.#reconcileScheduled = false;
|
|
440
|
-
const
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
this.#failFastChecks.clear();
|
|
841
|
+
const work = this.#reconcileWork();
|
|
842
|
+
if (work)
|
|
843
|
+
this.#reconcilingWork++;
|
|
444
844
|
try {
|
|
445
|
-
await this.#session.
|
|
845
|
+
await this.#session.commitInternal(async (tx) => {
|
|
446
846
|
if (this.#closing)
|
|
447
847
|
return;
|
|
848
|
+
// A cascade scans the queued inputs, which waits for every unpublished commit: a request made before the scan
|
|
849
|
+
// is covered by it, and none is published while the job holds the line after it.
|
|
850
|
+
const cascade = this.#cascadePending;
|
|
448
851
|
const queued = await this.#loadScopes(cascade);
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
const marks =
|
|
452
|
-
const mark = (record, reason) => {
|
|
453
|
-
const staged = marks.get(record.id);
|
|
454
|
-
if (staged !== undefined && staged.reason === undefined)
|
|
455
|
-
return;
|
|
456
|
-
if (record.abortRequested && (record.abortReason === undefined || reason !== undefined))
|
|
457
|
-
return;
|
|
458
|
-
marks.set(record.id, { record, reason });
|
|
459
|
-
};
|
|
460
|
-
// Every live foreground task below a live owner with cancellation intent, which is its nearest one: a walk down
|
|
461
|
-
// from each owner with intent does not enter another, whose own walk covers what is below it. Derived again on
|
|
462
|
-
// every pass, as loading edges can reveal a cancelled owner.
|
|
463
|
-
for (const id of this.#intent) {
|
|
464
|
-
const owner = this.#live.get(id);
|
|
465
|
-
// Only an owner abandoned after a restart, and nothing else, passes its reason on.
|
|
466
|
-
const restart = owner.abortReason === "restart" && !failedOutcome(owner);
|
|
467
|
-
for (const { node, live } of this.#walkDown(id, { enter: (below) => !this.#intent.has(below) })) {
|
|
468
|
-
if (live !== undefined && !node.background)
|
|
469
|
-
mark(live, restart ? "restart" : undefined);
|
|
470
|
-
}
|
|
471
|
-
}
|
|
472
|
-
for (const id of checks) {
|
|
473
|
-
const waiter = this.#live.get(id);
|
|
474
|
-
if (waiter?.state.status !== "waiting" || !(await this.#anyFailed(waiter.state.on)))
|
|
475
|
-
continue;
|
|
476
|
-
// Every other live task: the failed one keeps its own outcome.
|
|
477
|
-
for (const member of waiter.state.on) {
|
|
478
|
-
const record = this.#live.get(member);
|
|
479
|
-
if (record !== undefined && !failedOutcome(record))
|
|
480
|
-
mark(record, undefined);
|
|
481
|
-
}
|
|
482
|
-
}
|
|
852
|
+
if (cascade)
|
|
853
|
+
this.#cascadePending = false;
|
|
854
|
+
const marks = await this.#marks();
|
|
483
855
|
for (const { record, reason } of marks.values())
|
|
484
856
|
tx.setTask(withAbortMark(record, reason));
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
857
|
+
const withdrawn = await this.#region(() => queued.filter((id) => this.#belowCancelled({ conversation: id })), this.#reloadScopes);
|
|
858
|
+
for (const id of withdrawn)
|
|
859
|
+
await this.#withdrawInputs(tx, id);
|
|
488
860
|
await this.#finalize(tx);
|
|
489
861
|
}, this.#context);
|
|
490
862
|
}
|
|
@@ -493,59 +865,172 @@ export class TaskScheduler {
|
|
|
493
865
|
// running the pass again, so it fails the Session, which reports it.
|
|
494
866
|
this.#failSession(error);
|
|
495
867
|
}
|
|
496
|
-
|
|
868
|
+
if (work)
|
|
869
|
+
this.#reconcilingWork--;
|
|
870
|
+
this.#scheduleWake();
|
|
871
|
+
// Its commit's task changes were settled at their publication; a waiter that asked for this pass checks again, and
|
|
872
|
+
// what its chain loads resolved can make any scope idle.
|
|
873
|
+
if (this.#unknownsChanged) {
|
|
874
|
+
this.#settleIdle(undefined);
|
|
875
|
+
return;
|
|
876
|
+
}
|
|
877
|
+
this.#resolveIdleWaiters(undefined);
|
|
878
|
+
// A pass checks expiry too, for hosts that decline the expiry timer.
|
|
879
|
+
this.#expireContexts(this.#now(), this.#contextRetentionMs());
|
|
880
|
+
this.#scheduleExpiry();
|
|
497
881
|
}
|
|
498
|
-
/**
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
882
|
+
/**
|
|
883
|
+
* The abort marks committed records imply (spec §5.4, §5.5): every live foreground task below a live owner with
|
|
884
|
+
* cancellation intent, which is its nearest one (a walk down from each owner with intent does not enter another,
|
|
885
|
+
* whose own walk covers what is below it), and every other live task in the `on` of a `failFast` waiter one of whose
|
|
886
|
+
* tasks failed. A cascade from an abandoned owner passes its `restart` reason on; any other intent marks, or upgrades
|
|
887
|
+
* a `restart` mark, as a request. Collected first, so a request wins over a `restart` mark of the same pass. Derived
|
|
888
|
+
* again on every pass, as loading edges can reveal a cancelled owner. Takes the `failFast` checks once its decision
|
|
889
|
+
* stands.
|
|
890
|
+
*/
|
|
891
|
+
async #marks() {
|
|
892
|
+
// Terminal tasks in a checked `on`, read from Storage: each one's outcome is final.
|
|
893
|
+
const ended = new Map();
|
|
894
|
+
for (;;) {
|
|
895
|
+
const plan = await this.#region(() => this.#planMarks(ended), this.#reloadScopes);
|
|
896
|
+
if ("missing" in plan) {
|
|
897
|
+
for (const id of plan.missing)
|
|
898
|
+
ended.set(id, (await this.#session.lineStorage.task(id, this.#context)) ?? null);
|
|
899
|
+
continue;
|
|
900
|
+
}
|
|
901
|
+
for (const id of plan.checked)
|
|
902
|
+
this.#failFastChecks.delete(id);
|
|
903
|
+
return plan.marks;
|
|
504
904
|
}
|
|
505
|
-
|
|
905
|
+
}
|
|
906
|
+
/** `#marks()`' decision region; the `failFast` members it needs from Storage first, when `ended` lacks them. */
|
|
907
|
+
#planMarks(ended) {
|
|
908
|
+
const marks = new Map();
|
|
909
|
+
// A mark rewrites the whole record, so the record it copies is read as a task record, keyed: a commit of the task
|
|
910
|
+
// that is not published yet makes the region wait and run again, rather than stage a mark over a stale checkpoint.
|
|
911
|
+
const mark = (found, reason) => {
|
|
912
|
+
const record = this.#liveRecord(found.id);
|
|
913
|
+
const staged = marks.get(record.id);
|
|
914
|
+
if (staged !== undefined && staged.reason === undefined)
|
|
915
|
+
return;
|
|
916
|
+
if (record.abortRequested && (record.abortReason === undefined || reason !== undefined))
|
|
917
|
+
return;
|
|
918
|
+
marks.set(record.id, { record, reason });
|
|
919
|
+
};
|
|
920
|
+
this.#record(INTENT_KEY);
|
|
921
|
+
for (const id of this.#intent) {
|
|
922
|
+
const owner = this.#liveRecord(id);
|
|
923
|
+
// Only an owner abandoned after a restart, and nothing else, passes its reason on.
|
|
924
|
+
const restart = owner.abortReason === "restart" && !failedOutcome(owner);
|
|
925
|
+
for (const { node, live } of this.#walkDown(id, { enter: (below) => !this.#intent.has(below) })) {
|
|
926
|
+
if (live !== undefined && !node.background)
|
|
927
|
+
mark(live, restart ? "restart" : undefined);
|
|
928
|
+
}
|
|
929
|
+
}
|
|
930
|
+
const checked = [...this.#failFastChecks];
|
|
931
|
+
const missing = [];
|
|
932
|
+
for (const id of checked) {
|
|
933
|
+
const waiter = this.#liveRecord(id);
|
|
934
|
+
if (waiter?.state.status !== "waiting")
|
|
935
|
+
continue;
|
|
936
|
+
let failed = false;
|
|
937
|
+
for (const member of waiter.state.on) {
|
|
938
|
+
const record = this.#liveRecord(member) ?? ended.get(member);
|
|
939
|
+
if (record === undefined)
|
|
940
|
+
missing.push(member);
|
|
941
|
+
else if (record !== null && failedOutcome(record))
|
|
942
|
+
failed = true;
|
|
943
|
+
}
|
|
944
|
+
if (!failed)
|
|
945
|
+
continue;
|
|
946
|
+
// Every other live task: the failed one keeps its own outcome.
|
|
947
|
+
for (const member of waiter.state.on) {
|
|
948
|
+
const record = this.#liveRecord(member);
|
|
949
|
+
if (record !== undefined && !failedOutcome(record))
|
|
950
|
+
mark(record, undefined);
|
|
951
|
+
}
|
|
952
|
+
}
|
|
953
|
+
return missing.length > 0 ? { missing } : { marks, checked };
|
|
954
|
+
}
|
|
955
|
+
/** Whether a reconcile pass would have work: a cascade to derive, held outcomes to check, or `failFast` waits. */
|
|
956
|
+
#reconcileWork() {
|
|
957
|
+
return this.#cascadePending || this.#finalizeChecks.size > 0 || this.#failFastChecks.size > 0;
|
|
506
958
|
}
|
|
507
959
|
/**
|
|
508
960
|
* Write the terminal record of every `completing` task without live ordinary owned work. Finalizing one can free its
|
|
509
961
|
* owner, so this repeats over the commit's candidates until nothing changes. A held scheduler outcome gets its
|
|
510
|
-
* Harness cleanup here.
|
|
962
|
+
* Harness cleanup here. In rounds, one decision region each: the holders the tasks of one round free are checked once,
|
|
963
|
+
* in the next, so a holder of many tasks that end together is walked once, not once per task. The first round takes
|
|
964
|
+
* the `completing` tasks to check once its decision stands.
|
|
511
965
|
*/
|
|
512
966
|
async #finalize(tx) {
|
|
513
|
-
// A worklist: the tasks that started holding, and the holders of work that ended. Finalizing one can free the
|
|
514
|
-
// nearest live task above it, which goes on the list. Tasks finalized here have no live work below them, so walks
|
|
515
|
-
// down skip them.
|
|
516
|
-
// In rounds: the holders the tasks of one round free are checked once, in the next, so a holder of many tasks that
|
|
517
|
-
// end together is walked once, not once per task.
|
|
518
|
-
let round = [...this.#finalizeChecks];
|
|
519
|
-
this.#finalizeChecks.clear();
|
|
520
967
|
const done = new Set();
|
|
968
|
+
// The commit's candidates, kept in step with the terminal records each round stages.
|
|
521
969
|
const overlay = overlayOf(tx);
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
continue;
|
|
530
|
-
const outcome = record.state.outcome;
|
|
531
|
-
const terminal = withState(record, { status: "terminal", outcome });
|
|
970
|
+
let round;
|
|
971
|
+
for (;;) {
|
|
972
|
+
const ids = round;
|
|
973
|
+
const plan = await this.#region(() => this.#planRound(ids ?? [...this.#finalizeChecks], overlay, done), this.#reloadScopes);
|
|
974
|
+
if (ids === undefined)
|
|
975
|
+
this.#finalizeChecks.clear();
|
|
976
|
+
for (const { record, terminal } of plan.finalized) {
|
|
532
977
|
tx.setTask(terminal);
|
|
533
|
-
overlay.tasks.set(id, terminal);
|
|
534
|
-
done.add(id);
|
|
978
|
+
overlay.tasks.set(record.id, terminal);
|
|
979
|
+
done.add(record.id);
|
|
980
|
+
const outcome = record.state.outcome;
|
|
535
981
|
// REMINDER: only the scheduler writes `faulted` and `orphaned` (spec §5.4); their cleanup waits for this commit.
|
|
536
982
|
if (outcome.status === "faulted" || outcome.status === "orphaned") {
|
|
537
983
|
await this.#settleOutcome(tx, record, outcome);
|
|
538
984
|
}
|
|
985
|
+
}
|
|
986
|
+
if (plan.next.length === 0)
|
|
987
|
+
return;
|
|
988
|
+
round = plan.next;
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
/**
|
|
992
|
+
* One finalize round's decision: which of `ids` are `completing` without live ordinary owned work below them, in the
|
|
993
|
+
* commit's candidates with the round's own finalizations applied in order, and the holders above them to check next.
|
|
994
|
+
* Tasks finalized have no live work below them, so walks down skip them. It applies the round's terminal records to
|
|
995
|
+
* `overlay` while it decides and takes them back before it returns, so a run again starts from the same candidates.
|
|
996
|
+
*/
|
|
997
|
+
#planRound(ids, overlay, finalized) {
|
|
998
|
+
const tasks = overlay.tasks;
|
|
999
|
+
// The tasks finalized so far, earlier rounds' and this one's, without copying the former.
|
|
1000
|
+
const round = new Set();
|
|
1001
|
+
const done = { has: (id) => round.has(id) || finalized.has(id) };
|
|
1002
|
+
const ended = [];
|
|
1003
|
+
const next = new Set();
|
|
1004
|
+
try {
|
|
1005
|
+
for (const id of ids) {
|
|
1006
|
+
const staged = tasks.get(id);
|
|
1007
|
+
const record = staged ?? this.#liveRecord(id);
|
|
1008
|
+
if (record?.state.status !== "completing" || done.has(id))
|
|
1009
|
+
continue;
|
|
1010
|
+
if (this.#hasOwnedLive(id, overlay, done))
|
|
1011
|
+
continue;
|
|
1012
|
+
const terminal = withState(record, { status: "terminal", outcome: record.state.outcome });
|
|
1013
|
+
tasks.set(id, terminal);
|
|
1014
|
+
round.add(id);
|
|
1015
|
+
ended.push({ record, terminal, staged: staged !== undefined });
|
|
539
1016
|
const above = this.#nearestLiveAbove(record, overlay);
|
|
540
1017
|
if (above?.state.status === "completing")
|
|
541
1018
|
next.add(above.id);
|
|
542
1019
|
}
|
|
543
|
-
round = [...next];
|
|
544
1020
|
}
|
|
1021
|
+
finally {
|
|
1022
|
+
for (const { record, staged } of ended) {
|
|
1023
|
+
if (staged)
|
|
1024
|
+
tasks.set(record.id, record);
|
|
1025
|
+
else
|
|
1026
|
+
tasks.delete(record.id);
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
1029
|
+
return { finalized: ended, next: [...next] };
|
|
545
1030
|
}
|
|
546
1031
|
/**
|
|
547
1032
|
* Load the owner chains of the tasks in `#unloaded` and, with `queued`, of every conversation with queued inputs,
|
|
548
|
-
* on the Session line; returns the latter. Reads
|
|
1033
|
+
* on the Session line; returns the latter. Reads through the line's Storage, so it may run inside a commit callback.
|
|
549
1034
|
*/
|
|
550
1035
|
async #loadScopes(queued) {
|
|
551
1036
|
for (const id of [...this.#unloaded]) {
|
|
@@ -555,7 +1040,8 @@ export class TaskScheduler {
|
|
|
555
1040
|
}
|
|
556
1041
|
if (!queued)
|
|
557
1042
|
return [];
|
|
558
|
-
|
|
1043
|
+
// A scan, which cannot be keyed: it waits for every unpublished commit.
|
|
1044
|
+
const submissions = await scanAll((cursor) => this.#session.lineStorage.scanSubmissions({ status: "queued" }, SCAN_PAGE_SIZE, cursor, this.#context));
|
|
559
1045
|
const inputs = submissions.filter((submission) => submission.type === "input");
|
|
560
1046
|
const conversations = [...new Set(inputs.map((submission) => submission.conversationId))];
|
|
561
1047
|
for (const id of conversations)
|
|
@@ -565,48 +1051,76 @@ export class TaskScheduler {
|
|
|
565
1051
|
/**
|
|
566
1052
|
* Load the owner edges and task nodes from `start` up to its ownerless root. A known task outside `#unloaded` ends the
|
|
567
1053
|
* walk, as its chain is loaded; the `#unloaded` tasks passed have a loaded chain afterwards. Loading a node can reveal
|
|
568
|
-
* a cancelled owner, so it makes the next reconcile derive cascade marks.
|
|
1054
|
+
* a cancelled owner, so it makes the next reconcile derive cascade marks. The nodes it installs stay until the job's
|
|
1055
|
+
* commit is published (`#pin()`).
|
|
569
1056
|
*/
|
|
570
1057
|
async #loadChain(start, overlay) {
|
|
571
1058
|
const passed = [];
|
|
1059
|
+
let installed = false;
|
|
572
1060
|
let at = start;
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
1061
|
+
try {
|
|
1062
|
+
while (at !== undefined) {
|
|
1063
|
+
if ("task" in at) {
|
|
1064
|
+
const id = at.task;
|
|
1065
|
+
let node = this.#live.get(id) ?? this.#settled.get(id);
|
|
1066
|
+
// Created in this commit: the load from its own record covers what is above it.
|
|
1067
|
+
if (node === undefined && overlay?.tasks.has(id) === true)
|
|
1068
|
+
break;
|
|
1069
|
+
if (node !== undefined && !this.#unloaded.has(id))
|
|
1070
|
+
break;
|
|
1071
|
+
if (node === undefined) {
|
|
1072
|
+
// Through the line's Storage: a task whose creation or end is not published yet waits for it.
|
|
1073
|
+
const record = await this.#session.lineStorage.task(id, this.#context);
|
|
1074
|
+
if (record === undefined)
|
|
1075
|
+
return;
|
|
1076
|
+
node = nodeOf(record);
|
|
1077
|
+
if (record.state.status === "terminal") {
|
|
1078
|
+
this.#settle(id, node);
|
|
1079
|
+
this.#pin(id);
|
|
1080
|
+
installed = true;
|
|
1081
|
+
}
|
|
1082
|
+
// A loaded node can reveal a cancelling owner: the next reconcile derives marks again.
|
|
1083
|
+
this.#scheduleReconcile();
|
|
1084
|
+
}
|
|
1085
|
+
passed.push(id);
|
|
1086
|
+
at = parentOf(node);
|
|
591
1087
|
}
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
1088
|
+
else {
|
|
1089
|
+
let edge = this.#edge(at.conversation, overlay);
|
|
1090
|
+
if (edge === undefined) {
|
|
1091
|
+
// Through the line's Storage: a conversation whose creation is not published yet waits for it, so a missing
|
|
1092
|
+
// record is never mistaken for an ownerless one.
|
|
1093
|
+
const record = await this.#session.lineStorage.conversation(at.conversation, this.#context);
|
|
1094
|
+
edge = record?.owner?.taskId ?? null;
|
|
1095
|
+
this.#setEdge(at.conversation, edge);
|
|
1096
|
+
this.#pin(conversationNode(at.conversation));
|
|
1097
|
+
installed = true;
|
|
1098
|
+
// A loaded node can reveal a cancelling owner: the next reconcile derives marks again.
|
|
1099
|
+
this.#scheduleReconcile();
|
|
1100
|
+
}
|
|
1101
|
+
at = edge === null ? undefined : { task: edge };
|
|
603
1102
|
}
|
|
604
|
-
|
|
1103
|
+
}
|
|
1104
|
+
}
|
|
1105
|
+
finally {
|
|
1106
|
+
// `#idle()` counts a live task whose chain is not loaded as inside every scope it may be in: what this load
|
|
1107
|
+
// resolved, also when it stopped early, can make such a scope idle without a task change below it.
|
|
1108
|
+
if (passed.some((id) => this.#live.has(id)) || (installed && this.#liveUnloaded())) {
|
|
1109
|
+
this.#unknownsChanged = true;
|
|
1110
|
+
// A waiter such a task held is checked again at the pass's end, also when this load was not a reconcile's.
|
|
1111
|
+
this.#scheduleReconcile();
|
|
605
1112
|
}
|
|
606
1113
|
}
|
|
607
1114
|
for (const id of passed)
|
|
608
1115
|
this.#unloaded.delete(id);
|
|
609
1116
|
}
|
|
1117
|
+
/** Whether a live task's chain is not loaded yet: one `#idle()` counts as inside every scope it may be in. */
|
|
1118
|
+
#liveUnloaded() {
|
|
1119
|
+
for (const id of this.#unloaded)
|
|
1120
|
+
if (this.#live.has(id))
|
|
1121
|
+
return true;
|
|
1122
|
+
return false;
|
|
1123
|
+
}
|
|
610
1124
|
#setEdge(conversationId, owner) {
|
|
611
1125
|
this.#edges.set(conversationId, owner);
|
|
612
1126
|
// The conversation now hangs from its owner, or is a root; with nothing below it, it waits for the sweep.
|
|
@@ -653,7 +1167,8 @@ export class TaskScheduler {
|
|
|
653
1167
|
*/
|
|
654
1168
|
#sweep() {
|
|
655
1169
|
for (const node of this.#dropQueue) {
|
|
656
|
-
|
|
1170
|
+
// Queued again once its pins go.
|
|
1171
|
+
if (this.#below.has(node) || this.#pins.has(node))
|
|
657
1172
|
continue;
|
|
658
1173
|
if (typeof node === "string") {
|
|
659
1174
|
this.#edges.delete(conversationOf(node));
|
|
@@ -666,18 +1181,93 @@ export class TaskScheduler {
|
|
|
666
1181
|
}
|
|
667
1182
|
this.#dropQueue.clear();
|
|
668
1183
|
}
|
|
1184
|
+
/**
|
|
1185
|
+
* Run `decide`, a decision region (group commit §6): synchronous, reading scheduler memory and recording the keys it
|
|
1186
|
+
* reads, staging nothing and changing no scheduler state. When a key it read has an unpublished writer, its result,
|
|
1187
|
+
* or its error, is dropped: once that writer is published, `refresh` reloads what the region needs and it runs
|
|
1188
|
+
* again. A waiting job holds the line, so unpublished writes only shrink, and the runs converge. What a region read
|
|
1189
|
+
* that had no unpublished writer gets none for the rest of the job, so its result stands while the job applies it.
|
|
1190
|
+
*/
|
|
1191
|
+
async #region(decide, refresh) {
|
|
1192
|
+
for (;;) {
|
|
1193
|
+
const keys = new Set();
|
|
1194
|
+
this.#reading = keys;
|
|
1195
|
+
let result;
|
|
1196
|
+
try {
|
|
1197
|
+
result = { value: decide() };
|
|
1198
|
+
}
|
|
1199
|
+
catch (error) {
|
|
1200
|
+
result = { error };
|
|
1201
|
+
}
|
|
1202
|
+
finally {
|
|
1203
|
+
this.#reading = undefined;
|
|
1204
|
+
}
|
|
1205
|
+
const gates = [];
|
|
1206
|
+
for (const key of keys) {
|
|
1207
|
+
const gate = this.#session.touch(key);
|
|
1208
|
+
if (gate !== undefined)
|
|
1209
|
+
gates.push(gate);
|
|
1210
|
+
}
|
|
1211
|
+
if (gates.length === 0) {
|
|
1212
|
+
if ("error" in result)
|
|
1213
|
+
throw result.error;
|
|
1214
|
+
return result.value;
|
|
1215
|
+
}
|
|
1216
|
+
await Promise.all(gates);
|
|
1217
|
+
await refresh?.();
|
|
1218
|
+
}
|
|
1219
|
+
}
|
|
1220
|
+
/** Load the chains a region walks, again after it waited: what was published meanwhile may list new work. */
|
|
1221
|
+
#reloadScopes = () => this.#loadScopes(false);
|
|
1222
|
+
/** In a decision region, note that it reads `key`; anywhere else, nothing. */
|
|
1223
|
+
#record(key) {
|
|
1224
|
+
this.#reading?.add(key);
|
|
1225
|
+
}
|
|
1226
|
+
/** A live task's committed record, read by a decision region. */
|
|
1227
|
+
#liveRecord(id) {
|
|
1228
|
+
this.#record(taskKey(id));
|
|
1229
|
+
return this.#live.get(id);
|
|
1230
|
+
}
|
|
1231
|
+
/**
|
|
1232
|
+
* Keep `node`, which a chain load of the job on the line installed, until that job's commit is published or fails:
|
|
1233
|
+
* work it creates below the node needs it, and the sweep would drop it as unneeded before then.
|
|
1234
|
+
*/
|
|
1235
|
+
#pin(node) {
|
|
1236
|
+
this.#pins.set(node, (this.#pins.get(node) ?? 0) + 1);
|
|
1237
|
+
this.#session.onJobEnd(() => {
|
|
1238
|
+
const count = this.#pins.get(node) - 1;
|
|
1239
|
+
if (count > 0) {
|
|
1240
|
+
this.#pins.set(node, count);
|
|
1241
|
+
return;
|
|
1242
|
+
}
|
|
1243
|
+
this.#pins.delete(node);
|
|
1244
|
+
this.#dropIfUnneeded(node);
|
|
1245
|
+
});
|
|
1246
|
+
}
|
|
669
1247
|
/**
|
|
670
1248
|
* Bring the scheduler's sets in line with a task's committed record. A task that ended may free the nearest live task
|
|
671
1249
|
* above it: one holding its outcome finalizes, an abort-marked one runs its abort handler; its waiters may run.
|
|
672
1250
|
*/
|
|
673
1251
|
#track(previous, record) {
|
|
674
1252
|
const id = record.id;
|
|
1253
|
+
const invocation = this.#invocations.get(id);
|
|
1254
|
+
if (invocation !== undefined)
|
|
1255
|
+
invocation.record = record;
|
|
1256
|
+
if (record.state.status === "terminal") {
|
|
1257
|
+
if (previous !== undefined && !this.#closing)
|
|
1258
|
+
this.#countActive(-1, record.kind);
|
|
1259
|
+
this.#resumed.delete(id);
|
|
1260
|
+
}
|
|
1261
|
+
else if (previous === undefined && !this.#closing) {
|
|
1262
|
+
this.#countActive(1, record.kind);
|
|
1263
|
+
}
|
|
675
1264
|
if (previous?.state.status === "waiting")
|
|
676
1265
|
for (const on of previous.state.on)
|
|
677
1266
|
deleteFrom(this.#waiters, on, id);
|
|
678
1267
|
if (record.state.status === "terminal") {
|
|
679
1268
|
const node = nodeOf(record);
|
|
680
1269
|
this.#live.delete(id);
|
|
1270
|
+
this.#sleeping.delete(id);
|
|
681
1271
|
this.#intent.delete(id);
|
|
682
1272
|
this.#runnable.delete(id);
|
|
683
1273
|
this.#finalizeChecks.delete(id);
|
|
@@ -711,6 +1301,10 @@ export class TaskScheduler {
|
|
|
711
1301
|
if (record.state.status === "waiting")
|
|
712
1302
|
for (const on of record.state.on)
|
|
713
1303
|
addTo(this.#waiters, on, id);
|
|
1304
|
+
if (record.state.status === "sleeping")
|
|
1305
|
+
this.#sleeping.add(id);
|
|
1306
|
+
else
|
|
1307
|
+
this.#sleeping.delete(id);
|
|
714
1308
|
}
|
|
715
1309
|
/** The nearest live task above ended ordinary work may be free now: one holding its outcome, or an aborting one. */
|
|
716
1310
|
#release(node, overlay) {
|
|
@@ -773,11 +1367,12 @@ export class TaskScheduler {
|
|
|
773
1367
|
* at or below a background task holds nothing above it.
|
|
774
1368
|
*/
|
|
775
1369
|
#nearestLiveAbove(node, overlay) {
|
|
1370
|
+
this.#record(INTENT_KEY);
|
|
776
1371
|
for (let current = node; !current.background;) {
|
|
777
1372
|
const owner = current.owner ?? this.#edge(current.conversationId, overlay);
|
|
778
1373
|
if (owner === undefined || owner === null)
|
|
779
1374
|
return undefined;
|
|
780
|
-
const record = overlay?.tasks.get(owner) ?? this.#
|
|
1375
|
+
const record = overlay?.tasks.get(owner) ?? this.#liveRecord(owner);
|
|
781
1376
|
if (record !== undefined && record.state.status !== "terminal")
|
|
782
1377
|
return record;
|
|
783
1378
|
const above = record ?? this.#settled.get(owner);
|
|
@@ -797,7 +1392,10 @@ export class TaskScheduler {
|
|
|
797
1392
|
}
|
|
798
1393
|
/** Owner task of a conversation, `null` when ownerless, `undefined` while not loaded. */
|
|
799
1394
|
#edge(id, overlay) {
|
|
800
|
-
|
|
1395
|
+
if (overlay?.edges.has(id) === true)
|
|
1396
|
+
return overlay.edges.get(id);
|
|
1397
|
+
this.#record(conversationKey(id));
|
|
1398
|
+
return this.#edges.get(id);
|
|
801
1399
|
}
|
|
802
1400
|
#node(id, overlay) {
|
|
803
1401
|
return overlay?.tasks.get(id) ?? this.#live.get(id) ?? this.#settled.get(id);
|
|
@@ -836,6 +1434,7 @@ export class TaskScheduler {
|
|
|
836
1434
|
const links = overlay === undefined ? undefined : this.#links(overlay);
|
|
837
1435
|
const pending = [start];
|
|
838
1436
|
for (let at = pending.pop(); at !== undefined; at = pending.pop()) {
|
|
1437
|
+
this.#record(ownedKey(at));
|
|
839
1438
|
for (const children of [this.#below.get(at), links?.get(at)]) {
|
|
840
1439
|
for (const child of children ?? []) {
|
|
841
1440
|
if (typeof child === "string") {
|
|
@@ -923,6 +1522,7 @@ export class TaskScheduler {
|
|
|
923
1522
|
* edge is not loaded.
|
|
924
1523
|
*/
|
|
925
1524
|
#inScope(start, scope, crossBackground = false) {
|
|
1525
|
+
this.#record(INTENT_KEY);
|
|
926
1526
|
for (const step of this.#above(start)) {
|
|
927
1527
|
if ("unknown" in step)
|
|
928
1528
|
return undefined;
|
|
@@ -945,6 +1545,7 @@ export class TaskScheduler {
|
|
|
945
1545
|
}
|
|
946
1546
|
/** The nearest live owner above `start` whose cancellation intent reaches it, if any; see `#belowCancelled`. */
|
|
947
1547
|
#cancellingOwner(start) {
|
|
1548
|
+
this.#record(INTENT_KEY);
|
|
948
1549
|
// Nothing cancels, the common case: no walk.
|
|
949
1550
|
if (this.#intent.size === 0)
|
|
950
1551
|
return undefined;
|
|
@@ -961,29 +1562,149 @@ export class TaskScheduler {
|
|
|
961
1562
|
}
|
|
962
1563
|
return undefined;
|
|
963
1564
|
}
|
|
1565
|
+
/**
|
|
1566
|
+
* Count a task in or out of `pi.durable.tasks.active`. Commit observers count at a commit's publication, which must not
|
|
1567
|
+
* throw, so a throwing adapter is reported.
|
|
1568
|
+
*/
|
|
1569
|
+
#countActive(delta, kind) {
|
|
1570
|
+
try {
|
|
1571
|
+
this.#activeMetric.add(delta, { task: kind });
|
|
1572
|
+
}
|
|
1573
|
+
catch (error) {
|
|
1574
|
+
this.#report(error, TELEMETRY);
|
|
1575
|
+
}
|
|
1576
|
+
}
|
|
964
1577
|
/** Fail the Session, which reports it, for a throw in one of the scheduler's own commits; ignored once closing. */
|
|
965
|
-
#failSession(error) {
|
|
1578
|
+
#failSession(error, context) {
|
|
966
1579
|
if (!this.#closing)
|
|
967
|
-
this.#session.fail(error);
|
|
1580
|
+
this.#session.fail(error, context);
|
|
968
1581
|
}
|
|
969
1582
|
/** Close listener: runs synchronously once admission is sealed, before `join()`, or when the Session fails. */
|
|
970
1583
|
#seal() {
|
|
1584
|
+
for (const record of this.#live.values())
|
|
1585
|
+
this.#countActive(-1, record.kind);
|
|
1586
|
+
this.#resumed.clear();
|
|
971
1587
|
this.#closing = true;
|
|
972
1588
|
try {
|
|
973
1589
|
this.#unsubscribeRegistry();
|
|
974
1590
|
}
|
|
975
1591
|
catch (error) {
|
|
976
|
-
this.#report(error);
|
|
1592
|
+
this.#report(error, REGISTRY);
|
|
977
1593
|
}
|
|
978
1594
|
const error = closedError(this.#session);
|
|
979
1595
|
this.#taskWaiters.rejectAll(error);
|
|
980
1596
|
this.#idleWaiters.rejectAll(error);
|
|
981
1597
|
this.#contexts.clear();
|
|
982
|
-
|
|
983
|
-
|
|
1598
|
+
this.#idleOrder.clear();
|
|
1599
|
+
this.#expiry?.disarm();
|
|
984
1600
|
this.#expiry = undefined;
|
|
1601
|
+
this.#timer?.disarm();
|
|
1602
|
+
this.#timer = undefined;
|
|
1603
|
+
}
|
|
1604
|
+
/**
|
|
1605
|
+
* Recompute the wake in a microtask the line never waits for, with no order relative to host commit listeners (one
|
|
1606
|
+
* already queued serves later changes too): arm the built-in timer, or tell `onWake` of a change.
|
|
1607
|
+
*/
|
|
1608
|
+
#scheduleWake() {
|
|
1609
|
+
if (this.#wakeScheduled || this.#closing || this.#readOnly)
|
|
1610
|
+
return;
|
|
1611
|
+
this.#wakeScheduled = true;
|
|
1612
|
+
queueMicrotask(() => {
|
|
1613
|
+
this.#wakeScheduled = false;
|
|
1614
|
+
if (this.#closing)
|
|
1615
|
+
return;
|
|
1616
|
+
const wake = this.wake();
|
|
1617
|
+
if (this.#onWake === undefined) {
|
|
1618
|
+
this.#armTimer(this.#enabled ? wake.at : undefined);
|
|
1619
|
+
return;
|
|
1620
|
+
}
|
|
1621
|
+
const last = this.#lastWake;
|
|
1622
|
+
if (last !== undefined && last.active === wake.active && last.at === wake.at)
|
|
1623
|
+
return;
|
|
1624
|
+
try {
|
|
1625
|
+
this.#onWake(wake);
|
|
1626
|
+
// Only a delivered wake counts: after a throw, the next change or kick offers it again.
|
|
1627
|
+
this.#lastWake = wake;
|
|
1628
|
+
}
|
|
1629
|
+
catch (error) {
|
|
1630
|
+
this.#report(error, ON_WAKE);
|
|
1631
|
+
}
|
|
1632
|
+
});
|
|
1633
|
+
}
|
|
1634
|
+
/**
|
|
1635
|
+
* Keep the timer for the earliest sleep. It keeps the process alive, unlike the context expiry timer: a process with a
|
|
1636
|
+
* sleeping task has work left, such as a retry backoff. Rechecks the clock when it fires, so a wake before `until`
|
|
1637
|
+
* only arms it again.
|
|
1638
|
+
*/
|
|
1639
|
+
#armTimer(at) {
|
|
1640
|
+
if (this.#timer !== undefined && this.#timer.at === at)
|
|
1641
|
+
return;
|
|
1642
|
+
this.#timer?.disarm();
|
|
1643
|
+
this.#timer = undefined;
|
|
1644
|
+
if (at === undefined || this.#closing)
|
|
1645
|
+
return;
|
|
1646
|
+
const disarm = this.#timers.foreground(() => {
|
|
1647
|
+
this.#timer = undefined;
|
|
1648
|
+
this.#wakeDue();
|
|
1649
|
+
this.#kick();
|
|
1650
|
+
}, at - this.#now());
|
|
1651
|
+
this.#timer = { at, disarm };
|
|
1652
|
+
}
|
|
1653
|
+
/** Make every sleeping task whose time has come a reservation candidate. */
|
|
1654
|
+
#wakeDue() {
|
|
1655
|
+
for (const id of this.#sleeping) {
|
|
1656
|
+
const record = this.#live.get(id);
|
|
1657
|
+
if (record !== undefined && !this.#notDue(record))
|
|
1658
|
+
this.#runnable.add(id);
|
|
1659
|
+
}
|
|
1660
|
+
}
|
|
1661
|
+
/**
|
|
1662
|
+
* The report info of a task's failure, with the extension that installed its definition: the code that failed is
|
|
1663
|
+
* that extension's. Built-in tasks have none.
|
|
1664
|
+
*/
|
|
1665
|
+
/** What a report about `task`'s own code says of it: the task, and the extension that installed its definition. */
|
|
1666
|
+
#aboutTask(task) {
|
|
1667
|
+
let extension;
|
|
1668
|
+
if ("extension" in task)
|
|
1669
|
+
extension = task.extension;
|
|
1670
|
+
else {
|
|
1671
|
+
try {
|
|
1672
|
+
const snapshot = this.#registry.snapshot();
|
|
1673
|
+
extension = installerOf(snapshot, snapshot.task(task.kind));
|
|
1674
|
+
}
|
|
1675
|
+
catch {
|
|
1676
|
+
// A registry that throws is reported where it is read for scheduling; a report goes out without the name.
|
|
1677
|
+
}
|
|
1678
|
+
}
|
|
1679
|
+
return { task: reportTask(task), ...optionalExtension(extension) };
|
|
1680
|
+
}
|
|
1681
|
+
/** `settings.restartLimit`, or the default when the host's settings throw. */
|
|
1682
|
+
#restartLimit() {
|
|
1683
|
+
try {
|
|
1684
|
+
return this.#settings().restartLimit;
|
|
1685
|
+
}
|
|
1686
|
+
catch (error) {
|
|
1687
|
+
this.#report(error, SETTINGS);
|
|
1688
|
+
return DEFAULT_RESTART_LIMIT;
|
|
1689
|
+
}
|
|
1690
|
+
}
|
|
1691
|
+
#notifyBlocked() {
|
|
1692
|
+
for (const listener of [...this.#blockedListeners]) {
|
|
1693
|
+
try {
|
|
1694
|
+
listener();
|
|
1695
|
+
}
|
|
1696
|
+
catch (error) {
|
|
1697
|
+
this.#report(error, { kind: "listener_error" });
|
|
1698
|
+
}
|
|
1699
|
+
}
|
|
1700
|
+
}
|
|
1701
|
+
/** A sleep whose `until` the clock has not reached; an abort mark ends a sleep. */
|
|
1702
|
+
#notDue(record) {
|
|
1703
|
+
const state = record.state;
|
|
1704
|
+
return state.status === "sleeping" && !record.abortRequested && state.until > this.#now();
|
|
985
1705
|
}
|
|
986
1706
|
#kick() {
|
|
1707
|
+
this.#scheduleWake();
|
|
987
1708
|
this.#dirty = true;
|
|
988
1709
|
if (this.#draining || !this.#enabled || this.#closing)
|
|
989
1710
|
return;
|
|
@@ -1016,45 +1737,57 @@ export class TaskScheduler {
|
|
|
1016
1737
|
*/
|
|
1017
1738
|
async #reserve() {
|
|
1018
1739
|
const reservations = [];
|
|
1740
|
+
const faulted = [];
|
|
1019
1741
|
let abandoned = [];
|
|
1020
1742
|
try {
|
|
1021
|
-
await this.#session.
|
|
1743
|
+
await this.#session.commitInternal(async (tx) => {
|
|
1022
1744
|
if (!this.#enabled || this.#closing)
|
|
1023
1745
|
return;
|
|
1024
|
-
for (const id of this.#abandoned) {
|
|
1025
|
-
const record = this.#live.get(id);
|
|
1026
|
-
if (record !== undefined && !record.abortRequested)
|
|
1027
|
-
tx.setTask(withAbortMark(record, "restart"));
|
|
1028
|
-
}
|
|
1029
1746
|
if (this.#abandoned.size > 0) {
|
|
1030
|
-
|
|
1747
|
+
const ids = [...this.#abandoned];
|
|
1748
|
+
const unmarked = await this.#region(() => ids
|
|
1749
|
+
.map((id) => this.#liveRecord(id))
|
|
1750
|
+
.filter((record) => record !== undefined && !record.abortRequested));
|
|
1751
|
+
for (const record of unmarked)
|
|
1752
|
+
tx.setTask(withAbortMark(record, "restart"));
|
|
1753
|
+
abandoned = ids;
|
|
1031
1754
|
return;
|
|
1032
1755
|
}
|
|
1033
1756
|
await this.#loadScopes(false);
|
|
1757
|
+
// Every pass picks up sleeps whose time has come: a pass started by any change may be the first to run after
|
|
1758
|
+
// a deadline, and the wake it schedules no longer keeps a timer for a due sleep.
|
|
1759
|
+
this.#wakeDue();
|
|
1760
|
+
const plan = await this.#region(() => this.#planReservation(), this.#reloadScopes);
|
|
1761
|
+
for (const id of plan.dropped)
|
|
1762
|
+
this.#runnable.delete(id);
|
|
1763
|
+
// Work below an owner with cancellation intent waits for its cascade mark instead of running a phase; schedule
|
|
1764
|
+
// the cascade, which may not have run yet.
|
|
1765
|
+
if (plan.cascade) {
|
|
1766
|
+
this.#cascadePending = true;
|
|
1767
|
+
this.#scheduleReconcile();
|
|
1768
|
+
}
|
|
1034
1769
|
// Taken once per pass, and only when some task is a candidate.
|
|
1035
1770
|
let snapshot;
|
|
1036
|
-
for (const
|
|
1037
|
-
const
|
|
1038
|
-
if (record === undefined || record.state.status === "completing" || this.#invocations.has(id)) {
|
|
1039
|
-
this.#runnable.delete(id);
|
|
1040
|
-
continue;
|
|
1041
|
-
}
|
|
1042
|
-
// It returns once what it waits for ends, its record changes, or the work below it ends.
|
|
1043
|
-
if (this.#waitingOn(record, true).length > 0) {
|
|
1044
|
-
this.#runnable.delete(id);
|
|
1045
|
-
continue;
|
|
1046
|
-
}
|
|
1047
|
-
const runnable = record;
|
|
1771
|
+
for (const record of plan.ready) {
|
|
1772
|
+
const id = record.id;
|
|
1048
1773
|
const mode = record.abortRequested ? "abort" : "run";
|
|
1049
|
-
// Work
|
|
1050
|
-
// the
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1774
|
+
// Work cut off `restartLimit` times in a row would run forever: an abort handler, or a phase of a kind that
|
|
1775
|
+
// does not end itself at the limit, ends the task `faulted` instead.
|
|
1776
|
+
// The settings are read only for a task that was cut off at all.
|
|
1777
|
+
const cutOff = record.restarts !== undefined && record.restarts >= this.#restartLimit();
|
|
1778
|
+
if (cutOff && (mode === "abort" || !this.#limitsRestarts(record.kind))) {
|
|
1779
|
+
const message = mode === "abort"
|
|
1780
|
+
? `Abort handler of task ${id} (${record.kind}) was cut off ${record.restarts} times`
|
|
1781
|
+
: `Task ${id} (${record.kind}) was cut off ${record.restarts} times without progress`;
|
|
1782
|
+
await this.#terminate(tx, record, {
|
|
1783
|
+
status: "faulted",
|
|
1784
|
+
error: { message, detail: { reason: "restart_limit" } },
|
|
1785
|
+
});
|
|
1786
|
+
faulted.push({ record, message });
|
|
1054
1787
|
continue;
|
|
1055
1788
|
}
|
|
1056
1789
|
snapshot ??= this.#registry.snapshot();
|
|
1057
|
-
const resolution = this.#resolve(
|
|
1790
|
+
const resolution = this.#resolve(record, snapshot);
|
|
1058
1791
|
if (resolution.kind === "blocked") {
|
|
1059
1792
|
// An abandoned task waits for its definition, so its abort handler can clean up.
|
|
1060
1793
|
if (mode === "abort" && record.abortReason === undefined) {
|
|
@@ -1062,7 +1795,7 @@ export class TaskScheduler {
|
|
|
1062
1795
|
}
|
|
1063
1796
|
continue;
|
|
1064
1797
|
}
|
|
1065
|
-
if (resolution.record !==
|
|
1798
|
+
if (resolution.record !== record || record.state.status !== "running") {
|
|
1066
1799
|
tx.setTask(withState(resolution.record, {
|
|
1067
1800
|
status: "running",
|
|
1068
1801
|
checkpoint: resolution.record.state.checkpoint,
|
|
@@ -1071,7 +1804,7 @@ export class TaskScheduler {
|
|
|
1071
1804
|
// Registered on the line, so marks and later reservations see it and close joins it; `#end` makes it a
|
|
1072
1805
|
// candidate again.
|
|
1073
1806
|
this.#runnable.delete(id);
|
|
1074
|
-
const invocation = this.#createInvocation(record, mode);
|
|
1807
|
+
const invocation = this.#createInvocation(record, mode, installerOf(snapshot, resolution.task));
|
|
1075
1808
|
reservations.push({ invocation, task: resolution.task, snapshot });
|
|
1076
1809
|
}
|
|
1077
1810
|
}, this.#context);
|
|
@@ -1084,6 +1817,13 @@ export class TaskScheduler {
|
|
|
1084
1817
|
}
|
|
1085
1818
|
throw error;
|
|
1086
1819
|
}
|
|
1820
|
+
for (const { record, message } of faulted) {
|
|
1821
|
+
this.#report(new Error(message), {
|
|
1822
|
+
kind: "task_faulted",
|
|
1823
|
+
...this.#aboutTask(recordTask(record)),
|
|
1824
|
+
reason: "restart_limit",
|
|
1825
|
+
});
|
|
1826
|
+
}
|
|
1087
1827
|
// Marked, or already marked or gone: done with them. Reserve again, now seeing the marks.
|
|
1088
1828
|
for (const id of abandoned)
|
|
1089
1829
|
this.#abandoned.delete(id);
|
|
@@ -1091,6 +1831,39 @@ export class TaskScheduler {
|
|
|
1091
1831
|
this.#dirty = true;
|
|
1092
1832
|
return reservations;
|
|
1093
1833
|
}
|
|
1834
|
+
/**
|
|
1835
|
+
* A reservation pass's decision region: the candidates it drops (gone, holding, waiting, asleep, or with an
|
|
1836
|
+
* invocation), whether one waits for a cascade, and the runnable records to resolve. Iterating the candidates is not
|
|
1837
|
+
* keyed: a pass that misses a task a pending commit makes runnable only delays it, as does skipping one with an
|
|
1838
|
+
* invocation without reading its record, or a waiter's check whether what it waits on is still live.
|
|
1839
|
+
*/
|
|
1840
|
+
#planReservation() {
|
|
1841
|
+
const dropped = [];
|
|
1842
|
+
const ready = [];
|
|
1843
|
+
let cascade = false;
|
|
1844
|
+
for (const id of this.#runnable) {
|
|
1845
|
+
if (this.#invocations.has(id)) {
|
|
1846
|
+
dropped.push(id);
|
|
1847
|
+
continue;
|
|
1848
|
+
}
|
|
1849
|
+
const record = this.#liveRecord(id);
|
|
1850
|
+
// It returns once what it waits for ends, its record changes, or the work below it ends; a sleep at the first pass
|
|
1851
|
+
// after its time comes, which the timer, `resume()`, or any change starts.
|
|
1852
|
+
if (record === undefined ||
|
|
1853
|
+
record.state.status === "completing" ||
|
|
1854
|
+
this.#waitingOn(record, true).length > 0 ||
|
|
1855
|
+
this.#notDue(record)) {
|
|
1856
|
+
dropped.push(id);
|
|
1857
|
+
continue;
|
|
1858
|
+
}
|
|
1859
|
+
if (!record.abortRequested && !record.background && this.#belowCancelled(parentOf(record))) {
|
|
1860
|
+
cascade = true;
|
|
1861
|
+
continue;
|
|
1862
|
+
}
|
|
1863
|
+
ready.push(record);
|
|
1864
|
+
}
|
|
1865
|
+
return { dropped, cascade, ready };
|
|
1866
|
+
}
|
|
1094
1867
|
/**
|
|
1095
1868
|
* Live tasks a task waits for before its next invocation: its live ordinary owned work when abort-marked, since
|
|
1096
1869
|
* abort runs bottom-up, otherwise the live part of the `on` of a wait.
|
|
@@ -1123,7 +1896,11 @@ export class TaskScheduler {
|
|
|
1123
1896
|
}
|
|
1124
1897
|
catch (error) {
|
|
1125
1898
|
this.#failedMigrations.set(record.id, { task: fit.task, error });
|
|
1126
|
-
|
|
1899
|
+
// Blocked now without a commit: the graph and the host's wake must hear it.
|
|
1900
|
+
queueMicrotask(() => this.#notifyBlocked());
|
|
1901
|
+
this.#scheduleWake();
|
|
1902
|
+
const extension = installerOf(snapshot, fit.task);
|
|
1903
|
+
this.#report(error, { kind: "migration_failed", ...this.#aboutTask({ ...recordTask(record), extension }) });
|
|
1127
1904
|
return { kind: "blocked", reason: "migration_failed" };
|
|
1128
1905
|
}
|
|
1129
1906
|
}
|
|
@@ -1153,6 +1930,36 @@ export class TaskScheduler {
|
|
|
1153
1930
|
roots: this.#roots.size,
|
|
1154
1931
|
unloaded: this.#unloaded.size,
|
|
1155
1932
|
dropQueue: this.#dropQueue.size,
|
|
1933
|
+
pins: this.#pins.size,
|
|
1934
|
+
};
|
|
1935
|
+
}
|
|
1936
|
+
/**
|
|
1937
|
+
* The kept contexts' idle state, for tests (a diagnostic, computed on demand): how many are kept and how many are
|
|
1938
|
+
* idle, how many disagree with a full recompute (an idle period without `#idle()`, or `#idle()` without one; an
|
|
1939
|
+
* expiry order that differs from the idle periods, or is not sorted), and how many idle checks covered every scope.
|
|
1940
|
+
*/
|
|
1941
|
+
get idleState() {
|
|
1942
|
+
let mismatches = 0;
|
|
1943
|
+
let idle = 0;
|
|
1944
|
+
for (const [conversationId, kept] of this.#contexts) {
|
|
1945
|
+
if (kept.idleSince !== undefined)
|
|
1946
|
+
idle++;
|
|
1947
|
+
if ((kept.idleSince !== undefined) !== this.#idle(conversationId))
|
|
1948
|
+
mismatches++;
|
|
1949
|
+
if (this.#idleOrder.get(conversationId) !== kept.idleSince)
|
|
1950
|
+
mismatches++;
|
|
1951
|
+
}
|
|
1952
|
+
let previous = Number.NEGATIVE_INFINITY;
|
|
1953
|
+
for (const [conversationId, idleSince] of this.#idleOrder) {
|
|
1954
|
+
if (idleSince < previous || this.#contexts.get(conversationId)?.idleSince !== idleSince)
|
|
1955
|
+
mismatches++;
|
|
1956
|
+
previous = idleSince;
|
|
1957
|
+
}
|
|
1958
|
+
return {
|
|
1959
|
+
contexts: this.#contexts.size,
|
|
1960
|
+
idleContexts: idle,
|
|
1961
|
+
idleMismatches: mismatches,
|
|
1962
|
+
fullIdleChecks: this.#fullIdleChecks,
|
|
1156
1963
|
};
|
|
1157
1964
|
}
|
|
1158
1965
|
/**
|
|
@@ -1161,6 +1968,7 @@ export class TaskScheduler {
|
|
|
1161
1968
|
* cannot exist, shows as failed.
|
|
1162
1969
|
*/
|
|
1163
1970
|
async inspect(snapshot) {
|
|
1971
|
+
await this.#published();
|
|
1164
1972
|
await this.#loadScopes(false);
|
|
1165
1973
|
const tasks = [];
|
|
1166
1974
|
for (const record of this.#live.values())
|
|
@@ -1176,24 +1984,31 @@ export class TaskScheduler {
|
|
|
1176
1984
|
const on = this.#waitingOn(record);
|
|
1177
1985
|
if (on.length > 0)
|
|
1178
1986
|
return { kind: "waiting", on };
|
|
1987
|
+
if (this.#notDue(record))
|
|
1988
|
+
return { kind: "sleeping", until: record.state.until };
|
|
1179
1989
|
const fit = this.#fit(record, snapshot.task(record.kind));
|
|
1180
1990
|
if ("reason" in fit)
|
|
1181
1991
|
return { kind: "blocked", ...fit };
|
|
1182
1992
|
if (fit.migrates && fit.task.definition.migrate === undefined) {
|
|
1183
1993
|
return { kind: "blocked", reason: "migration_failed", error: missingMigration(record, erased(fit.task)) };
|
|
1184
1994
|
}
|
|
1185
|
-
return { kind: "
|
|
1995
|
+
return { kind: "pending", migrates: fit.migrates };
|
|
1186
1996
|
}
|
|
1187
|
-
#createInvocation(record, mode) {
|
|
1997
|
+
#createInvocation(record, mode, extension) {
|
|
1188
1998
|
const controller = new AbortController();
|
|
1189
1999
|
const { promise: done, resolve: finish } = Promise.withResolvers();
|
|
1190
2000
|
const invocation = {
|
|
1191
2001
|
taskId: record.id,
|
|
2002
|
+
kind: record.kind,
|
|
2003
|
+
extension,
|
|
1192
2004
|
conversationId: record.conversationId,
|
|
1193
2005
|
mode,
|
|
1194
2006
|
controller,
|
|
2007
|
+
links: record.startedAt === undefined ? record.links : undefined,
|
|
1195
2008
|
context: withAbortSignal(controller.signal, this.#context),
|
|
2009
|
+
record: record,
|
|
1196
2010
|
watches: new Set(),
|
|
2011
|
+
timers: new Set(),
|
|
1197
2012
|
ended: false,
|
|
1198
2013
|
done,
|
|
1199
2014
|
finish: () => finish(),
|
|
@@ -1211,10 +2026,13 @@ export class TaskScheduler {
|
|
|
1211
2026
|
await this.#runAbort(reservation);
|
|
1212
2027
|
}
|
|
1213
2028
|
catch (error) {
|
|
1214
|
-
this.#report(error);
|
|
2029
|
+
this.#report(error, { kind: "task_error", ...this.#aboutTask(invocation) }, invocation.context);
|
|
1215
2030
|
}
|
|
1216
2031
|
finally {
|
|
1217
2032
|
this.#end(invocation);
|
|
2033
|
+
invocation.record = undefined;
|
|
2034
|
+
invocation.links = undefined;
|
|
2035
|
+
invocation.fault = undefined;
|
|
1218
2036
|
invocation.finish();
|
|
1219
2037
|
this.#kick();
|
|
1220
2038
|
}
|
|
@@ -1226,61 +2044,128 @@ export class TaskScheduler {
|
|
|
1226
2044
|
const state = { task: reservation.task, snapshot: reservation.snapshot, reported: undefined };
|
|
1227
2045
|
const phase = { snapshot: () => state.snapshot, task: () => state.task, agent: undefined };
|
|
1228
2046
|
const runtime = this.#runtime(invocation, phase);
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
2047
|
+
const progress = (tx, current) => this.#progress(tx, current, state, invocation.extension, invocation.context);
|
|
2048
|
+
let current = await this.#step(invocation, (task) => this.#rules(task, undefined), progress);
|
|
2049
|
+
while (current !== undefined && !this.#closing) {
|
|
2050
|
+
const task = current;
|
|
2051
|
+
current = await this.#phase(invocation, task, async () => {
|
|
2052
|
+
const checkpoint = task.state.checkpoint;
|
|
2053
|
+
// Each phase handler resolves its agent afresh, at first use.
|
|
2054
|
+
phase.agent = undefined;
|
|
2055
|
+
let previous;
|
|
2056
|
+
try {
|
|
2057
|
+
await erased(state.task).phases[checkpoint.phase](task, runtime, invocation.context);
|
|
2058
|
+
previous = { checkpoint };
|
|
2059
|
+
}
|
|
2060
|
+
catch (error) {
|
|
2061
|
+
previous = { checkpoint, failure: { error } };
|
|
2062
|
+
}
|
|
2063
|
+
return this.#step(invocation, (next) => this.#rules(next, previous), progress);
|
|
2064
|
+
});
|
|
2065
|
+
}
|
|
2066
|
+
}
|
|
2067
|
+
/** A phase owns its decision commit, not the next handler, a durable wait, or recovery. */
|
|
2068
|
+
#phase(invocation, task, run) {
|
|
2069
|
+
const parent = task.trace === undefined
|
|
2070
|
+
? this.#telemetry
|
|
2071
|
+
: (this.#telemetry.propagation?.extract(task.trace) ?? this.#telemetry);
|
|
2072
|
+
const checkpoint = task.state.checkpoint;
|
|
2073
|
+
const attempt = checkpoint.attempt;
|
|
2074
|
+
const resumed = this.#resumed.delete(task.id);
|
|
2075
|
+
return createTypedSpanStarter(parent, [DURABLE_TELEMETRY_SCHEMA])("pi.durable.phase", {
|
|
2076
|
+
"task.id": task.id,
|
|
2077
|
+
"task.name": task.kind,
|
|
2078
|
+
phase: checkpoint.phase,
|
|
2079
|
+
kind: invocation.mode,
|
|
2080
|
+
resumed,
|
|
2081
|
+
...(attempt === undefined ? {} : { attempt }),
|
|
2082
|
+
}, async (span) => {
|
|
2083
|
+
if (span.propagation !== undefined && invocation.links !== undefined) {
|
|
2084
|
+
for (const carrier of invocation.links)
|
|
2085
|
+
span.addLink({ context: span.propagation.extract(carrier) });
|
|
2086
|
+
}
|
|
2087
|
+
const previous = invocation.context;
|
|
2088
|
+
invocation.context = withTelemetry(span, previous);
|
|
2089
|
+
const started = performance.now();
|
|
1238
2090
|
try {
|
|
1239
|
-
await
|
|
1240
|
-
previous = { checkpoint };
|
|
2091
|
+
return await run();
|
|
1241
2092
|
}
|
|
1242
|
-
|
|
1243
|
-
|
|
2093
|
+
finally {
|
|
2094
|
+
invocation.context = previous;
|
|
2095
|
+
const state = invocation.record.state;
|
|
2096
|
+
const terminal = invocation.fault ??
|
|
2097
|
+
(state.status === "terminal" || state.status === "completing" ? state.outcome : undefined);
|
|
2098
|
+
const outcome = terminal !== undefined
|
|
2099
|
+
? terminal.status === "completed"
|
|
2100
|
+
? "done"
|
|
2101
|
+
: terminal.status === "aborted"
|
|
2102
|
+
? "aborted"
|
|
2103
|
+
: "faulted"
|
|
2104
|
+
: state.status === "waiting" || state.status === "sleeping"
|
|
2105
|
+
? state.status
|
|
2106
|
+
: invocation.ended || this.#closing
|
|
2107
|
+
? "aborted"
|
|
2108
|
+
: `next=${state.checkpoint.phase}`;
|
|
2109
|
+
span.setAttributes({ outcome });
|
|
2110
|
+
if (terminal?.status === "faulted" || terminal?.status === "failed" || terminal?.status === "orphaned") {
|
|
2111
|
+
const message = terminal.status === "orphaned" ? terminal.reason : terminal.error.message;
|
|
2112
|
+
span.setStatus({
|
|
2113
|
+
status: "error",
|
|
2114
|
+
error: { name: `task_${terminal.status}`, message: ownedSlice(message, 0, 500) },
|
|
2115
|
+
});
|
|
2116
|
+
}
|
|
2117
|
+
else if (outcome === "aborted") {
|
|
2118
|
+
span.setStatus({ status: "ok" });
|
|
2119
|
+
}
|
|
2120
|
+
this.#durationMetric.record(performance.now() - started, { task: task.kind, phase: checkpoint.phase, outcome }, span);
|
|
1244
2121
|
}
|
|
1245
|
-
}
|
|
2122
|
+
});
|
|
1246
2123
|
}
|
|
1247
2124
|
/**
|
|
1248
|
-
* Precedence rules for a run invocation,
|
|
1249
|
-
* are applied by `#step`.
|
|
2125
|
+
* Precedence rules for a run invocation, a decision region of its step. Rules 1 (terminal, `completing`, `waiting`, or
|
|
2126
|
+
* `sleeping`) and 2 (closing) are applied by `#step`. Progress (rule 5) is decided by `#progress()` once the decision
|
|
2127
|
+
* stands, as it takes a registry snapshot.
|
|
1250
2128
|
*/
|
|
1251
|
-
#
|
|
2129
|
+
#rules(current, previous) {
|
|
1252
2130
|
// 3. abort mark: end; a fresh abort invocation starts once the task's ordinary owned work is gone.
|
|
1253
2131
|
if (current.abortRequested)
|
|
1254
|
-
return
|
|
2132
|
+
return END;
|
|
1255
2133
|
// An owner's cancellation intent ends it too, before its cascade marks it; reservation then holds it back.
|
|
1256
|
-
if (!current.background && this.#belowCancelled(parentOf(current)))
|
|
1257
|
-
|
|
1258
|
-
this.#scheduleReconcile();
|
|
1259
|
-
return false;
|
|
1260
|
-
}
|
|
2134
|
+
if (!current.background && this.#belowCancelled(parentOf(current)))
|
|
2135
|
+
return CASCADE;
|
|
1261
2136
|
if (previous === undefined)
|
|
1262
|
-
return
|
|
2137
|
+
return CONTINUE;
|
|
1263
2138
|
// 4. uncaught error.
|
|
1264
2139
|
if (previous.failure !== undefined)
|
|
1265
|
-
return { fault: previous.failure.error };
|
|
2140
|
+
return { kind: "fault", error: previous.failure.error };
|
|
1266
2141
|
// 6. no durable progress.
|
|
1267
2142
|
if (jsonEqual(current.state.checkpoint, previous.checkpoint)) {
|
|
1268
2143
|
const message = `Task ${current.kind} phase ${previous.checkpoint.phase} returned without durable progress`;
|
|
1269
|
-
return { fault: new Error(message) };
|
|
2144
|
+
return { kind: "fault", error: new Error(message) };
|
|
1270
2145
|
}
|
|
1271
|
-
|
|
2146
|
+
return PROGRESS;
|
|
2147
|
+
}
|
|
2148
|
+
/**
|
|
2149
|
+
* Rule 5, progress: refresh the snapshot; hand over to a replacement definition that can take the task, writing it
|
|
2150
|
+
* back as `pending`, or keep running under the old one, reported once per resolved definition. Returns whether the
|
|
2151
|
+
* invocation continues.
|
|
2152
|
+
*/
|
|
2153
|
+
#progress(tx, current, state, extension, context) {
|
|
1272
2154
|
state.snapshot = this.#registry.snapshot();
|
|
1273
2155
|
const next = state.snapshot.task(current.kind);
|
|
1274
|
-
if (next
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
2156
|
+
if (next === state.task)
|
|
2157
|
+
return true;
|
|
2158
|
+
if (next !== undefined && canReserve(next, current)) {
|
|
2159
|
+
tx.setTask(withState(current, { status: "pending", checkpoint: current.state.checkpoint }));
|
|
2160
|
+
return false;
|
|
2161
|
+
}
|
|
2162
|
+
if (state.reported === undefined || state.reported.task !== next) {
|
|
2163
|
+
state.reported = { task: next };
|
|
2164
|
+
this.#report(new Error(`Task ${current.id} keeps running under its old ${current.kind} definition`), {
|
|
2165
|
+
kind: "task_outdated",
|
|
2166
|
+
...this.#aboutTask({ ...recordTask(current), extension }),
|
|
2167
|
+
reason: outdatedReason(next, current),
|
|
2168
|
+
}, context);
|
|
1284
2169
|
}
|
|
1285
2170
|
return true;
|
|
1286
2171
|
}
|
|
@@ -1290,45 +2175,72 @@ export class TaskScheduler {
|
|
|
1290
2175
|
const current = this.#live.get(invocation.taskId);
|
|
1291
2176
|
if (current === undefined || this.#closing)
|
|
1292
2177
|
return;
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
2178
|
+
await this.#phase(invocation, current, async () => {
|
|
2179
|
+
let failure;
|
|
2180
|
+
try {
|
|
2181
|
+
const phase = {
|
|
2182
|
+
snapshot: () => reservation.snapshot,
|
|
2183
|
+
task: () => reservation.task,
|
|
2184
|
+
agent: undefined,
|
|
2185
|
+
};
|
|
2186
|
+
const runtime = this.#runtime(invocation, phase);
|
|
2187
|
+
await erased(reservation.task).abort(current, runtime, invocation.context);
|
|
2188
|
+
}
|
|
2189
|
+
catch (error) {
|
|
2190
|
+
failure = { error };
|
|
2191
|
+
}
|
|
2192
|
+
const message = `Abort handler of task ${invocation.taskId} returned without a terminal outcome`;
|
|
2193
|
+
await this.#step(invocation, () => ({ kind: "fault", error: failure?.error ?? new Error(message) }));
|
|
2194
|
+
});
|
|
1304
2195
|
}
|
|
1305
2196
|
/**
|
|
1306
|
-
* One
|
|
1307
|
-
*
|
|
1308
|
-
*
|
|
1309
|
-
*
|
|
2197
|
+
* One decision on the Session line, before a phase: a decision region (`#region()`) that reads the task's record. A
|
|
2198
|
+
* task that is no longer running (rule 1: terminal, `completing`, `waiting`, or `sleeping`) or a closing Harness (rule
|
|
2199
|
+
* 2) ends the invocation without a write; otherwise `plan` decides, and a decision that stands is applied: the
|
|
2200
|
+
* invocation continues, with `progress` for rule 5, or ends, with a fault written. Ending happens inside the callback,
|
|
2201
|
+
* before a fault's Harness cleanup. A rejected step, such as admission after close, also ends the invocation. Its
|
|
2202
|
+
* outcome goes to the invocation only, so it is a scheduler commit (`commitInternal()`).
|
|
1310
2203
|
*/
|
|
1311
|
-
async #step(invocation,
|
|
2204
|
+
async #step(invocation, plan, progress) {
|
|
2205
|
+
let fault;
|
|
1312
2206
|
try {
|
|
1313
|
-
|
|
1314
|
-
const
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
2207
|
+
const continued = await this.#session.commitInternal(async (tx) => {
|
|
2208
|
+
const decided = await this.#region(() => {
|
|
2209
|
+
const found = this.#liveRecord(invocation.taskId);
|
|
2210
|
+
if (found?.state.status !== "running" || this.#closing)
|
|
2211
|
+
return { plan: END };
|
|
2212
|
+
const current = found;
|
|
2213
|
+
return { current, plan: plan(current) };
|
|
2214
|
+
});
|
|
2215
|
+
const { current } = decided;
|
|
2216
|
+
const step = decided.plan;
|
|
2217
|
+
if (step.kind === "continue")
|
|
2218
|
+
return current;
|
|
2219
|
+
if (step.kind === "progress" && progress(tx, current))
|
|
1318
2220
|
return current;
|
|
1319
2221
|
this.#end(invocation);
|
|
1320
|
-
if (
|
|
1321
|
-
|
|
1322
|
-
|
|
2222
|
+
if (step.kind === "cascade") {
|
|
2223
|
+
this.#cascadePending = true;
|
|
2224
|
+
this.#scheduleReconcile();
|
|
2225
|
+
}
|
|
2226
|
+
if (step.kind === "fault") {
|
|
2227
|
+
fault = { error: step.error };
|
|
2228
|
+
const message = errorMessage(step.error);
|
|
2229
|
+
invocation.fault = { status: "faulted", error: { message } };
|
|
2230
|
+
await this.#terminate(tx, current, invocation.fault);
|
|
1323
2231
|
}
|
|
1324
2232
|
return undefined;
|
|
1325
|
-
},
|
|
2233
|
+
}, withoutAbortSignal(invocation.context));
|
|
2234
|
+
// Reported once the fault is durable: a host may count it, for example toward a breaker.
|
|
2235
|
+
if (fault !== undefined)
|
|
2236
|
+
this.#report(fault.error, { kind: "task_faulted", ...this.#aboutTask(invocation), reason: "threw" }, invocation.context);
|
|
2237
|
+
return continued;
|
|
1326
2238
|
}
|
|
1327
2239
|
catch (error) {
|
|
1328
2240
|
// The step writes the scheduler's own decision, a fault or a terminal record; a throw there would otherwise
|
|
1329
2241
|
// leave the task running, to be reserved and run again.
|
|
1330
2242
|
this.#end(invocation);
|
|
1331
|
-
this.#failSession(error);
|
|
2243
|
+
this.#failSession(error, invocation.context);
|
|
1332
2244
|
return undefined;
|
|
1333
2245
|
}
|
|
1334
2246
|
}
|
|
@@ -1338,7 +2250,7 @@ export class TaskScheduler {
|
|
|
1338
2250
|
*/
|
|
1339
2251
|
async #terminate(tx, record, outcome) {
|
|
1340
2252
|
await this.#loadScopes(false);
|
|
1341
|
-
if (this.#hasOwnedLive(record.id, overlayOf(tx))) {
|
|
2253
|
+
if (await this.#region(() => this.#hasOwnedLive(record.id, overlayOf(tx)), this.#reloadScopes)) {
|
|
1342
2254
|
tx.setTask(withState(record, { status: "completing", outcome }));
|
|
1343
2255
|
return;
|
|
1344
2256
|
}
|
|
@@ -1353,17 +2265,22 @@ export class TaskScheduler {
|
|
|
1353
2265
|
async #commitState(tx, invocation, current, next) {
|
|
1354
2266
|
if (next.status === "waiting")
|
|
1355
2267
|
await this.#validateWait(tx, invocation, current, next.on, next.policy);
|
|
2268
|
+
if (next.status === "sleeping")
|
|
2269
|
+
validateSleep(invocation, current.id, next.until);
|
|
1356
2270
|
if (next.status === "terminal") {
|
|
1357
2271
|
const overlay = overlayOf(tx);
|
|
1358
2272
|
await this.#loadScopes(false);
|
|
1359
2273
|
for (const record of overlay.tasks.values())
|
|
1360
2274
|
await this.#loadChain(parentOf(record), overlay);
|
|
1361
|
-
if (this.#hasOwnedLive(current.id,
|
|
1362
|
-
tx.setTask(withState(current, { status: "completing", outcome: next.outcome }));
|
|
2275
|
+
if (await this.#region(() => this.#hasOwnedLive(current.id, overlayOf(tx)), this.#reloadScopes)) {
|
|
2276
|
+
tx.setTask(withState(progressed(current), { status: "completing", outcome: next.outcome }));
|
|
1363
2277
|
return;
|
|
1364
2278
|
}
|
|
1365
2279
|
}
|
|
1366
|
-
|
|
2280
|
+
// A state the task committed that differs from its current one is progress: its restart count starts over. The
|
|
2281
|
+
// same checkpoint committed again, for example as intent before an effect, is not.
|
|
2282
|
+
const same = jsonEqual(current.state, next);
|
|
2283
|
+
tx.setTask(withState(same ? current : progressed(current), next));
|
|
1367
2284
|
}
|
|
1368
2285
|
/**
|
|
1369
2286
|
* A wait names existing tasks, and `failFast` only tasks the waiter owns. It may not close a cycle in the wait-for
|
|
@@ -1376,37 +2293,64 @@ export class TaskScheduler {
|
|
|
1376
2293
|
async #validateWait(tx, invocation, current, on, policy) {
|
|
1377
2294
|
if (invocation.mode === "abort")
|
|
1378
2295
|
throw new Error(`Abort handler of task ${current.id} cannot wait`);
|
|
1379
|
-
|
|
2296
|
+
// Owned work is found by walks down, which need the chains of every live task loaded.
|
|
2297
|
+
await this.#loadScopes(false);
|
|
2298
|
+
// Tasks in `on` that are not live, read from Storage: terminal, or missing.
|
|
2299
|
+
const ended = new Map();
|
|
2300
|
+
for (;;) {
|
|
2301
|
+
const checked = await this.#region(() => this.#checkWait(current, on, policy, overlayOf(tx), ended), this.#reloadScopes);
|
|
2302
|
+
if (checked === undefined)
|
|
2303
|
+
return;
|
|
2304
|
+
if ("missing" in checked) {
|
|
2305
|
+
for (const id of checked.missing)
|
|
2306
|
+
ended.set(id, (await this.#session.lineStorage.task(id, this.#context)) ?? null);
|
|
2307
|
+
continue;
|
|
2308
|
+
}
|
|
2309
|
+
throw checked.error;
|
|
2310
|
+
}
|
|
2311
|
+
}
|
|
2312
|
+
/**
|
|
2313
|
+
* `#validateWait()`'s decision region: why the wait is refused, `undefined` when it may be stored, or the tasks in
|
|
2314
|
+
* `on` it needs from Storage first, when `ended` lacks them.
|
|
2315
|
+
*/
|
|
2316
|
+
#checkWait(current, on, policy, overlay, ended) {
|
|
1380
2317
|
const targets = [];
|
|
2318
|
+
const missing = [];
|
|
1381
2319
|
for (const id of on) {
|
|
1382
|
-
const member = overlay.tasks.get(id) ?? this.#
|
|
1383
|
-
if (member === undefined)
|
|
1384
|
-
|
|
2320
|
+
const member = overlay.tasks.get(id) ?? this.#liveRecord(id) ?? ended.get(id);
|
|
2321
|
+
if (member === undefined) {
|
|
2322
|
+
missing.push(id);
|
|
2323
|
+
continue;
|
|
2324
|
+
}
|
|
2325
|
+
if (member === null)
|
|
2326
|
+
return { error: new Error(`Task ${id} does not exist`) };
|
|
1385
2327
|
if (policy === "failFast" && member.owner !== current.id) {
|
|
1386
|
-
|
|
2328
|
+
return { error: new Error(`Task ${current.id} can wait failFast only on tasks it owns; ${id} is not one`) };
|
|
1387
2329
|
}
|
|
1388
2330
|
// A live child already holds the waiter (task-owned tasks are never background): waiting on it adds no edge. A
|
|
1389
2331
|
// terminal task has no edges.
|
|
1390
2332
|
if (member.state.status !== "terminal" && member.owner !== current.id)
|
|
1391
2333
|
targets.push(id);
|
|
1392
2334
|
}
|
|
2335
|
+
if (missing.length > 0)
|
|
2336
|
+
return { missing };
|
|
1393
2337
|
if (targets.length === 0)
|
|
1394
|
-
return;
|
|
1395
|
-
const cycle =
|
|
1396
|
-
if (cycle
|
|
1397
|
-
|
|
1398
|
-
|
|
2338
|
+
return undefined;
|
|
2339
|
+
const cycle = this.#waitCycle(current.id, targets, overlay);
|
|
2340
|
+
if (cycle === undefined)
|
|
2341
|
+
return undefined;
|
|
2342
|
+
return {
|
|
2343
|
+
error: new Error(`Task ${current.id} cannot wait on ${cycle[0]}: ${[current.id, ...cycle].join(" -> ")} is a wait cycle`),
|
|
2344
|
+
};
|
|
1399
2345
|
}
|
|
1400
2346
|
/**
|
|
1401
2347
|
* The path from a member of `on` back to `task` in the wait-for graph, if the wait would close a cycle (spec §5.1).
|
|
1402
2348
|
* An edge X -> Y means X cannot settle before Y: Y is in the `on` of waiting X that is not abort-marked, or Y is live
|
|
1403
|
-
* ordinary owned work of X,
|
|
1404
|
-
*
|
|
1405
|
-
*
|
|
2349
|
+
* ordinary owned work of X, which holds X as `completing`. Each task is visited once and each subtree walked once,
|
|
2350
|
+
* so a check is bounded by the live work it reaches. Part of a decision region: it records every record it reads.
|
|
2351
|
+
* Owned work is found by walks down, which need the chains of every live task loaded.
|
|
1406
2352
|
*/
|
|
1407
|
-
|
|
1408
|
-
// Owned work is found by walks down, which need the chains of every live task loaded.
|
|
1409
|
-
await this.#loadScopes(false);
|
|
2353
|
+
#waitCycle(task, on, overlay) {
|
|
1410
2354
|
const from = new Map();
|
|
1411
2355
|
const walked = new Set();
|
|
1412
2356
|
const pending = [];
|
|
@@ -1427,7 +2371,7 @@ export class TaskScheduler {
|
|
|
1427
2371
|
for (let id = pending.pop(); id !== undefined; id = pending.pop()) {
|
|
1428
2372
|
if (id === task)
|
|
1429
2373
|
return path(id);
|
|
1430
|
-
const record = overlay.tasks.get(id) ?? this.#
|
|
2374
|
+
const record = overlay.tasks.get(id) ?? this.#liveRecord(id);
|
|
1431
2375
|
if (record === undefined || record.state.status === "terminal")
|
|
1432
2376
|
continue;
|
|
1433
2377
|
// An abort-marked task waits only for its owned work, as abort runs bottom-up (`#waitingOn`).
|
|
@@ -1464,12 +2408,18 @@ export class TaskScheduler {
|
|
|
1464
2408
|
}
|
|
1465
2409
|
for (const watch of invocation.watches)
|
|
1466
2410
|
void watch.stop();
|
|
2411
|
+
for (const disarm of invocation.timers)
|
|
2412
|
+
disarm();
|
|
2413
|
+
invocation.timers.clear();
|
|
1467
2414
|
// Pending waits bound to the invocation, such as a tool's waitForTask(), reject with it.
|
|
1468
2415
|
invocation.controller.abort(endedError(invocation));
|
|
1469
2416
|
}
|
|
1470
2417
|
/** No live non-background task in the scope; a task whose owner edges are not loaded yet counts as inside. */
|
|
1471
2418
|
#idle(conversationId) {
|
|
1472
2419
|
const scope = conversationId === undefined ? { roots: true } : { conversation: conversationId };
|
|
2420
|
+
// In a decision region, global idle reads the roots it starts from.
|
|
2421
|
+
if (conversationId === undefined)
|
|
2422
|
+
this.#record(ROOTS_KEY);
|
|
1473
2423
|
// Tasks whose chain is not loaded are not listed below their scope yet; one counts as inside unless its walk up
|
|
1474
2424
|
// decides otherwise.
|
|
1475
2425
|
for (const id of this.#unloaded) {
|
|
@@ -1501,7 +2451,7 @@ export class TaskScheduler {
|
|
|
1501
2451
|
};
|
|
1502
2452
|
const hooks = {
|
|
1503
2453
|
each: async (name, invoke) => {
|
|
1504
|
-
for (const handlers of agentHooks(await agent(), phase.task().definition.name)) {
|
|
2454
|
+
for (const { extension, handlers } of agentHooks(await agent(), phase.task().definition.name)) {
|
|
1505
2455
|
const handler = handlers[name];
|
|
1506
2456
|
if (typeof handler !== "function")
|
|
1507
2457
|
continue;
|
|
@@ -1511,17 +2461,20 @@ export class TaskScheduler {
|
|
|
1511
2461
|
catch (error) {
|
|
1512
2462
|
if (invocation.controller.signal.aborted)
|
|
1513
2463
|
throw error;
|
|
1514
|
-
this.#report(error);
|
|
2464
|
+
this.#report(error, { kind: "extension_error", extension, task: reportTask(invocation) }, invocation.context);
|
|
1515
2465
|
}
|
|
1516
2466
|
}
|
|
1517
2467
|
},
|
|
1518
2468
|
};
|
|
1519
2469
|
const settings = this.#settings;
|
|
2470
|
+
const models = this.#models;
|
|
1520
2471
|
return {
|
|
1521
2472
|
taskId: invocation.taskId,
|
|
1522
2473
|
conversationId: invocation.conversationId,
|
|
1523
2474
|
signal: invocation.controller.signal,
|
|
1524
|
-
models
|
|
2475
|
+
get models() {
|
|
2476
|
+
return models();
|
|
2477
|
+
},
|
|
1525
2478
|
agent: (context) => invocation.ended ? Promise.reject(endedError(invocation)) : awaitWithContext(agent(), context),
|
|
1526
2479
|
get settings() {
|
|
1527
2480
|
return settings();
|
|
@@ -1531,11 +2484,46 @@ export class TaskScheduler {
|
|
|
1531
2484
|
get registry() {
|
|
1532
2485
|
return phase.snapshot();
|
|
1533
2486
|
},
|
|
1534
|
-
commit: (change, context
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
2487
|
+
commit: async (change, context, options) => {
|
|
2488
|
+
let checkpoint;
|
|
2489
|
+
let interrupted;
|
|
2490
|
+
await this.#gated(invocation, async (tx, current) => {
|
|
2491
|
+
const next = await change(tx, current);
|
|
2492
|
+
if (next !== undefined)
|
|
2493
|
+
await this.#commitState(tx, invocation, current, next);
|
|
2494
|
+
if (next?.status === "running" && current.state.checkpoint.phase !== next.checkpoint.phase) {
|
|
2495
|
+
checkpoint = next.checkpoint.phase;
|
|
2496
|
+
}
|
|
2497
|
+
// A task that ends for crashes says so (`task_interrupted`), so a host keeps a crash-loop signal.
|
|
2498
|
+
if (next?.status === "terminal") {
|
|
2499
|
+
const restarts = current.restarts ?? 0;
|
|
2500
|
+
const reason = next.outcome.status === "failed" ? interruptionOf(next.outcome.error) : undefined;
|
|
2501
|
+
if (reason !== undefined && next.outcome.status === "failed") {
|
|
2502
|
+
interrupted = { message: next.outcome.error.message, reason, restarts };
|
|
2503
|
+
}
|
|
2504
|
+
else {
|
|
2505
|
+
// A task that ended for crashes with another outcome says so itself, as a generation whose run
|
|
2506
|
+
// answered ends `completed` without its after-run tasks.
|
|
2507
|
+
const message = endedByCrashesStates.get(next);
|
|
2508
|
+
if (message !== undefined)
|
|
2509
|
+
interrupted = { message, reason: "restart_limit", restarts };
|
|
2510
|
+
}
|
|
2511
|
+
}
|
|
2512
|
+
}, context, options);
|
|
2513
|
+
if (interrupted !== undefined) {
|
|
2514
|
+
this.#report(new Error(interrupted.message), {
|
|
2515
|
+
kind: "task_interrupted",
|
|
2516
|
+
...this.#aboutTask(invocation),
|
|
2517
|
+
reason: interrupted.reason,
|
|
2518
|
+
restarts: interrupted.restarts,
|
|
2519
|
+
}, invocation.context);
|
|
2520
|
+
}
|
|
2521
|
+
if (checkpoint !== undefined) {
|
|
2522
|
+
invocation.context.telemetry.addEvent("pi.durable.checkpoint", {
|
|
2523
|
+
phase: checkpoint,
|
|
2524
|
+
});
|
|
2525
|
+
}
|
|
2526
|
+
},
|
|
1539
2527
|
memo: ((name, ...rest) => {
|
|
1540
2528
|
if (rest.length === 1) {
|
|
1541
2529
|
return this.#read(invocation, async () => memoOf(this.#live.get(invocation.taskId), name));
|
|
@@ -1549,11 +2537,11 @@ export class TaskScheduler {
|
|
|
1549
2537
|
return candidate;
|
|
1550
2538
|
}, rest[1]);
|
|
1551
2539
|
}),
|
|
1552
|
-
|
|
2540
|
+
sleepUntil: (until, checkpoint, context) => this.#gated(invocation, (tx, current) => this.#commitState(tx, invocation, current, { status: "sleeping", checkpoint, until }), context),
|
|
1553
2541
|
watchDoc: ((...args) => this.#watchDoc(invocation, args)),
|
|
1554
2542
|
snapshot: ((...args) => this.#read(invocation, () => sessionMethod(this.#session, "snapshot")(...args))),
|
|
1555
2543
|
snapshotAsOf: ((...args) => this.#read(invocation, () => sessionMethod(this.#session, "snapshotAsOf")(...args))),
|
|
1556
|
-
getTask: ((id, context) => this.#read(invocation, () => this.#session.readOnLine(() =>
|
|
2544
|
+
getTask: ((id, context) => this.#read(invocation, () => this.#session.readOnLine((storage) => storage.task(id, context)))),
|
|
1557
2545
|
waitForTask: ((id, context) => this.#read(invocation, () => this.waitForTask(id, withAbortSignal(invocation.controller.signal, context)))),
|
|
1558
2546
|
ownedTasks: (context) => this.#read(invocation, () => this.#session.readOnLine(async () => {
|
|
1559
2547
|
// The step that ends the invocation may have run on the line before this.
|
|
@@ -1561,12 +2549,16 @@ export class TaskScheduler {
|
|
|
1561
2549
|
throw endedError(invocation);
|
|
1562
2550
|
context.abortSignal?.throwIfAborted();
|
|
1563
2551
|
// Every live task is listed under its parent node, its owner task when it has one.
|
|
1564
|
-
const owned =
|
|
1565
|
-
|
|
1566
|
-
const
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
2552
|
+
const owned = await this.#region(() => {
|
|
2553
|
+
this.#record(ownedKey(invocation.taskId));
|
|
2554
|
+
const records = [];
|
|
2555
|
+
for (const node of this.#below.get(invocation.taskId) ?? []) {
|
|
2556
|
+
const record = typeof node === "string" ? undefined : this.#liveRecord(node);
|
|
2557
|
+
if (record?.owner === invocation.taskId)
|
|
2558
|
+
records.push(record);
|
|
2559
|
+
}
|
|
2560
|
+
return records;
|
|
2561
|
+
});
|
|
1570
2562
|
// Copies, as from storage: the scheduler reads its own.
|
|
1571
2563
|
return owned
|
|
1572
2564
|
.sort((a, b) => a.id - b.id)
|
|
@@ -1574,7 +2566,7 @@ export class TaskScheduler {
|
|
|
1574
2566
|
})),
|
|
1575
2567
|
abortOwned: (id, context) => this.#read(invocation, async () => {
|
|
1576
2568
|
const bound = withAbortSignal(invocation.controller.signal, context);
|
|
1577
|
-
const record = await this.#session.readOnLine(() =>
|
|
2569
|
+
const record = await this.#session.readOnLine((storage) => storage.task(id, bound));
|
|
1578
2570
|
if (record?.owner !== invocation.taskId) {
|
|
1579
2571
|
throw new Error(`Task ${id} is not owned by task ${invocation.taskId}`);
|
|
1580
2572
|
}
|
|
@@ -1583,10 +2575,10 @@ export class TaskScheduler {
|
|
|
1583
2575
|
await this.abort(id, bound, true);
|
|
1584
2576
|
await this.waitForTask(id, bound);
|
|
1585
2577
|
}),
|
|
1586
|
-
outcomes: ((ids, context) => this.#read(invocation, () => this.#session.readOnLine(async () => {
|
|
2578
|
+
outcomes: ((ids, context) => this.#read(invocation, () => this.#session.readOnLine(async (storage) => {
|
|
1587
2579
|
const outcomes = [];
|
|
1588
2580
|
for (const id of ids) {
|
|
1589
|
-
const state = (await
|
|
2581
|
+
const state = (await storage.task(id, context))?.state;
|
|
1590
2582
|
if (state?.status !== "terminal")
|
|
1591
2583
|
throw new Error(`Task ${id} is not terminal`);
|
|
1592
2584
|
outcomes.push(state.outcome);
|
|
@@ -1605,13 +2597,24 @@ export class TaskScheduler {
|
|
|
1605
2597
|
? [undefined, args[0], args[1]]
|
|
1606
2598
|
: [args[0], args[1], args[2]];
|
|
1607
2599
|
return this.#read(invocation, async () => {
|
|
1608
|
-
|
|
1609
|
-
const
|
|
2600
|
+
// An entry of the conversation's kept context range is committed and visible: no Storage read.
|
|
2601
|
+
const kept = this.#contexts.get(invocation.conversationId);
|
|
2602
|
+
const entry = (kept === undefined ? undefined : rangeEntry(kept.range, id)) ??
|
|
2603
|
+
(await this.#session.readOnLine((storage) => storage.entry(invocation.conversationId, id, context)))
|
|
2604
|
+
?.entry;
|
|
1610
2605
|
return token === undefined || entry?.kind === token.kind ? entry : undefined;
|
|
1611
2606
|
});
|
|
1612
2607
|
}),
|
|
2608
|
+
entries: (query, limit, cursor, context) => this.#read(invocation, () => this.#session.readOnLine((storage) => storage.scanEntries(conversationEntries(invocation.conversationId, query), limit, cursor, context))),
|
|
1613
2609
|
context: (conversationId, context, options) => this.#read(invocation, async () => {
|
|
1614
|
-
const
|
|
2610
|
+
const read = await readContextFrom(this.#session, this.#storage, conversationId, context, options?.at, this.#contexts.get(conversationId)?.range);
|
|
2611
|
+
const { view } = read;
|
|
2612
|
+
// Checked again here, synchronously up to the install below: a commit published since the read returned
|
|
2613
|
+
// was not seen by `#advanceContexts`, so a range that missed it must not be advanced.
|
|
2614
|
+
const range = read.range?.current === true &&
|
|
2615
|
+
this.#session.knownLatestEntryId(conversationId) !== read.range.bounds.tail
|
|
2616
|
+
? { ...read.range, current: false }
|
|
2617
|
+
: read.range;
|
|
1615
2618
|
// Keep it unless the invocation ended or a concurrent read already kept a newer range.
|
|
1616
2619
|
const kept = this.#contexts.get(conversationId);
|
|
1617
2620
|
if (range !== undefined &&
|
|
@@ -1621,9 +2624,17 @@ export class TaskScheduler {
|
|
|
1621
2624
|
// A read of another, idle conversation starts or continues its retention period.
|
|
1622
2625
|
if (!this.#idle(conversationId)) {
|
|
1623
2626
|
this.#contexts.set(conversationId, { range, idleSince: undefined });
|
|
2627
|
+
this.#idleOrder.delete(conversationId);
|
|
1624
2628
|
}
|
|
1625
2629
|
else if (this.#contextRetentionMs() > 0) {
|
|
1626
|
-
|
|
2630
|
+
// An idle period that already started keeps its expiry and its place; a new one starts now.
|
|
2631
|
+
const idleSince = kept?.idleSince ?? this.#now();
|
|
2632
|
+
const expiresAt = kept?.idleSince !== undefined && kept.expiresAt !== undefined
|
|
2633
|
+
? kept.expiresAt
|
|
2634
|
+
: idleSince + this.#contextRetentionMs();
|
|
2635
|
+
this.#contexts.set(conversationId, { range, idleSince, expiresAt });
|
|
2636
|
+
if (kept?.idleSince === undefined)
|
|
2637
|
+
this.#markIdle(conversationId, idleSince);
|
|
1627
2638
|
this.#scheduleExpiry();
|
|
1628
2639
|
}
|
|
1629
2640
|
}
|
|
@@ -1634,13 +2645,43 @@ export class TaskScheduler {
|
|
|
1634
2645
|
throw endedError(invocation);
|
|
1635
2646
|
return this.#now();
|
|
1636
2647
|
},
|
|
1637
|
-
|
|
2648
|
+
setTimer: (callback, delayMs) => {
|
|
2649
|
+
// An ended invocation's timers never fire.
|
|
2650
|
+
if (invocation.ended)
|
|
2651
|
+
return () => { };
|
|
2652
|
+
const disarm = this.#timers.foreground(() => {
|
|
2653
|
+
invocation.timers.delete(disarm);
|
|
2654
|
+
if (!invocation.ended)
|
|
2655
|
+
callback();
|
|
2656
|
+
}, delayMs);
|
|
2657
|
+
invocation.timers.add(disarm);
|
|
2658
|
+
return () => {
|
|
2659
|
+
invocation.timers.delete(disarm);
|
|
2660
|
+
disarm();
|
|
2661
|
+
};
|
|
2662
|
+
},
|
|
2663
|
+
report: (error, from) => {
|
|
1638
2664
|
if (invocation.ended)
|
|
1639
2665
|
throw endedError(invocation);
|
|
1640
|
-
|
|
2666
|
+
// The task's own failure names the task's extension; any other names whose code failed, if anyone's.
|
|
2667
|
+
const task = reportTask(invocation);
|
|
2668
|
+
this.#report(error, from === undefined
|
|
2669
|
+
? { kind: "task_error", ...this.#aboutTask(invocation) }
|
|
2670
|
+
: from.kind === "extension"
|
|
2671
|
+
? { kind: "extension_error", task, ...optionalExtension(from.extension) }
|
|
2672
|
+
: { kind: "host_error", callback: from.callback, task }, invocation.context);
|
|
1641
2673
|
},
|
|
1642
2674
|
};
|
|
1643
2675
|
}
|
|
2676
|
+
/**
|
|
2677
|
+
* On the line: wait until every commit admitted before is published, for a decision that reads what cannot be keyed,
|
|
2678
|
+
* every live task (open, inspection) or every task a conversation abort reaches.
|
|
2679
|
+
*/
|
|
2680
|
+
async #published() {
|
|
2681
|
+
const gate = this.#session.touch(ALL_KEYS);
|
|
2682
|
+
if (gate !== undefined)
|
|
2683
|
+
await gate;
|
|
2684
|
+
}
|
|
1644
2685
|
/** Run a committed-state read unless the invocation has ended. */
|
|
1645
2686
|
async #read(invocation, read) {
|
|
1646
2687
|
if (invocation.ended)
|
|
@@ -1648,7 +2689,7 @@ export class TaskScheduler {
|
|
|
1648
2689
|
return read();
|
|
1649
2690
|
}
|
|
1650
2691
|
/** Commit after rereading the task on the line and gating the invocation. */
|
|
1651
|
-
#gated(invocation, change, context) {
|
|
2692
|
+
#gated(invocation, change, context, options) {
|
|
1652
2693
|
if (invocation.ended)
|
|
1653
2694
|
return Promise.reject(endedError(invocation));
|
|
1654
2695
|
return this.#session.commitWith(async (tx) => {
|
|
@@ -1656,33 +2697,22 @@ export class TaskScheduler {
|
|
|
1656
2697
|
throw endedError(invocation);
|
|
1657
2698
|
if (this.#closing)
|
|
1658
2699
|
throw closedError(this.#session);
|
|
1659
|
-
const
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
2700
|
+
const current = await this.#region(() => {
|
|
2701
|
+
const found = this.#liveRecord(invocation.taskId);
|
|
2702
|
+
if (found === undefined)
|
|
2703
|
+
throw new Error(`Task ${invocation.taskId} is terminal`);
|
|
2704
|
+
if (found.state.status !== "running")
|
|
2705
|
+
throw new Error(`Task ${invocation.taskId} is ${found.state.status}`);
|
|
2706
|
+
if (invocation.mode === "run" && found.abortRequested) {
|
|
2707
|
+
throw new Error(`Task ${invocation.taskId} has a durable abort mark`);
|
|
2708
|
+
}
|
|
2709
|
+
return found;
|
|
2710
|
+
});
|
|
2711
|
+
// The step that ends it may have run while the region waited.
|
|
2712
|
+
if (invocation.ended)
|
|
2713
|
+
throw endedError(invocation);
|
|
1668
2714
|
return change(tx, current);
|
|
1669
|
-
}, context, { conversationId: invocation.conversationId, taskId: invocation.taskId });
|
|
1670
|
-
}
|
|
1671
|
-
/** Wait until the Harness clock reaches `until`, rechecking it after every timer. */
|
|
1672
|
-
async #sleep(invocation, until, context) {
|
|
1673
|
-
if (invocation.ended)
|
|
1674
|
-
throw endedError(invocation);
|
|
1675
|
-
const signals = [invocation.controller.signal];
|
|
1676
|
-
if (context.abortSignal !== undefined)
|
|
1677
|
-
signals.push(context.abortSignal);
|
|
1678
|
-
const signal = AbortSignal.any(signals);
|
|
1679
|
-
for (;;) {
|
|
1680
|
-
signal.throwIfAborted();
|
|
1681
|
-
const remaining = until - this.#now();
|
|
1682
|
-
if (remaining <= 0)
|
|
1683
|
-
return;
|
|
1684
|
-
await delay(Math.min(remaining, MAX_TIMER_DELAY), signal);
|
|
1685
|
-
}
|
|
2715
|
+
}, context, { conversationId: invocation.conversationId, taskId: invocation.taskId }, options);
|
|
1686
2716
|
}
|
|
1687
2717
|
async #watchDoc(invocation, args) {
|
|
1688
2718
|
if (invocation.ended)
|
|
@@ -1699,15 +2729,64 @@ export class TaskScheduler {
|
|
|
1699
2729
|
return watch;
|
|
1700
2730
|
}
|
|
1701
2731
|
}
|
|
1702
|
-
|
|
2732
|
+
const REGISTRY = { kind: "host_error", callback: "registry" };
|
|
2733
|
+
const SETTINGS = { kind: "host_error", callback: "settings" };
|
|
2734
|
+
const ON_WAKE = { kind: "host_error", callback: "onWake" };
|
|
2735
|
+
const TELEMETRY = { kind: "host_error", callback: "telemetry" };
|
|
2736
|
+
/** The extension that installed `task` in `snapshot`, the last one that lists it; none for a built-in task. */
|
|
2737
|
+
function installerOf(snapshot, task) {
|
|
2738
|
+
if (task === undefined)
|
|
2739
|
+
return undefined;
|
|
2740
|
+
return snapshot.installed().findLast((installed) => installed.tasks?.includes(task) === true)?.name;
|
|
2741
|
+
}
|
|
2742
|
+
/** The report info of a failure of a task. */
|
|
2743
|
+
function reportTask(task) {
|
|
2744
|
+
return { id: task.taskId, kind: task.kind, conversationId: task.conversationId };
|
|
2745
|
+
}
|
|
2746
|
+
/** A stored task's identity, as an invocation names it. */
|
|
2747
|
+
function recordTask(record) {
|
|
2748
|
+
return { taskId: record.id, kind: record.kind, conversationId: record.conversationId };
|
|
2749
|
+
}
|
|
2750
|
+
/**
|
|
2751
|
+
* The record with an abort mark: `restart` for an abandonment; without a reason, a request, which replaces `restart`.
|
|
2752
|
+
* The restart count starts over: it now counts cut-offs of the abort handler.
|
|
2753
|
+
*/
|
|
2754
|
+
/** Why a failed outcome's error says crashes ended the task (`detail.reason`), if it does. */
|
|
2755
|
+
/**
|
|
2756
|
+
* Terminal states a built-in task commits because it was cut off `settings.restartLimit` times, when their outcome does
|
|
2757
|
+
* not say so (a failure with detail `{ reason: "restart_limit" }` does), with the message the scheduler reports them
|
|
2758
|
+
* `task_interrupted` with. Keyed by the state object the task returns, so nothing is added to the stored state.
|
|
2759
|
+
*/
|
|
2760
|
+
const endedByCrashesStates = new WeakMap();
|
|
2761
|
+
/** `state`, marked as ending its task for crashes (`task_interrupted` with `message`), whatever its outcome. */
|
|
2762
|
+
export function endedByCrashes(state, message) {
|
|
2763
|
+
endedByCrashesStates.set(state, message);
|
|
2764
|
+
return state;
|
|
2765
|
+
}
|
|
2766
|
+
function interruptionOf(error) {
|
|
2767
|
+
const detail = error.detail;
|
|
2768
|
+
if (typeof detail !== "object" || detail === null || !("reason" in detail))
|
|
2769
|
+
return undefined;
|
|
2770
|
+
return detail.reason === "restart_limit" || detail.reason === "interrupted" ? detail.reason : undefined;
|
|
2771
|
+
}
|
|
1703
2772
|
function withAbortMark(record, reason) {
|
|
1704
|
-
const { abortReason: _reason, ...rest } = record;
|
|
2773
|
+
const { abortReason: _reason, restarts: _restarts, ...rest } = record;
|
|
1705
2774
|
return { ...rest, abortRequested: true, ...(reason === undefined ? {} : { abortReason: reason }) };
|
|
1706
2775
|
}
|
|
1707
2776
|
/** A live owner's durable cancellation intent: its abort mark, or a held outcome other than `completed`. */
|
|
1708
2777
|
function cancellationIntent(record) {
|
|
1709
2778
|
return record.state.status !== "terminal" && (record.abortRequested || failedOutcome(record));
|
|
1710
2779
|
}
|
|
2780
|
+
/**
|
|
2781
|
+
* Whether replacing `previous` (absent: a new task) with `next` changes the cancellation intent set or what it means: a
|
|
2782
|
+
* mark set, upgraded, or cleared, or a held or ended outcome other than `completed`.
|
|
2783
|
+
*/
|
|
2784
|
+
function intentChanges(previous, next) {
|
|
2785
|
+
return ((previous?.abortRequested ?? false) !== next.abortRequested ||
|
|
2786
|
+
previous?.abortReason !== next.abortReason ||
|
|
2787
|
+
(previous !== undefined && failedOutcome(previous)) !== failedOutcome(next) ||
|
|
2788
|
+
(previous !== undefined && cancellationIntent(previous)) !== cancellationIntent(next));
|
|
2789
|
+
}
|
|
1711
2790
|
/** Whether the record holds or ends with an outcome other than `completed`. */
|
|
1712
2791
|
function failedOutcome(record) {
|
|
1713
2792
|
const state = record.state;
|
|
@@ -1761,30 +2840,41 @@ function erased(task) {
|
|
|
1761
2840
|
function endedError(invocation) {
|
|
1762
2841
|
return new Error(`Task ${invocation.taskId} invocation has ended`);
|
|
1763
2842
|
}
|
|
1764
|
-
/**
|
|
2843
|
+
/** The record without its restart count, as after a state the task committed. */
|
|
2844
|
+
function progressed(record) {
|
|
2845
|
+
if (record.restarts === undefined)
|
|
2846
|
+
return record;
|
|
2847
|
+
const { restarts: _restarts, ...rest } = record;
|
|
2848
|
+
return rest;
|
|
2849
|
+
}
|
|
2850
|
+
/** Replace a live record's state; memos and the restart count disappear once an outcome is decided. */
|
|
1765
2851
|
function withState(record, state) {
|
|
1766
2852
|
if (state.status !== "terminal" && state.status !== "completing")
|
|
1767
2853
|
return { ...record, state };
|
|
1768
|
-
const { memos: _memos, ...rest } = record;
|
|
2854
|
+
const { memos: _memos, restarts: _restarts, ...rest } = record;
|
|
1769
2855
|
return { ...rest, state };
|
|
1770
2856
|
}
|
|
1771
2857
|
/** Whether a definition can take the task at reservation: same version, or newer with a migration. */
|
|
2858
|
+
/**
|
|
2859
|
+
* Why a replacement definition cannot take a running task, in `#fit()`'s words: none is registered, it is older than
|
|
2860
|
+
* the task, or it is newer without a `migrate`.
|
|
2861
|
+
*/
|
|
2862
|
+
function outdatedReason(task, record) {
|
|
2863
|
+
if (task === undefined)
|
|
2864
|
+
return "missing_task";
|
|
2865
|
+
return task.definition.version < record.version ? "task_too_old" : "migration_failed";
|
|
2866
|
+
}
|
|
1772
2867
|
function canReserve(task, record) {
|
|
1773
2868
|
const definition = task.definition;
|
|
1774
2869
|
return (definition.version === record.version || (definition.version > record.version && definition.migrate !== undefined));
|
|
1775
2870
|
}
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
};
|
|
1782
|
-
|
|
1783
|
-
signal.removeEventListener("abort", onAbort);
|
|
1784
|
-
resolve();
|
|
1785
|
-
}, ms);
|
|
1786
|
-
signal.addEventListener("abort", onAbort, { once: true });
|
|
1787
|
-
});
|
|
2871
|
+
/** A sleep's `until` is a time on the Harness clock; an abort handler cannot sleep. */
|
|
2872
|
+
function validateSleep(invocation, task, until) {
|
|
2873
|
+
if (invocation.mode === "abort")
|
|
2874
|
+
throw new Error(`Abort handler of task ${task} cannot sleep`);
|
|
2875
|
+
if (typeof until !== "number" || !Number.isFinite(until) || until < 0) {
|
|
2876
|
+
throw new Error(`Task ${task} cannot sleep until ${until}: not a time in milliseconds`);
|
|
2877
|
+
}
|
|
1788
2878
|
}
|
|
1789
2879
|
/** Structural equality of two JSON values; object key order is ignored. */
|
|
1790
2880
|
function jsonEqual(left, right) {
|