@plurnk/plurnk-service 1.6.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.defaults +7 -21
- package/INSTALL.md +18 -17
- package/README.md +1 -1
- package/SPEC.md +209 -136
- package/dist/build-info.json +1 -1
- package/dist/core/ChannelWrite.sql +3 -3
- package/dist/core/ClientInteractions.d.ts +19 -0
- package/dist/core/ClientInteractions.d.ts.map +1 -0
- package/dist/core/ClientInteractions.js +124 -0
- package/dist/core/ClientInteractions.js.map +1 -0
- package/dist/core/ClientInteractions.sql +34 -0
- package/dist/core/CoreSchemeServices.d.ts +9 -3
- package/dist/core/CoreSchemeServices.d.ts.map +1 -1
- package/dist/core/CoreSchemeServices.js +8 -2
- package/dist/core/CoreSchemeServices.js.map +1 -1
- package/dist/core/Dispatcher.d.ts +4 -2
- package/dist/core/Dispatcher.d.ts.map +1 -1
- package/dist/core/Dispatcher.js +116 -1149
- package/dist/core/Dispatcher.js.map +1 -1
- package/dist/core/Engine.d.ts +24 -34
- package/dist/core/Engine.d.ts.map +1 -1
- package/dist/core/Engine.js +137 -1672
- package/dist/core/Engine.js.map +1 -1
- package/dist/core/Engine.sql +46 -32
- package/dist/core/ExecutableTools.d.ts +1 -0
- package/dist/core/ExecutableTools.d.ts.map +1 -1
- package/dist/core/ExecutableTools.js +17 -9
- package/dist/core/ExecutableTools.js.map +1 -1
- package/dist/core/ExecutorRegistry.d.ts +11 -3
- package/dist/core/ExecutorRegistry.d.ts.map +1 -1
- package/dist/core/ExecutorRegistry.js +139 -16
- package/dist/core/ExecutorRegistry.js.map +1 -1
- package/dist/core/LogBody.d.ts +1 -0
- package/dist/core/LogBody.d.ts.map +1 -1
- package/dist/core/LogBody.js +4 -0
- package/dist/core/LogBody.js.map +1 -1
- package/dist/core/ModelCall.d.ts +2 -2
- package/dist/core/ModelCall.d.ts.map +1 -1
- package/dist/core/ModelCall.js +4 -2
- package/dist/core/ModelCall.js.map +1 -1
- package/dist/core/PacketBuilder.d.ts +11 -22
- package/dist/core/PacketBuilder.d.ts.map +1 -1
- package/dist/core/PacketBuilder.js +90 -180
- package/dist/core/PacketBuilder.js.map +1 -1
- package/dist/core/ProblemLog.d.ts +1 -1
- package/dist/core/ProblemLog.d.ts.map +1 -1
- package/dist/core/ProblemLog.js +14 -3
- package/dist/core/ProblemLog.js.map +1 -1
- package/dist/core/ProposalLifecycle.d.ts +4 -2
- package/dist/core/ProposalLifecycle.d.ts.map +1 -1
- package/dist/core/ProposalLifecycle.js +19 -7
- package/dist/core/ProposalLifecycle.js.map +1 -1
- package/dist/core/ProviderInstantiate.js +2 -2
- package/dist/core/ProviderInstantiate.js.map +1 -1
- package/dist/core/ResourceMutations.d.ts +73 -0
- package/dist/core/ResourceMutations.d.ts.map +1 -0
- package/dist/core/ResourceMutations.js +1142 -0
- package/dist/core/ResourceMutations.js.map +1 -0
- package/dist/core/SchemeRegistry.d.ts +18 -10
- package/dist/core/SchemeRegistry.d.ts.map +1 -1
- package/dist/core/SchemeRegistry.js +195 -43
- package/dist/core/SchemeRegistry.js.map +1 -1
- package/dist/core/StoredPacket.d.ts +2 -2
- package/dist/core/StoredPacket.js +7 -7
- package/dist/core/TurnRunner.d.ts +74 -0
- package/dist/core/TurnRunner.d.ts.map +1 -0
- package/dist/core/TurnRunner.js +1733 -0
- package/dist/core/TurnRunner.js.map +1 -0
- package/dist/core/WorkspaceGate.d.ts +1 -0
- package/dist/core/WorkspaceGate.d.ts.map +1 -1
- package/dist/core/WorkspaceGate.js +39 -0
- package/dist/core/WorkspaceGate.js.map +1 -1
- package/dist/core/caps/CoreInteractionCaps.d.ts +9 -0
- package/dist/core/caps/CoreInteractionCaps.d.ts.map +1 -0
- package/dist/core/caps/CoreInteractionCaps.js +14 -0
- package/dist/core/caps/CoreInteractionCaps.js.map +1 -0
- package/dist/core/caps/DbChannelCaps.js +4 -4
- package/dist/core/caps/DbChannelCaps.js.map +1 -1
- package/dist/core/caps/SchemeCtxImpl.d.ts +2 -1
- package/dist/core/caps/SchemeCtxImpl.d.ts.map +1 -1
- package/dist/core/caps/SchemeCtxImpl.js +3 -0
- package/dist/core/caps/SchemeCtxImpl.js.map +1 -1
- package/dist/core/content-hash.js +2 -2
- package/dist/core/content-hash.js.map +1 -1
- package/dist/core/content-weight.d.ts +2 -0
- package/dist/core/content-weight.d.ts.map +1 -0
- package/dist/core/content-weight.js +4 -0
- package/dist/core/content-weight.js.map +1 -0
- package/dist/core/content_weight.d.ts +4 -0
- package/dist/core/content_weight.d.ts.map +1 -0
- package/dist/core/content_weight.js +6 -0
- package/dist/core/content_weight.js.map +1 -0
- package/dist/core/fork.sql +5 -5
- package/dist/core/packet-wire.d.ts +4 -4
- package/dist/core/packet-wire.d.ts.map +1 -1
- package/dist/core/packet-wire.js +21 -21
- package/dist/core/packet-wire.js.map +1 -1
- package/dist/core/scheme-types.d.ts +3 -2
- package/dist/core/scheme-types.d.ts.map +1 -1
- package/dist/core/scheme-types.js.map +1 -1
- package/dist/digest/Digest.js +1 -1
- package/dist/digest/Digest.js.map +1 -1
- package/dist/schemes/Exec.d.ts.map +1 -1
- package/dist/schemes/Exec.js +53 -15
- package/dist/schemes/Exec.js.map +1 -1
- package/dist/schemes/ExecOutputScheme.js +1 -1
- package/dist/schemes/ExecOutputScheme.js.map +1 -1
- package/dist/schemes/File.d.ts.map +1 -1
- package/dist/schemes/File.js +1 -1
- package/dist/schemes/File.js.map +1 -1
- package/dist/schemes/Log.d.ts.map +1 -1
- package/dist/schemes/Log.js +2 -2
- package/dist/schemes/Log.sql +2 -2
- package/dist/schemes/Worker.js +2 -2
- package/dist/schemes/Worker.js.map +1 -1
- package/dist/schemes/_entry-crud.js +4 -4
- package/dist/schemes/_entry-crud.js.map +1 -1
- package/dist/schemes/_entry-crud.sql +2 -2
- package/dist/schemes/_entry-find.d.ts +5 -5
- package/dist/schemes/_entry-find.d.ts.map +1 -1
- package/dist/schemes/_entry-find.js +29 -28
- package/dist/schemes/_entry-find.js.map +1 -1
- package/dist/schemes/_entry-manifest.js +4 -4
- package/dist/schemes/_entry-manifest.js.map +1 -1
- package/dist/schemes/_entry-ops.d.ts.map +1 -1
- package/dist/schemes/_entry-ops.js +3 -3
- package/dist/schemes/_entry-ops.js.map +1 -1
- package/dist/schemes/_entry-ops.sql +3 -3
- package/dist/server/Daemon.d.ts +20 -49
- package/dist/server/Daemon.d.ts.map +1 -1
- package/dist/server/Daemon.js +311 -908
- package/dist/server/Daemon.js.map +1 -1
- package/dist/server/DaemonModule.d.ts +31 -3
- package/dist/server/DaemonModule.d.ts.map +1 -1
- package/dist/server/DrainSupervisor.d.ts +117 -0
- package/dist/server/DrainSupervisor.d.ts.map +1 -0
- package/dist/server/DrainSupervisor.js +802 -0
- package/dist/server/DrainSupervisor.js.map +1 -0
- package/dist/server/client-input.d.ts +2 -0
- package/dist/server/client-input.d.ts.map +1 -1
- package/dist/server/client-input.js +12 -0
- package/dist/server/client-input.js.map +1 -1
- package/dist/server/lifecycle-recovery.sql +9 -0
- package/dist/server/logEntry.d.ts +1 -1
- package/dist/server/logEntry.js +1 -1
- package/dist/server/loopDocs.d.ts +1 -0
- package/dist/server/loopDocs.d.ts.map +1 -1
- package/dist/server/loopDocs.js +37 -8
- package/dist/server/loopDocs.js.map +1 -1
- package/dist/server/loopDocs.sql +10 -0
- package/dist/server/seam-entry-read.sql +2 -2
- package/dist/server/workspace-capabilities.sql +19 -0
- package/dist/service.js +1 -1
- package/dist/service.js.map +1 -1
- package/migrations/001_schema.sql +51 -10
- package/package.json +34 -33
- package/dist/core/ruler_count.d.ts +0 -4
- package/dist/core/ruler_count.d.ts.map +0 -1
- package/dist/core/ruler_count.js +0 -7
- package/dist/core/ruler_count.js.map +0 -1
- package/dist/core/token-ruler.d.ts +0 -2
- package/dist/core/token-ruler.d.ts.map +0 -1
- package/dist/core/token-ruler.js +0 -5
- package/dist/core/token-ruler.js.map +0 -1
package/dist/server/Daemon.js
CHANGED
|
@@ -2,16 +2,13 @@
|
|
|
2
2
|
// the transport-free plugin-module seam ({§rpc}).
|
|
3
3
|
import { readFile } from "node:fs/promises";
|
|
4
4
|
import { resolve, dirname } from "node:path";
|
|
5
|
-
import
|
|
6
|
-
import { execPollBackoffMs } from "./exec-poll-backoff.js";
|
|
7
|
-
import ChannelWrite from "../core/ChannelWrite.js";
|
|
8
|
-
import { Paths } from "../index.js";
|
|
5
|
+
import Paths from "../Paths.js";
|
|
9
6
|
import Engine from "../core/Engine.js";
|
|
10
7
|
import ExecutorRegistry from "../core/ExecutorRegistry.js";
|
|
11
8
|
import SchemeRegistry from "../core/SchemeRegistry.js";
|
|
12
9
|
import { Mimetypes } from "@plurnk/plurnk-mimetypes";
|
|
13
10
|
import { RuntimeDeclaration } from "@plurnk/plurnk-execs";
|
|
14
|
-
import
|
|
11
|
+
import DrainSupervisor from "./DrainSupervisor.js";
|
|
15
12
|
import { parsePath, Validator, } from "@plurnk/plurnk-contracts";
|
|
16
13
|
import LogEntry from "./logEntry.js";
|
|
17
14
|
import Envelope from "./envelope.js";
|
|
@@ -23,19 +20,15 @@ import Fork from "../core/fork.js";
|
|
|
23
20
|
import WorkerName from "../core/WorkerName.js";
|
|
24
21
|
import LoopLifecycle from "../core/LoopLifecycle.js";
|
|
25
22
|
import { promptLoopPrefix } from "../core/plurnk-uri.js";
|
|
26
|
-
import {
|
|
23
|
+
import { contentWeight } from "../core/content-weight.js";
|
|
27
24
|
import { parseAliasesFromEnv, resolveActiveAlias } from "@plurnk/plurnk-providers";
|
|
28
25
|
import ProviderInstantiate from "../core/ProviderInstantiate.js";
|
|
29
26
|
import { resolveLoopAlias } from "./loop-model.js";
|
|
30
|
-
import { DEFAULT_LOOP_FLAGS } from "../core/scheme-types.js";
|
|
31
27
|
import LoopFlagsReader from "../core/LoopFlagsReader.js";
|
|
32
28
|
import Results, { OperationFailureError } from "../core/results.js";
|
|
33
29
|
import WorkspaceGate from "../core/WorkspaceGate.js";
|
|
34
30
|
import BranchBatches from "./BranchBatches.js";
|
|
35
|
-
import ErrorDetail from "../core/ErrorDetail.js";
|
|
36
31
|
import { observed, observedSync } from "../observe/spans.js";
|
|
37
|
-
import { LOOP_TERMINALS, recordCounter } from "../observe/metrics.js";
|
|
38
|
-
import { readOptimisticSettlementMs } from "../core/optimistic-settlement.js";
|
|
39
32
|
const clientActionFailure = (error) => {
|
|
40
33
|
if (error instanceof OperationFailureError)
|
|
41
34
|
return error.result;
|
|
@@ -53,6 +46,7 @@ export default class Daemon {
|
|
|
53
46
|
#workspaceGate;
|
|
54
47
|
#branchBatches;
|
|
55
48
|
#lifecycle;
|
|
49
|
+
#drains;
|
|
56
50
|
#schemes;
|
|
57
51
|
#mimetypes;
|
|
58
52
|
#ownsMimetypes;
|
|
@@ -64,44 +58,11 @@ export default class Daemon {
|
|
|
64
58
|
#modules = [];
|
|
65
59
|
#moduleClosers = [];
|
|
66
60
|
#moduleActions = new Map();
|
|
61
|
+
#workspaceCapabilityProviders = new Map();
|
|
67
62
|
// {§methods-event-subscribe} — the broadcast's in-process event source. A transport
|
|
68
63
|
// module (plurnk-agui) subscribes and fans out to its OWN clients; core emits, never owns
|
|
69
64
|
// client transport or connection state.
|
|
70
65
|
#eventSubscribers = new Set();
|
|
71
|
-
// Worker-level drain registry. At most one drain per worker. The stored object
|
|
72
|
-
// is the drain's identity handle: start/exit compare it by reference so a
|
|
73
|
-
// drain exiting never clobbers a successor that raced in, and a loop
|
|
74
|
-
// enqueued during teardown is never stranded. A drain is a pure queue
|
|
75
|
-
// consumer (claim → run → exit on empty queue); streams live independently
|
|
76
|
-
// (subscriptions + Exec.idle), and a concluding stream routes through
|
|
77
|
-
// inject() like any other loop source.
|
|
78
|
-
#activeDrains = new Map();
|
|
79
|
-
#drainExitTasks = new Set();
|
|
80
|
-
// Per-worker cancellation scope. Loops AND the streams they spawn (execs)
|
|
81
|
-
// share this signal, so loop.cancel / shutdown abort it once and every
|
|
82
|
-
// in-flight subscription tears down — even a spawn that registers AFTER the
|
|
83
|
-
// cancel self-aborts against the already-aborted signal (no race). Outlives
|
|
84
|
-
// any single (ephemeral) drain; replaced with a fresh controller once
|
|
85
|
-
// aborted so a later runLoop request isn't born cancelled.
|
|
86
|
-
#workerAborts = new Map();
|
|
87
|
-
// grammar 0.74.20 EXEC `<T,P>` — per-worker hibernation poll-wake timer. When a loop parks at
|
|
88
|
-
// a park with a polled stream, a timer fires every P seconds to resume it ({§exec-poll}). One
|
|
89
|
-
// per worker (the tightest cadence); cleared/replaced on each park and on cancel.
|
|
90
|
-
#parkTimers = new Map();
|
|
91
|
-
#pollTimers = new Map();
|
|
92
|
-
#pollBackoff = new Map(); // {§exec-poll} — backoff step per worker
|
|
93
|
-
// Per-worker drain-transition lock — see #withDrainLock (R4 / {§worker-lifecycle-single-drain}).
|
|
94
|
-
#drainLocks = new Map();
|
|
95
|
-
// {§worker-lifecycle-child-wake} — workers owed a wake: a child/stream conclusion fired while the worker was
|
|
96
|
-
// mid-turn (not yet slept), so #wakeParkedWorker could not resume it. A child-worker conclusion is a
|
|
97
|
-
// BOUNDED, lossless wake (a worker always concludes), so a hibernation awaiting one MUST return —
|
|
98
|
-
// never deadlock. The drain honors the owed wake at the worker's next park, closing the conclude-
|
|
99
|
-
// before-park race. (Only a live exec stream, unbounded absent a timeout, may hold a park open.)
|
|
100
|
-
#owedWakes = new Set();
|
|
101
|
-
// {§worker-optimistic-settlement} — one non-sliding completion-wake batch
|
|
102
|
-
// per worker. Durable conclusions and client events land before this gate;
|
|
103
|
-
// only the parked loop's provider-dispatch requeue waits.
|
|
104
|
-
#completionWakeGates = new Map();
|
|
105
66
|
constructor({ db, schemes, mimetypes, provider, nodeModulesPath, }) {
|
|
106
67
|
this.#db = db;
|
|
107
68
|
this.#lifecycle = new LoopLifecycle(db);
|
|
@@ -149,7 +110,7 @@ export default class Daemon {
|
|
|
149
110
|
}))?.id;
|
|
150
111
|
if (workerId === undefined)
|
|
151
112
|
throw new Error("Branch worker insert returned no row");
|
|
152
|
-
const loopId = await this.#enqueueFreshLoop({
|
|
113
|
+
const loopId = await this.#drains.enqueueFreshLoop({
|
|
153
114
|
workerId,
|
|
154
115
|
prompt,
|
|
155
116
|
providerSpec,
|
|
@@ -160,7 +121,7 @@ export default class Daemon {
|
|
|
160
121
|
},
|
|
161
122
|
startChild: async (workspaceId, workerId, loopId) => {
|
|
162
123
|
const systemPrompt = await readFile(Paths.instructionsSystem, "utf8");
|
|
163
|
-
const started = await this.#ensureDrain({ workspaceId, workerId, systemPrompt });
|
|
124
|
+
const started = await this.#drains.ensureDrain({ workspaceId, workerId, systemPrompt });
|
|
164
125
|
if (started === null)
|
|
165
126
|
throw new Error(`Branch worker ${workerId} already has a live drain`);
|
|
166
127
|
const result = await started.firstLoopPromise;
|
|
@@ -171,7 +132,7 @@ export default class Daemon {
|
|
|
171
132
|
},
|
|
172
133
|
wakeParent: async (workspaceId, workerId) => {
|
|
173
134
|
const systemPrompt = await readFile(Paths.instructionsSystem, "utf8");
|
|
174
|
-
await this.#settleCompletionWake(workspaceId, workerId, systemPrompt, false);
|
|
135
|
+
await this.#drains.settleCompletionWake(workspaceId, workerId, systemPrompt, false);
|
|
175
136
|
},
|
|
176
137
|
notify: (workspaceId, payload) => {
|
|
177
138
|
this.#broadcast({ workspaceId }, "workspace/branch-batch", payload);
|
|
@@ -179,13 +140,12 @@ export default class Daemon {
|
|
|
179
140
|
});
|
|
180
141
|
this.#engine = new Engine({
|
|
181
142
|
db, schemes: this.#schemes, mimetypes: this.#mimetypes,
|
|
182
|
-
// {§tokenomics-agnostic-ruler} —
|
|
183
|
-
//
|
|
184
|
-
//
|
|
185
|
-
|
|
186
|
-
tokenize: rulerCount,
|
|
143
|
+
// {§tokenomics-agnostic-ruler} — stored and catalog curation weights
|
|
144
|
+
// are workspace-wide across concurrent models, so they remain
|
|
145
|
+
// model-independent. Request-shaped token facts stay provider-owned.
|
|
146
|
+
weigh: contentWeight,
|
|
187
147
|
streamEventNotify: (workspaceId, event) => this.notifyStreamEvent(workspaceId, event),
|
|
188
|
-
wakeWorkerNotify: (payload) =>
|
|
148
|
+
wakeWorkerNotify: (payload) => this.#drains.notifyWakeWorker(payload),
|
|
189
149
|
// worker:// loop-start primitive — spawn/fork/irc deliver through
|
|
190
150
|
// Daemon.inject (active sister → fold; idle → enqueue + drain). The
|
|
191
151
|
// daemon owns provider + the law-file system prompt; the worker scheme
|
|
@@ -218,16 +178,68 @@ export default class Daemon {
|
|
|
218
178
|
workspaceTurnCompleted: async ({ turnId }) => this.#branchBatches.sealTurn(turnId),
|
|
219
179
|
// worker:// KILL (terminate) — cancel the addressed worker subtree and
|
|
220
180
|
// tear down its held streams before the operation completes.
|
|
221
|
-
cancelWorker: async (workerId, reason) => this.#cancelWorkerTree(workerId, reason),
|
|
222
|
-
cancelDescendants: async (workerId, reason) => this.#
|
|
181
|
+
cancelWorker: async (workerId, reason) => this.#drains.cancelWorkerTree(workerId, reason),
|
|
182
|
+
cancelDescendants: async (workerId, reason) => this.#drains.cancelDescendants(workerId, reason),
|
|
223
183
|
noticeNotify: (workspaceId, payload) => this.notifyNotice(workspaceId, payload),
|
|
224
184
|
});
|
|
185
|
+
this.#drains = new DrainSupervisor({
|
|
186
|
+
db,
|
|
187
|
+
lifecycle: this.#lifecycle,
|
|
188
|
+
injectPrompt: (workerId, prompt, openPaths) => this.#engine.inject(workerId, prompt, openPaths),
|
|
189
|
+
assertInjectionCompatibility: async ({ workerId, loopId, providerSpec, childProviderSpec, turnCeiling, flags, }) => {
|
|
190
|
+
await this.#assertFoldPosture(workerId, flags, loopId);
|
|
191
|
+
await this.#assertLoopProvider(loopId, providerSpec);
|
|
192
|
+
if (childProviderSpec !== undefined) {
|
|
193
|
+
await this.#assertLoopChildProvider(loopId, childProviderSpec);
|
|
194
|
+
}
|
|
195
|
+
await this.#assertLoopMaxTurns(loopId, turnCeiling?.source === "explicit" ? turnCeiling.effective : undefined);
|
|
196
|
+
},
|
|
197
|
+
reconcilePrompts: (workerId, endedLoopId) => this.#reconcileOrphanedPrompts(workerId, endedLoopId),
|
|
198
|
+
runLoop: async ({ workspaceId, workerId, loopId, maxTurns, prompt, systemPrompt, signal, onDispatch, }) => {
|
|
199
|
+
const { provider, childProvider } = await this.#providersForLoop(loopId);
|
|
200
|
+
return this.#engine.runLoop({
|
|
201
|
+
provider,
|
|
202
|
+
childProvider,
|
|
203
|
+
workspaceId,
|
|
204
|
+
workerId,
|
|
205
|
+
loopId,
|
|
206
|
+
maxTurns,
|
|
207
|
+
messages: [
|
|
208
|
+
{ role: "system", content: systemPrompt },
|
|
209
|
+
{ role: "user", content: prompt },
|
|
210
|
+
],
|
|
211
|
+
origin: "model",
|
|
212
|
+
signal,
|
|
213
|
+
onDispatch,
|
|
214
|
+
});
|
|
215
|
+
},
|
|
216
|
+
loopUsage: (loopId) => this.#engine.loopUsage(loopId),
|
|
217
|
+
loopAttributions: (loopId) => this.#engine.loopAttributions(loopId),
|
|
218
|
+
takeParkDeadline: (loopId) => {
|
|
219
|
+
const deadline = this.#engine.parkDeadlines.get(loopId);
|
|
220
|
+
this.#engine.parkDeadlines.delete(loopId);
|
|
221
|
+
return deadline;
|
|
222
|
+
},
|
|
223
|
+
settleSearch: (target, status) => this.#engine.searchGate.settle(target, status),
|
|
224
|
+
cancelSubscription: (subscriptionId) => this.#engine.cancelSubscription(subscriptionId),
|
|
225
|
+
hasActiveStreams: (workerId) => this.#workerHasActiveStreams(workerId),
|
|
226
|
+
readSystemPrompt: () => readFile(Paths.instructionsSystem, "utf8"),
|
|
227
|
+
emitLogEntry: async (workspaceId, logEntryId) => {
|
|
228
|
+
const entry = await LogEntry.fetchLogEntry(this.#db, logEntryId);
|
|
229
|
+
this.#broadcast({ workspaceId }, "log/entry", { entry });
|
|
230
|
+
},
|
|
231
|
+
emit: (workspaceId, method, params) => this.#broadcast({ workspaceId }, method, params),
|
|
232
|
+
});
|
|
225
233
|
// Wire proposal-pending events to the loop/proposal WS notification.
|
|
226
234
|
// Sessionid scopes the broadcast to clients on the same workspace.
|
|
227
235
|
this.#engine.onProposalPending((event) => {
|
|
228
236
|
const { workspaceId, ...proposal } = event;
|
|
229
237
|
this.#broadcast({ workspaceId }, "loop/proposal", proposal);
|
|
230
238
|
});
|
|
239
|
+
this.#engine.onClientInteractionPending((event) => {
|
|
240
|
+
const { workspaceId, ...interaction } = event;
|
|
241
|
+
this.#broadcast({ workspaceId }, "loop/interaction", interaction);
|
|
242
|
+
});
|
|
231
243
|
}
|
|
232
244
|
// {§methods-event-subscribe}. A transport module subscribes to the daemon's in-process
|
|
233
245
|
// event source: it receives every workspace-scoped engine event as `(workspaceId, method, params)`
|
|
@@ -251,6 +263,15 @@ export default class Daemon {
|
|
|
251
263
|
const checkedResolution = ClientInput.assertProposalResolution("resolveProposal", resolution);
|
|
252
264
|
this.#engine.resolveProposal(checkedLogEntryId, checkedResolution);
|
|
253
265
|
}
|
|
266
|
+
async pendingClientInteractions(workspaceId) {
|
|
267
|
+
const checkedWorkspaceId = ClientInput.assertId("pendingClientInteractions", "workspaceId", workspaceId);
|
|
268
|
+
return this.#engine.pendingClientInteractions(checkedWorkspaceId);
|
|
269
|
+
}
|
|
270
|
+
async resolveClientInteraction(interactionId, resolution) {
|
|
271
|
+
const checkedInteractionId = ClientInput.assertId("resolveClientInteraction", "interactionId", interactionId);
|
|
272
|
+
const checkedResolution = ClientInput.assertClientInteractionResolution("resolveClientInteraction", resolution);
|
|
273
|
+
await this.#engine.resolveClientInteraction(checkedInteractionId, checkedResolution);
|
|
274
|
+
}
|
|
254
275
|
// {§methods-loop-run} — drive/steer a loop. The module supplies only workspace/worker/prompt;
|
|
255
276
|
// the provider and the law-file system prompt are core's and stay inside. Returns immediately — the
|
|
256
277
|
// loop runs async and its outcome arrives on the event source (loop/terminated). `cancelDrain` (public)
|
|
@@ -615,9 +636,7 @@ export default class Daemon {
|
|
|
615
636
|
const isActive = active !== null && active.alias === a.alias;
|
|
616
637
|
return {
|
|
617
638
|
alias: a.alias, provider: a.provider, model: a.model, active: isActive,
|
|
618
|
-
|
|
619
|
-
// optional virtual pressure; known for the active alias, null elsewhere.
|
|
620
|
-
promptBudget: isActive && this.#provider !== null ? this.#engine.promptBudgetFor(this.#provider) : null,
|
|
639
|
+
inputCapacity: isActive && this.#provider !== null ? this.#provider.inputCapacity : null,
|
|
621
640
|
};
|
|
622
641
|
}),
|
|
623
642
|
};
|
|
@@ -694,6 +713,9 @@ export default class Daemon {
|
|
|
694
713
|
}
|
|
695
714
|
if (constraints.length > 0)
|
|
696
715
|
await GitMembership.resolveGitMembership(this.#db, envelope.workspaceId, undefined);
|
|
716
|
+
for (const provider of this.#workspaceCapabilityProviders.values()) {
|
|
717
|
+
await provider.hydrate(envelope.workspaceId);
|
|
718
|
+
}
|
|
697
719
|
await LoopDocs.materialize(this.#engine, this.#db, envelope.workspaceId);
|
|
698
720
|
void this.#engine.warmWorkspaceDerivations(envelope.workspaceId).catch(() => { });
|
|
699
721
|
this.#broadcast("all", "workspace/created", { id: envelope.workspaceId, name: envelope.workspaceName, projectRoot: envelope.projectRoot });
|
|
@@ -860,7 +882,7 @@ export default class Daemon {
|
|
|
860
882
|
contentOffset: c.contentOffset,
|
|
861
883
|
contentLength: c.contentLength,
|
|
862
884
|
mimetype: c.mimetype,
|
|
863
|
-
|
|
885
|
+
weight: c.weight,
|
|
864
886
|
state: c.state,
|
|
865
887
|
};
|
|
866
888
|
}
|
|
@@ -923,26 +945,18 @@ export default class Daemon {
|
|
|
923
945
|
const branch = await this.#db.envelope_get_worker_by_id.get({ id: branchWorkerId });
|
|
924
946
|
return { workerId: branchWorkerId, workerName: branch?.name ?? null, parentWorkerId: workerId };
|
|
925
947
|
}
|
|
926
|
-
async
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
}
|
|
930
|
-
const runtime = RuntimeDeclaration.assert(decl, namespaceOwner);
|
|
931
|
-
this.#engine.registerRuntime(runtime.name, {
|
|
932
|
-
executor,
|
|
933
|
-
namespaceOwner: { kind: "module", name: namespaceOwner },
|
|
934
|
-
glyph: runtime.glyph ?? "",
|
|
935
|
-
invocation: runtime.invocation,
|
|
936
|
-
documentation: runtime.documentation ?? "",
|
|
937
|
-
available: availability.available,
|
|
938
|
-
detail: availability.detail,
|
|
939
|
-
}, scheme);
|
|
948
|
+
async registerRuntimes(registrations) {
|
|
949
|
+
const normalized = registrations.map((registration) => this.#normalizeRuntime(registration));
|
|
950
|
+
this.#engine.registerRuntimes(normalized);
|
|
940
951
|
if (this.#capabilitiesPublished) {
|
|
941
952
|
for (const workspace of await Envelope.listWorkspaces(this.#db)) {
|
|
942
953
|
await LoopDocs.materialize(this.#engine, this.#db, workspace.id);
|
|
943
954
|
}
|
|
944
955
|
}
|
|
945
956
|
}
|
|
957
|
+
async registerRuntime(registration) {
|
|
958
|
+
await this.registerRuntimes([registration]);
|
|
959
|
+
}
|
|
946
960
|
async registerScheme(name, handler) {
|
|
947
961
|
this.#schemes.register(name, handler);
|
|
948
962
|
if (this.#capabilitiesPublished) {
|
|
@@ -952,21 +966,172 @@ export default class Daemon {
|
|
|
952
966
|
}
|
|
953
967
|
}
|
|
954
968
|
}
|
|
955
|
-
|
|
969
|
+
#normalizeRuntime({ namespaceOwner, decl, executor, availability, scheme }) {
|
|
970
|
+
if (typeof namespaceOwner !== "string" || namespaceOwner.trim().length === 0) {
|
|
971
|
+
throw new Error("runtime registration namespaceOwner must be a non-empty string");
|
|
972
|
+
}
|
|
973
|
+
const runtime = RuntimeDeclaration.assert(decl, namespaceOwner);
|
|
974
|
+
return {
|
|
975
|
+
tag: runtime.name,
|
|
976
|
+
entry: {
|
|
977
|
+
executor,
|
|
978
|
+
namespaceOwner: { kind: "module", name: namespaceOwner },
|
|
979
|
+
glyph: runtime.glyph ?? "",
|
|
980
|
+
invocation: runtime.invocation,
|
|
981
|
+
documentation: runtime.documentation ?? "",
|
|
982
|
+
available: availability.available,
|
|
983
|
+
detail: availability.detail,
|
|
984
|
+
},
|
|
985
|
+
scheme,
|
|
986
|
+
};
|
|
987
|
+
}
|
|
988
|
+
registerModuleAction(registration) {
|
|
989
|
+
const { name, scope, handler } = registration;
|
|
956
990
|
if (name.length === 0)
|
|
957
991
|
throw new Error("registerModuleAction: action name must not be empty");
|
|
992
|
+
if (scope !== "worldless" && scope !== "workspace") {
|
|
993
|
+
throw new Error(`module action '${name}' has invalid scope '${String(scope)}'`);
|
|
994
|
+
}
|
|
995
|
+
if (typeof handler !== "function")
|
|
996
|
+
throw new Error(`module action '${name}' has no handler`);
|
|
958
997
|
if (this.#moduleActions.has(name))
|
|
959
998
|
throw new Error(`module action '${name}' is already registered`);
|
|
960
|
-
this.#moduleActions.set(name,
|
|
999
|
+
this.#moduleActions.set(name, registration);
|
|
961
1000
|
}
|
|
962
1001
|
listModuleActions() {
|
|
963
|
-
return [...this.#moduleActions.
|
|
1002
|
+
return [...this.#moduleActions.values()]
|
|
1003
|
+
.map(({ name, scope }) => ({ name, scope }))
|
|
1004
|
+
.toSorted((left, right) => left.name.localeCompare(right.name));
|
|
964
1005
|
}
|
|
965
|
-
async invokeModuleAction(name, params) {
|
|
966
|
-
const
|
|
967
|
-
if (
|
|
1006
|
+
async invokeModuleAction(name, params, context) {
|
|
1007
|
+
const registration = this.#moduleActions.get(name);
|
|
1008
|
+
if (registration === undefined)
|
|
968
1009
|
throw new Error(`module action '${name}' is not registered`);
|
|
969
|
-
|
|
1010
|
+
if (registration.scope !== context.scope) {
|
|
1011
|
+
throw new Error(`module action '${name}' requires ${registration.scope} context, not ${context.scope}`);
|
|
1012
|
+
}
|
|
1013
|
+
if (context.scope === "workspace") {
|
|
1014
|
+
ClientInput.assertId(`module action '${name}'`, "workspaceId", context.workspaceId);
|
|
1015
|
+
}
|
|
1016
|
+
return registration.handler(params, context);
|
|
1017
|
+
}
|
|
1018
|
+
registerWorkspaceCapabilityProvider(namespaceOwner, provider) {
|
|
1019
|
+
if (namespaceOwner.trim().length === 0) {
|
|
1020
|
+
throw new Error("workspace capability provider requires a non-empty namespace owner");
|
|
1021
|
+
}
|
|
1022
|
+
if (this.#workspaceCapabilityProviders.has(namespaceOwner)) {
|
|
1023
|
+
throw new Error(`workspace capability provider '${namespaceOwner}' is already registered`);
|
|
1024
|
+
}
|
|
1025
|
+
this.#workspaceCapabilityProviders.set(namespaceOwner, provider);
|
|
1026
|
+
}
|
|
1027
|
+
async readWorkspaceModuleState(workspaceId, namespaceOwner) {
|
|
1028
|
+
const checkedWorkspaceId = ClientInput.assertId("workspace module state", "workspaceId", workspaceId);
|
|
1029
|
+
if (namespaceOwner.trim().length === 0) {
|
|
1030
|
+
throw new Error("workspace module state requires a non-empty namespace owner");
|
|
1031
|
+
}
|
|
1032
|
+
const row = await this.#db.workspace_module_state_get.get({
|
|
1033
|
+
workspace_id: checkedWorkspaceId,
|
|
1034
|
+
namespace_owner: namespaceOwner,
|
|
1035
|
+
});
|
|
1036
|
+
return row === undefined ? null : JSON.parse(row.state);
|
|
1037
|
+
}
|
|
1038
|
+
async replaceWorkspaceCapabilities({ workspaceId, namespaceOwner, state, runtimes, }) {
|
|
1039
|
+
const checkedWorkspaceId = ClientInput.assertId("workspace capability replacement", "workspaceId", workspaceId);
|
|
1040
|
+
if (namespaceOwner.trim().length === 0) {
|
|
1041
|
+
throw new Error("workspace capability replacement requires a non-empty namespace owner");
|
|
1042
|
+
}
|
|
1043
|
+
const workspace = await this.#db.envelope_get_workspace.get({ id: checkedWorkspaceId });
|
|
1044
|
+
if (workspace === undefined) {
|
|
1045
|
+
throw daemonFailure("daemon:workspace-capability", "workspace-not-found", 404, `Workspace ${checkedWorkspaceId} does not exist.`, { workspaceId: checkedWorkspaceId, retryable: false });
|
|
1046
|
+
}
|
|
1047
|
+
const encoded = state === null ? null : JSON.stringify(state);
|
|
1048
|
+
if (state !== null && encoded === undefined) {
|
|
1049
|
+
throw daemonFailure("daemon:workspace-capability", "state-not-json", 400, "Workspace module state is not JSON-serializable.", { namespaceOwner, retryable: false });
|
|
1050
|
+
}
|
|
1051
|
+
if (encoded !== null)
|
|
1052
|
+
JSON.parse(encoded);
|
|
1053
|
+
const normalized = runtimes.map((registration) => {
|
|
1054
|
+
if (registration.namespaceOwner !== namespaceOwner) {
|
|
1055
|
+
throw new Error(`workspace runtime owner '${registration.namespaceOwner}' does not match '${namespaceOwner}'`);
|
|
1056
|
+
}
|
|
1057
|
+
return this.#normalizeRuntime(registration);
|
|
1058
|
+
});
|
|
1059
|
+
const gate = this.#workspaceGate.tryExclusive(checkedWorkspaceId);
|
|
1060
|
+
if (gate === null) {
|
|
1061
|
+
throw daemonFailure("daemon:workspace-capability", "workspace-busy", 409, `Workspace ${checkedWorkspaceId} is running an operation or another capability change.`, {
|
|
1062
|
+
workspaceId: checkedWorkspaceId,
|
|
1063
|
+
namespaceOwner,
|
|
1064
|
+
recovery: "Settle the current operation and retry the capability change.",
|
|
1065
|
+
retryable: true,
|
|
1066
|
+
});
|
|
1067
|
+
}
|
|
1068
|
+
await gate.acquired;
|
|
1069
|
+
const prior = await this.#db.workspace_module_state_get.get({
|
|
1070
|
+
workspace_id: checkedWorkspaceId,
|
|
1071
|
+
namespace_owner: namespaceOwner,
|
|
1072
|
+
});
|
|
1073
|
+
let rollbackRuntimes;
|
|
1074
|
+
let stateChanged = false;
|
|
1075
|
+
try {
|
|
1076
|
+
const commitRuntimes = await this.#engine.prepareWorkspaceRuntimes(checkedWorkspaceId, namespaceOwner, normalized);
|
|
1077
|
+
if (encoded === null) {
|
|
1078
|
+
await this.#db.workspace_module_state_delete.run({
|
|
1079
|
+
workspace_id: checkedWorkspaceId,
|
|
1080
|
+
namespace_owner: namespaceOwner,
|
|
1081
|
+
});
|
|
1082
|
+
}
|
|
1083
|
+
else {
|
|
1084
|
+
await this.#db.workspace_module_state_put.run({
|
|
1085
|
+
workspace_id: checkedWorkspaceId,
|
|
1086
|
+
namespace_owner: namespaceOwner,
|
|
1087
|
+
state: encoded,
|
|
1088
|
+
});
|
|
1089
|
+
}
|
|
1090
|
+
stateChanged = true;
|
|
1091
|
+
rollbackRuntimes = commitRuntimes();
|
|
1092
|
+
if (this.#capabilitiesPublished) {
|
|
1093
|
+
await LoopDocs.materialize(this.#engine, this.#db, checkedWorkspaceId);
|
|
1094
|
+
}
|
|
1095
|
+
}
|
|
1096
|
+
catch (cause) {
|
|
1097
|
+
rollbackRuntimes?.();
|
|
1098
|
+
const rollbackErrors = [];
|
|
1099
|
+
if (stateChanged) {
|
|
1100
|
+
try {
|
|
1101
|
+
if (prior === undefined) {
|
|
1102
|
+
await this.#db.workspace_module_state_delete.run({
|
|
1103
|
+
workspace_id: checkedWorkspaceId,
|
|
1104
|
+
namespace_owner: namespaceOwner,
|
|
1105
|
+
});
|
|
1106
|
+
}
|
|
1107
|
+
else {
|
|
1108
|
+
await this.#db.workspace_module_state_put.run({
|
|
1109
|
+
workspace_id: checkedWorkspaceId,
|
|
1110
|
+
namespace_owner: namespaceOwner,
|
|
1111
|
+
state: prior.state,
|
|
1112
|
+
});
|
|
1113
|
+
}
|
|
1114
|
+
}
|
|
1115
|
+
catch (rollbackCause) {
|
|
1116
|
+
rollbackErrors.push(rollbackCause);
|
|
1117
|
+
}
|
|
1118
|
+
if (this.#capabilitiesPublished) {
|
|
1119
|
+
try {
|
|
1120
|
+
await LoopDocs.materialize(this.#engine, this.#db, checkedWorkspaceId);
|
|
1121
|
+
}
|
|
1122
|
+
catch (rollbackCause) {
|
|
1123
|
+
rollbackErrors.push(rollbackCause);
|
|
1124
|
+
}
|
|
1125
|
+
}
|
|
1126
|
+
}
|
|
1127
|
+
if (rollbackErrors.length > 0) {
|
|
1128
|
+
throw new AggregateError([cause, ...rollbackErrors], "Workspace capability replacement and rollback failed");
|
|
1129
|
+
}
|
|
1130
|
+
throw cause;
|
|
1131
|
+
}
|
|
1132
|
+
finally {
|
|
1133
|
+
gate.release();
|
|
1134
|
+
}
|
|
970
1135
|
}
|
|
971
1136
|
get engine() { return this.#engine; }
|
|
972
1137
|
get provider() { return this.#provider; }
|
|
@@ -981,6 +1146,7 @@ export default class Daemon {
|
|
|
981
1146
|
if (this.#started)
|
|
982
1147
|
throw new Error("daemon already started");
|
|
983
1148
|
this.#started = true;
|
|
1149
|
+
this.#drains.start();
|
|
984
1150
|
// Mimetypes owns its own discovery scan over @plurnk/plurnk-mimetypes-*
|
|
985
1151
|
// packages; pre-warm it so first index render doesn't pay the cost.
|
|
986
1152
|
await this.#mimetypes.ready();
|
|
@@ -1005,6 +1171,11 @@ export default class Daemon {
|
|
|
1005
1171
|
this.#moduleClosers.push(module);
|
|
1006
1172
|
await module.setup?.(setupSeam);
|
|
1007
1173
|
}
|
|
1174
|
+
for (const workspace of await Envelope.listWorkspaces(this.#db)) {
|
|
1175
|
+
for (const provider of this.#workspaceCapabilityProviders.values()) {
|
|
1176
|
+
await provider.hydrate(workspace.id);
|
|
1177
|
+
}
|
|
1178
|
+
}
|
|
1008
1179
|
await this.#schemes.ready();
|
|
1009
1180
|
// Reconcile the kernel-published documentation surface once per existing workspace.
|
|
1010
1181
|
// Installed capabilities and operator configuration are now fully known; model loops
|
|
@@ -1028,18 +1199,19 @@ export default class Daemon {
|
|
|
1028
1199
|
await this.#db.recovery_settle_open_provider_requests.run({});
|
|
1029
1200
|
await this.#db.recovery_fail_open_model_calls.run({});
|
|
1030
1201
|
await this.#db.recovery_fail_ownerless_proposals.run({});
|
|
1202
|
+
await this.#db.recovery_remove_ownerless_client_interactions.run({});
|
|
1031
1203
|
await this.#db.recovery_error_orphan_subscription_channels.run({});
|
|
1032
1204
|
await this.#db.recovery_fail_orphan_subscriptions.run({});
|
|
1033
1205
|
await this.#db.recovery_resume_unblocked_parks.run({});
|
|
1034
1206
|
await this.#branchBatches.recover();
|
|
1035
1207
|
const orphanSources = await this.#db.recovery_orphan_prompt_sources.all({});
|
|
1036
1208
|
for (const source of orphanSources) {
|
|
1037
|
-
await this.#reconcileOrphanedPrompts(source.worker_id, source.loop_id);
|
|
1209
|
+
await this.#drains.reconcileOrphanedPrompts(source.worker_id, source.loop_id);
|
|
1038
1210
|
}
|
|
1039
1211
|
const systemPrompt = await readFile(Paths.instructionsSystem, "utf8");
|
|
1040
1212
|
const queued = await this.#db.recovery_queued_workers.all({});
|
|
1041
1213
|
for (const row of queued) {
|
|
1042
|
-
const started = await this.#ensureDrain({
|
|
1214
|
+
const started = await this.#drains.ensureDrain({
|
|
1043
1215
|
workspaceId: row.workspace_id,
|
|
1044
1216
|
workerId: row.worker_id,
|
|
1045
1217
|
systemPrompt,
|
|
@@ -1050,7 +1222,7 @@ export default class Daemon {
|
|
|
1050
1222
|
}
|
|
1051
1223
|
const parked = await this.#db.recovery_parked_workers.all({});
|
|
1052
1224
|
for (const row of parked) {
|
|
1053
|
-
await this.#schedulePollWake(row.workspace_id, row.worker_id, systemPrompt);
|
|
1225
|
+
await this.#drains.schedulePollWake(row.workspace_id, row.worker_id, systemPrompt);
|
|
1054
1226
|
}
|
|
1055
1227
|
}
|
|
1056
1228
|
async stop() {
|
|
@@ -1064,8 +1236,8 @@ export default class Daemon {
|
|
|
1064
1236
|
.toReversed()
|
|
1065
1237
|
.map((module) => Promise.resolve().then(() => module.close())));
|
|
1066
1238
|
this.#moduleClosers = [];
|
|
1067
|
-
// Drain order: (1)
|
|
1068
|
-
// strike paths don't keep going, (2) await
|
|
1239
|
+
// Drain order: (1) tell the supervisor to abort worker scopes so
|
|
1240
|
+
// strike paths don't keep going, (2) await its active drains
|
|
1069
1241
|
// to completion, (3) drain streaming schemes' background work
|
|
1070
1242
|
// (exec spawn cleanup, channel writes). Only THEN close the DB
|
|
1071
1243
|
// upstream — drain queries hit the DB right up until they exit.
|
|
@@ -1080,24 +1252,9 @@ export default class Daemon {
|
|
|
1080
1252
|
this.#engine.cancelAllProposals("daemon_stopping");
|
|
1081
1253
|
this.#engine.cancelDerivations(derivationAbort);
|
|
1082
1254
|
this.#branchBatches.beginStop();
|
|
1083
|
-
|
|
1084
|
-
if (!scope.signal.aborted)
|
|
1085
|
-
scope.abort("daemon_stopping");
|
|
1086
|
-
}
|
|
1087
|
-
for (const t of this.#pollTimers.values())
|
|
1088
|
-
clearTimeout(t); // drop pending hibernation poll-wakes
|
|
1089
|
-
this.#pollBackoff.clear();
|
|
1090
|
-
this.#pollTimers.clear();
|
|
1091
|
-
// Cancel park-deadline timers before DB close; otherwise a late #wakeParkedWorker would run after
|
|
1092
|
-
// stop/db-close if left pending — an unhandled rejection (SqlRite closed) that abnormally
|
|
1093
|
-
// exits the worker under load. Symmetric with the poll-wakes above; both must be reaped.
|
|
1094
|
-
for (const t of this.#parkTimers.values())
|
|
1095
|
-
clearTimeout(t);
|
|
1096
|
-
this.#parkTimers.clear();
|
|
1255
|
+
this.#drains.beginStop("daemon_stopping");
|
|
1097
1256
|
await this.#branchBatches.idle();
|
|
1098
|
-
const
|
|
1099
|
-
await Promise.allSettled(drainPromises);
|
|
1100
|
-
await Promise.allSettled([...this.#drainExitTasks]);
|
|
1257
|
+
const [drainResult] = await Promise.allSettled([this.#drains.idle()]);
|
|
1101
1258
|
const closeResults = await moduleClose;
|
|
1102
1259
|
const [streamingResult] = await Promise.allSettled([this.#drainStreamingSchemes()]);
|
|
1103
1260
|
const [derivationResult] = await Promise.allSettled([
|
|
@@ -1107,7 +1264,19 @@ export default class Daemon {
|
|
|
1107
1264
|
? await Promise.allSettled([this.#mimetypes.dispose()])
|
|
1108
1265
|
: [];
|
|
1109
1266
|
const [schemeResult] = await Promise.allSettled([this.#schemes.close()]);
|
|
1110
|
-
|
|
1267
|
+
// Streaming and scheme closure are the last producers of synchronous
|
|
1268
|
+
// conclusion notifications. Join the supervisor-owned async tails only
|
|
1269
|
+
// after those producers settle, before the caller may close SQLite.
|
|
1270
|
+
const [wakeResult] = await Promise.allSettled([this.#drains.idle()]);
|
|
1271
|
+
const closeErrors = [
|
|
1272
|
+
...closeResults,
|
|
1273
|
+
drainResult,
|
|
1274
|
+
streamingResult,
|
|
1275
|
+
derivationResult,
|
|
1276
|
+
...mimetypeResults,
|
|
1277
|
+
schemeResult,
|
|
1278
|
+
wakeResult,
|
|
1279
|
+
]
|
|
1111
1280
|
.filter((result) => result.status === "rejected")
|
|
1112
1281
|
.flatMap((result) => result.reason instanceof AggregateError
|
|
1113
1282
|
? [...result.reason.errors]
|
|
@@ -1136,18 +1305,6 @@ export default class Daemon {
|
|
|
1136
1305
|
notifyNotice(workspaceId, payload) {
|
|
1137
1306
|
this.#broadcast({ workspaceId }, "notice/event", payload);
|
|
1138
1307
|
}
|
|
1139
|
-
/**
|
|
1140
|
-
* Inject a prompt into a worker. Two paths:
|
|
1141
|
-
* - Active drain: writes the next prompt:///<loop>/<N> entry via
|
|
1142
|
-
* Engine.inject. The current loop publishes it at its next
|
|
1143
|
-
* turn. Returns immediately with {action: "injected_next_turn"}.
|
|
1144
|
-
* - No active drain: enqueues a fresh loop with the prompt at
|
|
1145
|
-
* status=100, starts a drain. Returns the drain promise so the
|
|
1146
|
-
* caller can await full completion.
|
|
1147
|
-
*
|
|
1148
|
-
* Both `runLoop` and wake-on-completion go through this method
|
|
1149
|
-
* ({§actor-boundary-passive-wake}).
|
|
1150
|
-
*/
|
|
1151
1308
|
// {§methods-loop-run-fold-consistency} — a folded prompt cannot reconfigure its loop.
|
|
1152
1309
|
async #assertFoldPosture(workerId, flags, loopId) {
|
|
1153
1310
|
if (flags === undefined || Object.keys(flags).length === 0)
|
|
@@ -1168,810 +1325,56 @@ export default class Daemon {
|
|
|
1168
1325
|
});
|
|
1169
1326
|
}
|
|
1170
1327
|
}
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
// Active loop (status=102)? Fold the wake/prompt into its next turn.
|
|
1174
|
-
// engine.inject returns null when no loop is currently executing, so
|
|
1175
|
-
// we enqueue a fresh loop below and ensure a drain claims it.
|
|
1176
|
-
if (this.#activeDrains.has(workerId)) {
|
|
1177
|
-
const active = await this.#db.drain_current_loop_for_worker.get({ worker_id: workerId });
|
|
1178
|
-
if (active !== undefined) {
|
|
1179
|
-
await this.#assertFoldPosture(workerId, args.flags, active.id); // compare with the exact durable loop
|
|
1180
|
-
await this.#assertLoopProvider(active.id, args.providerSpec);
|
|
1181
|
-
if (args.childProviderSpec !== undefined)
|
|
1182
|
-
await this.#assertLoopChildProvider(active.id, args.childProviderSpec);
|
|
1183
|
-
await this.#assertLoopMaxTurns(active.id, args.turnCeiling?.source === "explicit" ? args.turnCeiling.effective : undefined);
|
|
1184
|
-
}
|
|
1185
|
-
const result = await this.#engine.inject(workerId, prompt, args.openPaths ?? []);
|
|
1186
|
-
if (result !== null) {
|
|
1187
|
-
return { action: "injected_next_turn", loopId: result.loopId, turnSeq: result.turnSeq };
|
|
1188
|
-
}
|
|
1189
|
-
}
|
|
1190
|
-
// {§worker-lifecycle-wake-requeue-not-terminal} — a worker parked at 202 resumes that loop in place:
|
|
1191
|
-
// is a wake edge like a stream/child conclusion, not a fresh loop that orphans the parked one
|
|
1192
|
-
// (which would leave the worker non-quiescent forever). engine.inject writes the message as the
|
|
1193
|
-
// slept loop's next-turn prompt (the directed message — distinct from the env door, which
|
|
1194
|
-
// resumes promptless); then re-queue + drain it. {§worker-lifecycle-wake-liveness}.
|
|
1195
|
-
if (!this.#activeDrains.has(workerId)) {
|
|
1196
|
-
const slept = await this.#db.drain_find_slept_loop.get({ worker_id: workerId });
|
|
1197
|
-
if (slept !== undefined) {
|
|
1198
|
-
await this.#assertFoldPosture(workerId, args.flags, slept.id); // resume drops nothing silently
|
|
1199
|
-
await this.#assertLoopProvider(slept.id, args.providerSpec);
|
|
1200
|
-
if (args.childProviderSpec !== undefined)
|
|
1201
|
-
await this.#assertLoopChildProvider(slept.id, args.childProviderSpec);
|
|
1202
|
-
await this.#assertLoopMaxTurns(slept.id, args.turnCeiling?.source === "explicit" ? args.turnCeiling.effective : undefined);
|
|
1203
|
-
const injected = await this.#engine.inject(workerId, prompt, args.openPaths ?? []);
|
|
1204
|
-
await this.#lifecycle.wake(slept.id);
|
|
1205
|
-
const started = await this.#ensureDrain({
|
|
1206
|
-
workspaceId, workerId, systemPrompt: args.systemPrompt,
|
|
1207
|
-
});
|
|
1208
|
-
return { action: "injected_next_turn", loopId: slept.id, ...(injected?.turnSeq !== undefined ? { turnSeq: injected.turnSeq } : {}), ...(started ?? {}) };
|
|
1209
|
-
}
|
|
1210
|
-
}
|
|
1211
|
-
const loopId = await this.#enqueueFreshLoop({
|
|
1212
|
-
workerId,
|
|
1213
|
-
prompt,
|
|
1214
|
-
providerSpec: args.providerSpec,
|
|
1215
|
-
childProviderSpec: args.childProviderSpec ?? null,
|
|
1216
|
-
maxTurns: args.turnCeiling?.effective,
|
|
1217
|
-
flags: args.flags,
|
|
1218
|
-
openPaths: args.openPaths,
|
|
1219
|
-
});
|
|
1220
|
-
// Guarantee a drain claims the loop we just enqueued. #ensureDrain runs its
|
|
1221
|
-
// check-and-start UNDER the per-worker drain lock ({§worker-lifecycle-single-drain}),
|
|
1222
|
-
// serialized against a draining sibling's teardown relinquish so the two can't
|
|
1223
|
-
// both register a drain (R4). A live drain re-claims the loop in its own
|
|
1224
|
-
// iteration or its lock-held exit re-claim, so it's never stranded.
|
|
1225
|
-
// firstLoopPromise is present only when THIS call started the drain — runLoop
|
|
1226
|
-
// keys its fast-path response on that.
|
|
1227
|
-
const started = await this.#ensureDrain({
|
|
1228
|
-
workspaceId, workerId, systemPrompt: args.systemPrompt,
|
|
1229
|
-
});
|
|
1230
|
-
return { action: "enqueued_new_loop", loopId, ...(started ?? {}) };
|
|
1231
|
-
}
|
|
1232
|
-
async #enqueueFreshLoop(args) {
|
|
1233
|
-
// {§worker-lifecycle-single-drain}: sequence allocation and insertion
|
|
1234
|
-
// are one queue mutation; another accepted prompt cannot claim the gap.
|
|
1235
|
-
return this.#withDrainLock(args.workerId, async () => {
|
|
1236
|
-
const seqRow = await this.#db.loop_run_next_sequence.get({
|
|
1237
|
-
worker_id: args.workerId,
|
|
1238
|
-
});
|
|
1239
|
-
if (seqRow === undefined)
|
|
1240
|
-
throw new Error("enqueueFreshLoop: next-sequence query returned no row");
|
|
1241
|
-
const loopRow = await this.#db.drain_enqueue_loop.get({
|
|
1242
|
-
worker_id: args.workerId,
|
|
1243
|
-
sequence: seqRow.next,
|
|
1244
|
-
prompt: args.prompt,
|
|
1245
|
-
provider_spec: JSON.stringify(args.providerSpec),
|
|
1246
|
-
child_provider_spec: JSON.stringify(args.childProviderSpec),
|
|
1247
|
-
max_turns: args.maxTurns ?? Number(process.env.PLURNK_SERVICE_MAX_TURNS ?? "50"),
|
|
1248
|
-
});
|
|
1249
|
-
if (loopRow === undefined)
|
|
1250
|
-
throw new Error("enqueueFreshLoop: loop enqueue returned no row");
|
|
1251
|
-
if (args.flags !== undefined) {
|
|
1252
|
-
await this.#db.engine_set_loop_flags.run({
|
|
1253
|
-
loop_id: loopRow.id,
|
|
1254
|
-
flags: JSON.stringify({ ...DEFAULT_LOOP_FLAGS, ...args.flags }),
|
|
1255
|
-
});
|
|
1256
|
-
}
|
|
1257
|
-
if (args.openPaths !== undefined && args.openPaths.length > 0) {
|
|
1258
|
-
await this.#db.engine_set_loop_open_paths.run({
|
|
1259
|
-
loop_id: loopRow.id,
|
|
1260
|
-
open_paths: JSON.stringify(args.openPaths),
|
|
1261
|
-
});
|
|
1262
|
-
}
|
|
1263
|
-
return loopRow.id;
|
|
1264
|
-
});
|
|
1328
|
+
inject(args) {
|
|
1329
|
+
return this.#drains.inject(args);
|
|
1265
1330
|
}
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
* drain_claim_next_loop (atomic 100→102 flip), executes each via
|
|
1269
|
-
* Engine.runLoop, and re-checks. Stream-aware: when the queue is empty
|
|
1270
|
-
* but the worker has active subscriptions, the drain parks on a
|
|
1271
|
-
* #drainPokes signal — wake-on-completion → inject() wakes it. Drain
|
|
1272
|
-
* exits when queue is empty AND no active subscriptions remain.
|
|
1273
|
-
*
|
|
1274
|
-
* Returns both `firstLoopPromise` (resolves once the first loop the
|
|
1275
|
-
* drain processes completes — used by runLoop to give the caller a
|
|
1276
|
-
* fast response containing their loop's result) and `drainPromise`
|
|
1277
|
-
* (resolves only when the whole drain finishes, queue+subs settled).
|
|
1278
|
-
*/
|
|
1279
|
-
#startDrain(opts) {
|
|
1280
|
-
const { workspaceId, workerId, systemPrompt } = opts;
|
|
1281
|
-
// The drain runs under the worker's cancellation scope (shared with the
|
|
1282
|
-
// execs its loops spawn), so loop.cancel/shutdown abort it as a unit.
|
|
1283
|
-
const controller = this.#workerSignal(workerId);
|
|
1284
|
-
const handle = {
|
|
1285
|
-
controller, promise: Promise.resolve(),
|
|
1286
|
-
};
|
|
1287
|
-
let resolveFirst = () => { };
|
|
1288
|
-
let rejectFirst = () => { };
|
|
1289
|
-
const firstLoopPromise = new Promise((res, rej) => {
|
|
1290
|
-
resolveFirst = res;
|
|
1291
|
-
rejectFirst = rej;
|
|
1292
|
-
});
|
|
1293
|
-
let firstSettled = false;
|
|
1294
|
-
const claim = () => this.#db.drain_claim_next_loop.get({ worker_id: workerId });
|
|
1295
|
-
const drainPromise = (async () => {
|
|
1296
|
-
let loopsDrained = 0;
|
|
1297
|
-
let lastResult = null;
|
|
1298
|
-
let currentLoopId = null; // the loop being drained — for abort→499 settlement
|
|
1299
|
-
try {
|
|
1300
|
-
while (true) {
|
|
1301
|
-
controller.signal.throwIfAborted();
|
|
1302
|
-
let loopRow = await claim();
|
|
1303
|
-
if (loopRow === undefined) {
|
|
1304
|
-
// Queue empty → teardown UNDER the per-worker drain lock (R4 / I1),
|
|
1305
|
-
// serialized against #ensureDrain so a concurrent inject can't
|
|
1306
|
-
// start a 2nd drain in the gap. Re-claim while holding the lock;
|
|
1307
|
-
// relinquish the registry slot only if it's empty too. A loop
|
|
1308
|
-
// that raced in is returned and run — we stay registered, so
|
|
1309
|
-
// there's no transient delete for #ensureDrain to catch.
|
|
1310
|
-
loopRow = await this.#withDrainLock(workerId, async () => {
|
|
1311
|
-
const claimed = await claim();
|
|
1312
|
-
if (claimed === undefined && this.#activeDrains.get(workerId) === handle) {
|
|
1313
|
-
this.#activeDrains.delete(workerId);
|
|
1314
|
-
}
|
|
1315
|
-
return claimed;
|
|
1316
|
-
});
|
|
1317
|
-
if (loopRow === undefined)
|
|
1318
|
-
break;
|
|
1319
|
-
}
|
|
1320
|
-
currentLoopId = loopRow.id;
|
|
1321
|
-
// {§methods-loop-run-model} — provider identity belongs to the claimed loop, not the
|
|
1322
|
-
// drain that happened to claim it. A drain can consume multiple
|
|
1323
|
-
// queued loops; resolve each durable selection at this boundary.
|
|
1324
|
-
const { provider, childProvider } = await this.#providersForLoop(loopRow.id);
|
|
1325
|
-
const onDispatch = (logEntryId) => {
|
|
1326
|
-
// {§methods-event-subscribe} — a log-broadcast failure must never crash the drain.
|
|
1327
|
-
void (async () => {
|
|
1328
|
-
const entry = await LogEntry.fetchLogEntry(this.#db, logEntryId);
|
|
1329
|
-
this.#broadcast({ workspaceId }, "log/entry", { entry });
|
|
1330
|
-
})().catch((e) => console.error("log/entry broadcast failed:", e instanceof Error ? e.message : String(e)));
|
|
1331
|
-
};
|
|
1332
|
-
const result = await observed(// {§observability-boundary}
|
|
1333
|
-
"loop.run", { workspaceId, workerId, "loop.id": loopRow.id }, async (span) => {
|
|
1334
|
-
const loopResult = await this.#engine.runLoop({
|
|
1335
|
-
provider, childProvider, workspaceId, workerId, loopId: loopRow.id, maxTurns: loopRow.max_turns,
|
|
1336
|
-
messages: [
|
|
1337
|
-
{ role: "system", content: systemPrompt },
|
|
1338
|
-
{ role: "user", content: loopRow.prompt },
|
|
1339
|
-
],
|
|
1340
|
-
origin: "model",
|
|
1341
|
-
onDispatch,
|
|
1342
|
-
signal: controller.signal,
|
|
1343
|
-
});
|
|
1344
|
-
span.setAttribute("status", loopResult.result.status);
|
|
1345
|
-
recordCounter(LOOP_TERMINALS, { status: loopResult.result.status });
|
|
1346
|
-
return loopResult;
|
|
1347
|
-
});
|
|
1348
|
-
if (result.result.status === 202) {
|
|
1349
|
-
// The loop slept via SEND signal 202 — suspended, not terminated. Leave it at 202
|
|
1350
|
-
// (resumable); no loop/terminated, no orphan-reconcile. A stream conclusion
|
|
1351
|
-
// (#handleWakeWorker) re-queues it; and if it holds a polled stream, a poll timer
|
|
1352
|
-
// wakes it every P to inspect ({§exec-poll}). {§worker-lifecycle-wake-liveness}.
|
|
1353
|
-
void this.#schedulePollWake(workspaceId, workerId, systemPrompt).catch((err) => console.error("poll-wake scheduling failed:", err instanceof Error ? err.message : String(err)));
|
|
1354
|
-
// {§send-premature-terminate}/scoped SEND signal 202 — the park deadline:
|
|
1355
|
-
// dispatcher recorded the marker's seconds; a bounded park is woken at T
|
|
1356
|
-
// regardless of arrivals, so a park always has a next turn. -1 (indefinite:
|
|
1357
|
-
// the butler, a [300] ask) schedules nothing — irc/inject/conclusions wake it.
|
|
1358
|
-
// In-memory: a daemon restart drops pending deadlines.
|
|
1359
|
-
if (currentLoopId !== null) {
|
|
1360
|
-
const deadline = this.#engine.parkDeadlines.get(currentLoopId);
|
|
1361
|
-
this.#engine.parkDeadlines.delete(currentLoopId);
|
|
1362
|
-
const prior = this.#parkTimers.get(workerId);
|
|
1363
|
-
if (prior !== undefined) {
|
|
1364
|
-
clearTimeout(prior);
|
|
1365
|
-
this.#parkTimers.delete(workerId);
|
|
1366
|
-
}
|
|
1367
|
-
if (deadline !== undefined && deadline > 0) {
|
|
1368
|
-
const t = setTimeout(() => {
|
|
1369
|
-
this.#parkTimers.delete(workerId);
|
|
1370
|
-
void this.#wakeParkedWorker(workspaceId, workerId, systemPrompt).catch((err) => console.error("park-deadline wake failed:", err instanceof Error ? err.message : String(err)));
|
|
1371
|
-
}, deadline * 1000);
|
|
1372
|
-
t.unref();
|
|
1373
|
-
this.#parkTimers.set(workerId, t);
|
|
1374
|
-
}
|
|
1375
|
-
}
|
|
1376
|
-
// Honor an OWED wake ({§worker-lifecycle-child-wake}): a child/stream concluded while
|
|
1377
|
-
// this worker was mid-turn, before it slept — resume in place rather than park blind,
|
|
1378
|
-
// so a worker hibernation always returns. The loop is 202 here; reset to
|
|
1379
|
-
// claimable and the drain re-runs it on the next claim below.
|
|
1380
|
-
if (this.#owedWakes.delete(workerId)) {
|
|
1381
|
-
await this.#lifecycle.wake(loopRow.id);
|
|
1382
|
-
currentLoopId = null;
|
|
1383
|
-
continue;
|
|
1384
|
-
}
|
|
1385
|
-
// The loop is blocked at 202 on a live obligation ({§wait-obligation-matrix});
|
|
1386
|
-
// that obligation's conclusion is its wake edge (the owed-wake above covers the
|
|
1387
|
-
// conclude-before-block race). An idle wait never reaches here — it concluded at dispatch.
|
|
1388
|
-
currentLoopId = null;
|
|
1389
|
-
continue;
|
|
1390
|
-
}
|
|
1391
|
-
this.#owedWakes.delete(workerId); // the loop concluded (non-202) — no park to honor a held wake at
|
|
1392
|
-
const [usage, attributions, turnIds] = await Promise.all([
|
|
1393
|
-
this.#engine.loopUsage(loopRow.id),
|
|
1394
|
-
this.#engine.loopAttributions(loopRow.id),
|
|
1395
|
-
this.#lifecycle.turnIds(loopRow.id),
|
|
1396
|
-
]);
|
|
1397
|
-
this.#broadcast({ workspaceId }, "loop/terminated", {
|
|
1398
|
-
workerId,
|
|
1399
|
-
loopId: loopRow.id,
|
|
1400
|
-
result: result.result,
|
|
1401
|
-
hitMaxTurns: result.hitMaxTurns,
|
|
1402
|
-
turnIds,
|
|
1403
|
-
usage,
|
|
1404
|
-
attributions,
|
|
1405
|
-
});
|
|
1406
|
-
loopsDrained++;
|
|
1407
|
-
const loopResult = {
|
|
1408
|
-
loopId: loopRow.id,
|
|
1409
|
-
turnIds,
|
|
1410
|
-
result: result.result,
|
|
1411
|
-
hitMaxTurns: result.hitMaxTurns,
|
|
1412
|
-
usage,
|
|
1413
|
-
attributions,
|
|
1414
|
-
};
|
|
1415
|
-
lastResult = loopResult;
|
|
1416
|
-
if (!firstSettled) {
|
|
1417
|
-
firstSettled = true;
|
|
1418
|
-
resolveFirst(loopResult);
|
|
1419
|
-
}
|
|
1420
|
-
// A next-turn prompt this loop ended before consuming (a
|
|
1421
|
-
// wake conclusion or a runLoop-while-active prompt) is promoted to
|
|
1422
|
-
// a fresh queued loop so it's never silently dropped.
|
|
1423
|
-
await this.#reconcileOrphanedPrompts(workerId, loopRow.id);
|
|
1424
|
-
currentLoopId = null;
|
|
1425
|
-
}
|
|
1426
|
-
}
|
|
1427
|
-
catch (err) {
|
|
1428
|
-
if (controller.signal.aborted) {
|
|
1429
|
-
// {§methods-loop-cancel} — loop.cancel / shutdown aborted the live drain. A cancellation
|
|
1430
|
-
// is the loop's TERMINAL state (499), delivered via loop/terminated (runLoop no
|
|
1431
|
-
// longer blocks to return it). A genuine error rejects firstLoopPromise.
|
|
1432
|
-
let usage = {
|
|
1433
|
-
accounting: aggregateProviderAccounting([]),
|
|
1434
|
-
contextTokens: null,
|
|
1435
|
-
promptBudget: null,
|
|
1436
|
-
meta: {},
|
|
1437
|
-
};
|
|
1438
|
-
let attributions = [];
|
|
1439
|
-
const message = ErrorDetail.preview(controller.signal.reason ?? "user_cancelled")
|
|
1440
|
-
|| "no reason was supplied";
|
|
1441
|
-
if (currentLoopId !== null) {
|
|
1442
|
-
// {§methods-loop-cancel}/{§worker-lifecycle-terminal-result} —
|
|
1443
|
-
// persist the exact 499 cancellation result before broadcasting it.
|
|
1444
|
-
const cancelled = await this.#lifecycle.finish(currentLoopId, Results.failure("lifecycle:cancel", "loop-cancelled", 499, `The loop was cancelled: ${message}.`, {}, {
|
|
1445
|
-
reason: message,
|
|
1446
|
-
stage: "loop",
|
|
1447
|
-
retryable: false,
|
|
1448
|
-
}), { terminatedBy: "cancel" });
|
|
1449
|
-
[usage, attributions] = await Promise.all([
|
|
1450
|
-
this.#engine.loopUsage(currentLoopId),
|
|
1451
|
-
this.#engine.loopAttributions(currentLoopId),
|
|
1452
|
-
]);
|
|
1453
|
-
if (cancelled !== null) {
|
|
1454
|
-
this.#broadcast({ workspaceId }, "loop/terminated", {
|
|
1455
|
-
workerId,
|
|
1456
|
-
loopId: currentLoopId,
|
|
1457
|
-
result: cancelled,
|
|
1458
|
-
hitMaxTurns: false,
|
|
1459
|
-
turnIds: await this.#lifecycle.turnIds(currentLoopId),
|
|
1460
|
-
usage,
|
|
1461
|
-
attributions,
|
|
1462
|
-
});
|
|
1463
|
-
}
|
|
1464
|
-
}
|
|
1465
|
-
if (!firstSettled) {
|
|
1466
|
-
firstSettled = true;
|
|
1467
|
-
resolveFirst({
|
|
1468
|
-
loopId: currentLoopId ?? 0,
|
|
1469
|
-
turnIds: [],
|
|
1470
|
-
result: currentLoopId === null
|
|
1471
|
-
? Results.failure("lifecycle:cancel", "loop-cancelled", 499, `The loop was cancelled: ${message}.`, {}, {
|
|
1472
|
-
reason: message,
|
|
1473
|
-
stage: "loop",
|
|
1474
|
-
retryable: false,
|
|
1475
|
-
})
|
|
1476
|
-
: await this.#lifecycle.result(currentLoopId)
|
|
1477
|
-
?? Results.failure("lifecycle:cancel", "loop-cancelled", 499, `The loop was cancelled: ${message}.`, {}, {
|
|
1478
|
-
reason: message,
|
|
1479
|
-
stage: "loop",
|
|
1480
|
-
retryable: false,
|
|
1481
|
-
}),
|
|
1482
|
-
hitMaxTurns: false,
|
|
1483
|
-
usage,
|
|
1484
|
-
});
|
|
1485
|
-
}
|
|
1486
|
-
}
|
|
1487
|
-
else {
|
|
1488
|
-
// {§worker-lifecycle-terminal-result} — a non-abort drain
|
|
1489
|
-
// failure becomes an exact durable 500 and terminal notification;
|
|
1490
|
-
// daemon diagnostics retain the complete caught error.
|
|
1491
|
-
console.error(`drain error (workspace ${workspaceId}, worker ${workerId}, loop ${currentLoopId ?? "?"}):`, err);
|
|
1492
|
-
if (currentLoopId !== null) {
|
|
1493
|
-
const failure = err instanceof OperationFailureError
|
|
1494
|
-
? err.result
|
|
1495
|
-
: Results.failure("daemon:drain", "loop-threw", 500, "The loop failed outside its operation result contract.", {}, {
|
|
1496
|
-
stage: "loop",
|
|
1497
|
-
retryable: false,
|
|
1498
|
-
});
|
|
1499
|
-
const settled = await this.#lifecycle.finish(currentLoopId, failure)
|
|
1500
|
-
?? await this.#lifecycle.result(currentLoopId);
|
|
1501
|
-
if (settled === null) {
|
|
1502
|
-
throw new Error(`drain could not settle loop ${currentLoopId}`, { cause: err });
|
|
1503
|
-
}
|
|
1504
|
-
const [usage, attributions] = await Promise.all([
|
|
1505
|
-
this.#engine.loopUsage(currentLoopId),
|
|
1506
|
-
this.#engine.loopAttributions(currentLoopId),
|
|
1507
|
-
]);
|
|
1508
|
-
this.#broadcast({ workspaceId }, "loop/terminated", {
|
|
1509
|
-
workerId,
|
|
1510
|
-
loopId: currentLoopId,
|
|
1511
|
-
result: settled,
|
|
1512
|
-
hitMaxTurns: false,
|
|
1513
|
-
turnIds: await this.#lifecycle.turnIds(currentLoopId),
|
|
1514
|
-
usage,
|
|
1515
|
-
attributions,
|
|
1516
|
-
});
|
|
1517
|
-
}
|
|
1518
|
-
if (!firstSettled) {
|
|
1519
|
-
firstSettled = true;
|
|
1520
|
-
rejectFirst(err);
|
|
1521
|
-
}
|
|
1522
|
-
}
|
|
1523
|
-
throw err;
|
|
1524
|
-
}
|
|
1525
|
-
finally {
|
|
1526
|
-
if (!firstSettled) {
|
|
1527
|
-
firstSettled = true;
|
|
1528
|
-
rejectFirst(new Error("drain exited without producing a result"));
|
|
1529
|
-
}
|
|
1530
|
-
if (this.#activeDrains.get(workerId) === handle)
|
|
1531
|
-
this.#activeDrains.delete(workerId);
|
|
1532
|
-
}
|
|
1533
|
-
return { loopsDrained, lastResult };
|
|
1534
|
-
})();
|
|
1535
|
-
handle.promise = drainPromise;
|
|
1536
|
-
this.#activeDrains.set(workerId, handle);
|
|
1537
|
-
// Topology join ({§worker-loop-lifecycle}): when this drain exits having CONCLUDED the worker, wake its parent
|
|
1538
|
-
// if parked. Runs after the drain fully tears down (settled promise) so the quiescence check sees
|
|
1539
|
-
// final state; speculative (#onDrainExit no-ops unless the worker concluded AND the parent is parked).
|
|
1540
|
-
const drainExitTask = drainPromise.then(() => this.#onDrainExit(workspaceId, workerId, systemPrompt), () => this.#onDrainExit(workspaceId, workerId, systemPrompt));
|
|
1541
|
-
this.#drainExitTasks.add(drainExitTask);
|
|
1542
|
-
void drainExitTask.catch((err) => {
|
|
1543
|
-
console.error(`parent wake after worker ${workerId} settlement failed:`, err);
|
|
1544
|
-
}).finally(() => {
|
|
1545
|
-
this.#drainExitTasks.delete(drainExitTask);
|
|
1546
|
-
});
|
|
1547
|
-
// Swallow unhandled rejections (drain aborts with no awaiter); the
|
|
1548
|
-
// error already surfaced via firstLoopPromise or was logged inside.
|
|
1549
|
-
drainPromise.catch(() => { });
|
|
1550
|
-
firstLoopPromise.catch(() => { });
|
|
1551
|
-
return { firstLoopPromise, drainPromise };
|
|
1552
|
-
}
|
|
1553
|
-
// Per-worker drain-transition lock (R4 / {§worker-lifecycle-single-drain}). #ensureDrain's
|
|
1554
|
-
// start and a drain's teardown relinquish both run under it, serialized, so the two
|
|
1555
|
-
// can't interleave and register two drains for one worker. The critical section is the
|
|
1556
|
-
// registry decision only (never a loop's work) — a sub-ms hop at drain boundaries.
|
|
1557
|
-
// A promise-chain mutex: each caller awaits the prior holder; the tail self-prunes
|
|
1558
|
-
// when idle so the Map stays bounded to workers mid-transition.
|
|
1559
|
-
#withDrainLock(workerId, fn) {
|
|
1560
|
-
const prev = this.#drainLocks.get(workerId) ?? Promise.resolve();
|
|
1561
|
-
const run = prev.then(fn, fn);
|
|
1562
|
-
const tail = run.catch(() => { });
|
|
1563
|
-
this.#drainLocks.set(workerId, tail);
|
|
1564
|
-
void tail.then(() => { if (this.#drainLocks.get(workerId) === tail)
|
|
1565
|
-
this.#drainLocks.delete(workerId); });
|
|
1566
|
-
return run;
|
|
1567
|
-
}
|
|
1568
|
-
// The drain guarantee, serialized per worker via #withDrainLock so it can't race a
|
|
1569
|
-
// sibling drain's teardown relinquish into a double-drain (R4). A live drain
|
|
1570
|
-
// (registered, NOT aborting) will claim the just-enqueued loop in its own iteration
|
|
1571
|
-
// or its lock-held exit re-claim → return null. A registered-but-ABORTING drain is
|
|
1572
|
-
// in teardown and won't claim, so we don't defer to it — start fresh, or the loop
|
|
1573
|
-
// strands on a cancel/resume race (I6 no-lost-loop). Otherwise start one.
|
|
1574
|
-
#ensureDrain(opts) {
|
|
1575
|
-
return this.#withDrainLock(opts.workerId, async () => {
|
|
1576
|
-
const existing = this.#activeDrains.get(opts.workerId);
|
|
1577
|
-
if (existing !== undefined && !existing.controller.signal.aborted)
|
|
1578
|
-
return null;
|
|
1579
|
-
return this.#startDrain(opts);
|
|
1580
|
-
});
|
|
1581
|
-
}
|
|
1582
|
-
// After a loop terminates, promote every next-turn frame it never consumed
|
|
1583
|
-
// into one source-keyed queued loop. The first frame occupies the loop seed;
|
|
1584
|
-
// later frames retain separate prompt entries and publish in the same turn.
|
|
1585
|
-
// Re-entry and boot recovery complete that same queued identity.
|
|
1331
|
+
// Durable prompt promotion remains daemon policy; DrainSupervisor invokes
|
|
1332
|
+
// it under the same worker lock as enqueue and drain teardown.
|
|
1586
1333
|
async #reconcileOrphanedPrompts(workerId, endedLoopId) {
|
|
1587
|
-
await this.#
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
orphan_source_loop_id: endedLoopId,
|
|
1607
|
-
});
|
|
1608
|
-
if (recovery === undefined)
|
|
1609
|
-
throw new Error("reconcileOrphanedPrompts: enqueue returned no row");
|
|
1610
|
-
if (recovery.status !== 100)
|
|
1611
|
-
return;
|
|
1612
|
-
const moved = await this.#db.drain_rehome_orphaned_prompt_frames.all({
|
|
1613
|
-
owner_id: workerId,
|
|
1614
|
-
source_loop_id: endedLoopId,
|
|
1615
|
-
source_pattern: `${prefix}%`,
|
|
1616
|
-
source_prefix_len: prefix.length,
|
|
1617
|
-
target_prefix: promptLoopPrefix(recovery.sequence),
|
|
1618
|
-
});
|
|
1619
|
-
if (moved.length !== frames.length) {
|
|
1620
|
-
throw new Error(`reconcileOrphanedPrompts: expected to re-home ${frames.length} frames, moved ${moved.length}`);
|
|
1621
|
-
}
|
|
1334
|
+
const endedSeq = (await this.#db.engine_loop_sequence.get({ loop_id: endedLoopId }))?.sequence ?? endedLoopId;
|
|
1335
|
+
const prefix = promptLoopPrefix(endedSeq);
|
|
1336
|
+
const frames = await this.#db.drain_orphaned_prompts_for_loop.all({ loop_id: endedLoopId, owner_id: workerId, pattern: `${prefix}%`, prefix_len: prefix.length });
|
|
1337
|
+
const first = frames[0];
|
|
1338
|
+
if (first === undefined)
|
|
1339
|
+
return;
|
|
1340
|
+
const seqRow = await this.#db.loop_run_next_sequence.get({ worker_id: workerId });
|
|
1341
|
+
if (seqRow === undefined)
|
|
1342
|
+
throw new Error("reconcileOrphanedPrompts: next-sequence query returned no row");
|
|
1343
|
+
const recovery = await this.#db.drain_enqueue_orphan_recovery_loop.get({
|
|
1344
|
+
worker_id: workerId,
|
|
1345
|
+
sequence: seqRow.next,
|
|
1346
|
+
prompt: first.body,
|
|
1347
|
+
flags: first.flags,
|
|
1348
|
+
provider_spec: first.provider_spec,
|
|
1349
|
+
child_provider_spec: first.child_provider_spec,
|
|
1350
|
+
max_turns: first.max_turns,
|
|
1351
|
+
open_paths: first.open_paths ?? "[]",
|
|
1352
|
+
orphan_source_loop_id: endedLoopId,
|
|
1622
1353
|
});
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
const cancelled = await this.#lifecycle.cancelTree(workerId, reason, includeRoot);
|
|
1637
|
-
for (const targetWorkerId of cancelled.workerIds) {
|
|
1638
|
-
const pollTimer = this.#pollTimers.get(targetWorkerId);
|
|
1639
|
-
if (pollTimer !== undefined) {
|
|
1640
|
-
clearTimeout(pollTimer);
|
|
1641
|
-
this.#pollTimers.delete(targetWorkerId);
|
|
1642
|
-
}
|
|
1643
|
-
const parkTimer = this.#parkTimers.get(targetWorkerId);
|
|
1644
|
-
if (parkTimer !== undefined) {
|
|
1645
|
-
clearTimeout(parkTimer);
|
|
1646
|
-
this.#parkTimers.delete(targetWorkerId);
|
|
1647
|
-
}
|
|
1648
|
-
this.#pollBackoff.delete(targetWorkerId);
|
|
1649
|
-
this.#owedWakes.delete(targetWorkerId);
|
|
1650
|
-
const scope = this.#workerAborts.get(targetWorkerId);
|
|
1651
|
-
if (scope !== undefined && !scope.signal.aborted)
|
|
1652
|
-
scope.abort(reason);
|
|
1653
|
-
}
|
|
1654
|
-
await Promise.all(cancelled.workerIds.map(async (targetWorkerId) => this.#reapWorkerStreams(targetWorkerId)));
|
|
1655
|
-
for (const { loopId, workerId: targetWorkerId, result } of cancelled.loops) {
|
|
1656
|
-
const row = await this.#db.drain_get_worker_workspace.get({ worker_id: targetWorkerId });
|
|
1657
|
-
if (row === undefined)
|
|
1658
|
-
continue;
|
|
1659
|
-
const [usage, attributions] = await Promise.all([
|
|
1660
|
-
this.#engine.loopUsage(loopId),
|
|
1661
|
-
this.#engine.loopAttributions(loopId),
|
|
1662
|
-
]);
|
|
1663
|
-
this.#broadcast({ workspaceId: row.workspace_id }, "loop/terminated", {
|
|
1664
|
-
workerId: targetWorkerId,
|
|
1665
|
-
loopId,
|
|
1666
|
-
result,
|
|
1667
|
-
hitMaxTurns: false,
|
|
1668
|
-
turnIds: await this.#lifecycle.turnIds(loopId),
|
|
1669
|
-
usage,
|
|
1670
|
-
attributions,
|
|
1671
|
-
});
|
|
1354
|
+
if (recovery === undefined)
|
|
1355
|
+
throw new Error("reconcileOrphanedPrompts: enqueue returned no row");
|
|
1356
|
+
if (recovery.status !== 100)
|
|
1357
|
+
return;
|
|
1358
|
+
const moved = await this.#db.drain_rehome_orphaned_prompt_frames.all({
|
|
1359
|
+
owner_id: workerId,
|
|
1360
|
+
source_loop_id: endedLoopId,
|
|
1361
|
+
source_pattern: `${prefix}%`,
|
|
1362
|
+
source_prefix_len: prefix.length,
|
|
1363
|
+
target_prefix: promptLoopPrefix(recovery.sequence),
|
|
1364
|
+
});
|
|
1365
|
+
if (moved.length !== frames.length) {
|
|
1366
|
+
throw new Error(`reconcileOrphanedPrompts: expected to re-home ${frames.length} frames, moved ${moved.length}`);
|
|
1672
1367
|
}
|
|
1673
1368
|
}
|
|
1674
|
-
async #cancelWorkerTree(workerId, reason) {
|
|
1675
|
-
await this.#cancelTree(workerId, reason, true);
|
|
1676
|
-
}
|
|
1677
|
-
/**
|
|
1678
|
-
* Cancel the worker's in-flight work (loop.cancel). One abort, one scope: the
|
|
1679
|
-
* worker signal stops the running loop's turn generation AND tears down every
|
|
1680
|
-
* stream linked to it — a background exec that outlived its loop, or even a
|
|
1681
|
-
* spawn that registers after this abort (it self-aborts against the aborted
|
|
1682
|
-
* signal). Returns cancelled iff there was process-local work; durable
|
|
1683
|
-
* unresolved loops in the worker tree are terminalized independently.
|
|
1684
|
-
*/
|
|
1685
1369
|
cancelDrain(workerId, reason = "user_cancelled") {
|
|
1686
|
-
|
|
1687
|
-
const hadWork = hadDrain || this.#workerHasActiveStreams(workerId);
|
|
1688
|
-
// A cancel is deliberate — kill any pending hibernation poll-wake so it can't resurrect the worker.
|
|
1689
|
-
const pollTimer = this.#pollTimers.get(workerId);
|
|
1690
|
-
if (pollTimer !== undefined) {
|
|
1691
|
-
clearTimeout(pollTimer);
|
|
1692
|
-
this.#pollTimers.delete(workerId);
|
|
1693
|
-
}
|
|
1694
|
-
// Stop the active drain's turn-generation (its loop closes 499). The worker
|
|
1695
|
-
// signal is the optimization path — the fast, listener-driven reap.
|
|
1696
|
-
// Durable structured cancellation: one recursive transition claims the
|
|
1697
|
-
// worker and every unresolved descendant, then reaps each process-local scope.
|
|
1698
|
-
void this.#cancelWorkerTree(workerId, reason).catch((err) => {
|
|
1699
|
-
console.error(`cancelTree(${workerId}) failed:`, err);
|
|
1700
|
-
});
|
|
1701
|
-
return hadWork;
|
|
1370
|
+
return this.#drains.cancel(workerId, reason);
|
|
1702
1371
|
}
|
|
1703
|
-
//
|
|
1704
|
-
//
|
|
1705
|
-
// worker signal. Duck-typed like #drainStreamingSchemes.
|
|
1372
|
+
// Process-local stream activity is supplied to the drain owner through the
|
|
1373
|
+
// scheme registry rather than exposing that registry across the boundary.
|
|
1706
1374
|
#workerHasActiveStreams(workerId) {
|
|
1707
1375
|
const exec = this.#schemes.get("exec");
|
|
1708
1376
|
return exec?.hasActiveSpawns?.(workerId) ?? false;
|
|
1709
1377
|
}
|
|
1710
|
-
// The contract-routed reap ({§worker-lifecycle-total-reap}): durable rows enumerate
|
|
1711
|
-
// every open subscription; the live registry invokes its exact callable owner.
|
|
1712
|
-
// The worker signal is only the fast path. An exec mid-spawn or a background exec
|
|
1713
|
-
// from a past loop is caught regardless of listener timing. Idempotent — a stream
|
|
1714
|
-
// the signal already reaped shares the same registry cancellation.
|
|
1715
|
-
async #reapWorkerStreams(workerId) {
|
|
1716
|
-
const open = await ChannelWrite.findOpenSubscriptionsForWorker(this.#db, workerId);
|
|
1717
|
-
await Promise.all(open.map(({ id }) => this.#engine.cancelSubscription(id)));
|
|
1718
|
-
}
|
|
1719
|
-
/**
|
|
1720
|
-
* Wake-on-completion handler. Streaming schemes call this when a
|
|
1721
|
-
* subscription closes. A parked loop resumes in place; an active loop
|
|
1722
|
-
* observes the channel transition at its next turn boundary. No synthetic
|
|
1723
|
-
* prompt or replacement loop is created.
|
|
1724
|
-
*
|
|
1725
|
-
* Skipped on result.status=499 (aborted): the model already knows about
|
|
1726
|
-
* its own SEND signal 499, and a forcefully-cancelled loop's spawn-abort must
|
|
1727
|
-
* not resurrect the worker.
|
|
1728
|
-
*/
|
|
1729
|
-
async #handleWakeWorker(payload) {
|
|
1730
|
-
const { entryOwnerId, ...wake } = payload;
|
|
1731
|
-
const conclusion = { ...wake, workerId: entryOwnerId };
|
|
1732
|
-
// {§search-gate} — settle the dedup registration: promote on a 200 conclusion, drop on
|
|
1733
|
-
// failure (a dead search must never serve as a duplicate). No-op for non-search streams.
|
|
1734
|
-
this.#engine.searchGate.settle(payload.target.replace(/^[a-z+.-]+:\/\//, "/").replace(/^\/+/, "/"), payload.result.status);
|
|
1735
|
-
// Aborted streams don't wake — the abort was deliberate.
|
|
1736
|
-
if (payload.result.status === 499) {
|
|
1737
|
-
this.#broadcast({ workspaceId: payload.workspaceId }, "stream/concluded", {
|
|
1738
|
-
...conclusion, wakeAction: "skipped-aborted",
|
|
1739
|
-
});
|
|
1740
|
-
return;
|
|
1741
|
-
}
|
|
1742
|
-
// No resurrection ({§worker-lifecycle-no-resurrection}): a non-499 completion whose
|
|
1743
|
-
// worker was cancelled (idle + its scope aborted) must not start a fresh drain —
|
|
1744
|
-
// the cancel was deliberate. The deliverable is already in the channel/log and
|
|
1745
|
-
// surfaces as a `collect` environment delta ({§env-delta}) if the worker is read or
|
|
1746
|
-
// resumed; we just don't inject a turn. (An active worker folds the wake into its
|
|
1747
|
-
// next turn via inject below; a resumed worker is active, never aborted, so it is
|
|
1748
|
-
// unaffected.)
|
|
1749
|
-
const scope = this.#workerAborts.get(payload.workerId);
|
|
1750
|
-
if (scope?.signal.aborted === true && !this.#activeDrains.has(payload.workerId)) {
|
|
1751
|
-
this.#broadcast({ workspaceId: payload.workspaceId }, "stream/concluded", {
|
|
1752
|
-
...conclusion, wakeAction: "skipped-cancelled",
|
|
1753
|
-
});
|
|
1754
|
-
return;
|
|
1755
|
-
}
|
|
1756
|
-
try {
|
|
1757
|
-
const systemPrompt = await readFile(Paths.instructionsSystem, "utf8");
|
|
1758
|
-
// A slept (202) loop means the worker parked via SEND signal 202 → resume it in place: re-queue
|
|
1759
|
-
// it (202→100) so the drain re-claims and CONTINUES it (seq>1 → no re-foist). Checked
|
|
1760
|
-
// FIRST: the slept status is the worker's true disposition regardless of a draining
|
|
1761
|
-
// sibling mid-teardown (the #ensureDrain lock serializes the re-claim). No fresh loop,
|
|
1762
|
-
// no summary-as-prompt — the resumed loop reads the concluded stream's own state from
|
|
1763
|
-
// the manifest. {§worker-lifecycle-wake-liveness}.
|
|
1764
|
-
const slept = await this.#db.drain_find_slept_loop.get({ worker_id: payload.workerId });
|
|
1765
|
-
if (slept !== undefined) {
|
|
1766
|
-
// {§worker-optimistic-settlement} — publish this conclusion now,
|
|
1767
|
-
// then let the worker-local gate coalesce only the provider
|
|
1768
|
-
// dispatch. Concurrent stream/child callbacks join that one gate.
|
|
1769
|
-
void this.#settleCompletionWake(payload.workspaceId, payload.workerId, systemPrompt, false).catch((err) => {
|
|
1770
|
-
console.error("completion wake settlement failed:", err instanceof Error ? err.message : String(err));
|
|
1771
|
-
});
|
|
1772
|
-
this.#broadcast({ workspaceId: payload.workspaceId }, "stream/concluded", {
|
|
1773
|
-
...conclusion, wakeAction: "resumed-loop", wakeLoopId: slept.id,
|
|
1774
|
-
});
|
|
1775
|
-
return;
|
|
1776
|
-
}
|
|
1777
|
-
// No slept loop. A live loop surfaces the concluded stream ambiently via the
|
|
1778
|
-
// environment-observation injector ({§exec-stream}) on its next turn — there is no prompt
|
|
1779
|
-
// to inject and NO task to overwrite. The obsolete "automated environment update"
|
|
1780
|
-
// synthesis (which clobbered the model's actual goal) is retired; just tell the client.
|
|
1781
|
-
if (this.#activeDrains.has(payload.workerId)) {
|
|
1782
|
-
this.#broadcast({ workspaceId: payload.workspaceId }, "stream/concluded", {
|
|
1783
|
-
...conclusion, wakeAction: "no-op-active-loop",
|
|
1784
|
-
});
|
|
1785
|
-
return;
|
|
1786
|
-
}
|
|
1787
|
-
// No slept loop, no active drain — nothing to resume (e.g. a SEND-200-done worker whose
|
|
1788
|
-
// streams were swept). Surface the conclusion without opening a loop.
|
|
1789
|
-
this.#broadcast({ workspaceId: payload.workspaceId }, "stream/concluded", {
|
|
1790
|
-
...conclusion, wakeAction: "no-loop",
|
|
1791
|
-
});
|
|
1792
|
-
}
|
|
1793
|
-
catch (err) {
|
|
1794
|
-
console.error("wake-on-completion setup failed:", err instanceof Error ? err.message : String(err));
|
|
1795
|
-
}
|
|
1796
|
-
}
|
|
1797
|
-
/**
|
|
1798
|
-
* grammar 0.74.20 EXEC `<T,P>` — schedule a hibernation poll-wake. Called when a loop parks at
|
|
1799
|
-
* a park; if the worker holds an open polled stream, arm a timer for its tightest cadence P that
|
|
1800
|
-
* resumes the slept loop so the model inspects progress. While the loop is ACTIVE there is no
|
|
1801
|
-
* poll work — ambient folded stream deltas already surface progress ({§exec-stream}); the wake
|
|
1802
|
-
* matters only across hibernation. A wake-edge-less 202 (no polled stream) gets no timer. {§exec-poll}
|
|
1803
|
-
*/
|
|
1804
|
-
async #schedulePollWake(workspaceId, workerId, systemPrompt) {
|
|
1805
|
-
const existing = this.#pollTimers.get(workerId);
|
|
1806
|
-
if (existing !== undefined) {
|
|
1807
|
-
clearTimeout(existing);
|
|
1808
|
-
this.#pollTimers.delete(workerId);
|
|
1809
|
-
}
|
|
1810
|
-
const row = await this.#db.drain_worker_min_poll.get({ worker_id: workerId });
|
|
1811
|
-
if ((row?.open_count ?? 0) === 0) {
|
|
1812
|
-
this.#pollBackoff.delete(workerId);
|
|
1813
|
-
return;
|
|
1814
|
-
}
|
|
1815
|
-
const pollSec = row?.poll_seconds ?? null;
|
|
1816
|
-
// {§exec-poll} — a positive explicit cadence wins, zero opts out,
|
|
1817
|
-
// and an absent cadence uses the worker's exponential-backoff step.
|
|
1818
|
-
let delayMs;
|
|
1819
|
-
if (pollSec !== null && pollSec > 0) {
|
|
1820
|
-
this.#pollBackoff.delete(workerId);
|
|
1821
|
-
delayMs = pollSec * 1000;
|
|
1822
|
-
}
|
|
1823
|
-
else if (pollSec === 0) {
|
|
1824
|
-
this.#pollBackoff.delete(workerId);
|
|
1825
|
-
return; // explicit opt-out
|
|
1826
|
-
}
|
|
1827
|
-
else {
|
|
1828
|
-
// An open stream without an explicit cadence uses the stream polling floor.
|
|
1829
|
-
// Child joins never enter this branch: durable child settlement is their only wake edge.
|
|
1830
|
-
const base = Number(process.env.PLURNK_SERVICE_EXEC_POLL_SEC ?? "60");
|
|
1831
|
-
const turns = Number(process.env.PLURNK_SERVICE_EXEC_POLL_TURNS ?? "8");
|
|
1832
|
-
const step = this.#pollBackoff.get(workerId) ?? 0;
|
|
1833
|
-
delayMs = execPollBackoffMs(step, base, turns);
|
|
1834
|
-
this.#pollBackoff.set(workerId, step + 1);
|
|
1835
|
-
}
|
|
1836
|
-
// Floored by the optimistic settlement cap so a `<…,1>` cannot wake a
|
|
1837
|
-
// parked loop faster than the preceding turn's settlement scale.
|
|
1838
|
-
const optimisticWaitMs = readOptimisticSettlementMs();
|
|
1839
|
-
const timer = setTimeout(() => {
|
|
1840
|
-
this.#pollTimers.delete(workerId);
|
|
1841
|
-
void this.#wakeParkedWorker(workspaceId, workerId, systemPrompt);
|
|
1842
|
-
}, Math.max(delayMs, optimisticWaitMs));
|
|
1843
|
-
timer.unref();
|
|
1844
|
-
this.#pollTimers.set(workerId, timer);
|
|
1845
|
-
}
|
|
1846
|
-
/** Resume `workerId`'s slept (202) loop in place — the same 202→100 resume #handleWakeWorker uses, minus a
|
|
1847
|
-
* wake payload. The shared wake primitive: a poll cadence ({§exec-poll}), a watched stream concluding,
|
|
1848
|
-
* or a child worker finishing ({§worker-loop-lifecycle} topology join) all call this. A no-op if the worker was
|
|
1849
|
-
* cancelled or isn't actually parked (no slept loop) — so calling it speculatively is safe. */
|
|
1850
|
-
async #wakeParkedWorker(workspaceId, workerId, systemPrompt, oweIfActive = true) {
|
|
1851
|
-
if (!this.#started)
|
|
1852
|
-
return;
|
|
1853
|
-
const scope = this.#workerAborts.get(workerId);
|
|
1854
|
-
if (scope?.signal.aborted === true && !this.#activeDrains.has(workerId))
|
|
1855
|
-
return; // cancelled — no resurrection
|
|
1856
|
-
const slept = await this.#db.drain_find_slept_loop.get({ worker_id: workerId });
|
|
1857
|
-
if (slept === undefined) {
|
|
1858
|
-
// Not parked. If a drain is still ACTIVE, the worker is mid-turn and about to park — the
|
|
1859
|
-
// conclusion that fired this wake arrived before the 202 committed (the conclude-before-park
|
|
1860
|
-
// race). Owe the wake: the drain honors it at park so a worker hibernation never deadlocks.
|
|
1861
|
-
// (No active drain → already concluded/running; nothing to wake.)
|
|
1862
|
-
if (this.#activeDrains.has(workerId)) {
|
|
1863
|
-
if (oweIfActive)
|
|
1864
|
-
this.#owedWakes.add(workerId);
|
|
1865
|
-
}
|
|
1866
|
-
return;
|
|
1867
|
-
}
|
|
1868
|
-
const woke = await this.#lifecycle.wake(slept.id);
|
|
1869
|
-
if (!woke)
|
|
1870
|
-
return;
|
|
1871
|
-
const started = await this.#ensureDrain({
|
|
1872
|
-
workspaceId, workerId, systemPrompt,
|
|
1873
|
-
});
|
|
1874
|
-
started?.drainPromise?.catch((err) => {
|
|
1875
|
-
if (this.#started) {
|
|
1876
|
-
console.error("wake-parked resume drain failed:", err instanceof Error ? err.message : String(err));
|
|
1877
|
-
}
|
|
1878
|
-
});
|
|
1879
|
-
}
|
|
1880
|
-
async #workerHasLiveObligation(workerId) {
|
|
1881
|
-
const [openSubscriptions, liveChild] = await Promise.all([
|
|
1882
|
-
this.#db.find_open_subscriptions_for_worker.all({ worker_id: workerId }),
|
|
1883
|
-
this.#db.engine_worker_has_live_child.get({ worker_id: workerId }),
|
|
1884
|
-
]);
|
|
1885
|
-
return openSubscriptions.length > 0 || liveChild !== undefined;
|
|
1886
|
-
}
|
|
1887
|
-
#settleCompletionWake(workspaceId, workerId, systemPrompt, oweIfActive = true) {
|
|
1888
|
-
const existing = this.#completionWakeGates.get(workerId);
|
|
1889
|
-
if (existing !== undefined) {
|
|
1890
|
-
existing.conclusions++;
|
|
1891
|
-
existing.poke.resolve();
|
|
1892
|
-
return existing.promise;
|
|
1893
|
-
}
|
|
1894
|
-
const completed = Promise.withResolvers();
|
|
1895
|
-
const gate = {
|
|
1896
|
-
conclusions: 1,
|
|
1897
|
-
poke: Promise.withResolvers(),
|
|
1898
|
-
promise: completed.promise,
|
|
1899
|
-
};
|
|
1900
|
-
this.#completionWakeGates.set(workerId, gate);
|
|
1901
|
-
void this.#runCompletionWake(workspaceId, workerId, systemPrompt, oweIfActive, gate).then(completed.resolve, completed.reject).finally(() => {
|
|
1902
|
-
if (this.#completionWakeGates.get(workerId) === gate) {
|
|
1903
|
-
this.#completionWakeGates.delete(workerId);
|
|
1904
|
-
}
|
|
1905
|
-
});
|
|
1906
|
-
return gate.promise;
|
|
1907
|
-
}
|
|
1908
|
-
async #runCompletionWake(workspaceId, workerId, systemPrompt, oweIfActive, gate) {
|
|
1909
|
-
const slept = await this.#db.drain_find_slept_loop.get({ worker_id: workerId });
|
|
1910
|
-
if (slept === undefined) {
|
|
1911
|
-
this.#releaseCompletionWake(workerId, gate);
|
|
1912
|
-
return this.#wakeParkedWorker(workspaceId, workerId, systemPrompt, oweIfActive);
|
|
1913
|
-
}
|
|
1914
|
-
const timeoutMs = readOptimisticSettlementMs();
|
|
1915
|
-
if (timeoutMs === 0 || !(await this.#workerHasLiveObligation(workerId))) {
|
|
1916
|
-
this.#releaseCompletionWake(workerId, gate);
|
|
1917
|
-
return this.#wakeParkedWorker(workspaceId, workerId, systemPrompt, false);
|
|
1918
|
-
}
|
|
1919
|
-
return observed("worker.wake.settlement", { "window.ms": timeoutMs }, async (span) => {
|
|
1920
|
-
const startedAt = performance.now();
|
|
1921
|
-
const signal = this.#workerAborts.get(workerId)?.signal;
|
|
1922
|
-
const deadline = delay(timeoutMs, "deadline", { signal, ref: false })
|
|
1923
|
-
.catch((cause) => {
|
|
1924
|
-
if (signal?.aborted === true)
|
|
1925
|
-
return "cancelled";
|
|
1926
|
-
throw cause;
|
|
1927
|
-
});
|
|
1928
|
-
let release = "quiescent";
|
|
1929
|
-
while (await this.#workerHasLiveObligation(workerId)) {
|
|
1930
|
-
const poke = gate.poke;
|
|
1931
|
-
const outcome = await Promise.race([
|
|
1932
|
-
poke.promise.then(() => "arrival"),
|
|
1933
|
-
deadline,
|
|
1934
|
-
]);
|
|
1935
|
-
if (outcome === "arrival") {
|
|
1936
|
-
if (gate.poke === poke)
|
|
1937
|
-
gate.poke = Promise.withResolvers();
|
|
1938
|
-
continue;
|
|
1939
|
-
}
|
|
1940
|
-
release = outcome;
|
|
1941
|
-
break;
|
|
1942
|
-
}
|
|
1943
|
-
span.setAttribute("release", release);
|
|
1944
|
-
span.setAttribute("conclusions", gate.conclusions);
|
|
1945
|
-
span.setAttribute("elapsed.ms", Math.round(performance.now() - startedAt));
|
|
1946
|
-
this.#releaseCompletionWake(workerId, gate);
|
|
1947
|
-
if (release === "cancelled")
|
|
1948
|
-
return;
|
|
1949
|
-
return this.#wakeParkedWorker(workspaceId, workerId, systemPrompt, false);
|
|
1950
|
-
});
|
|
1951
|
-
}
|
|
1952
|
-
#releaseCompletionWake(workerId, gate) {
|
|
1953
|
-
if (this.#completionWakeGates.get(workerId) === gate) {
|
|
1954
|
-
this.#completionWakeGates.delete(workerId);
|
|
1955
|
-
}
|
|
1956
|
-
}
|
|
1957
|
-
/** A worker's drain exited. If the worker truly CONCLUDED — no 202-blocked loop, no open stream — then
|
|
1958
|
-
* wake its PARENT in place if the parent is blocked on the join (the structured-concurrency join — a
|
|
1959
|
-
* child finishing is the wake edge for a parent that waited on it, {§worker-lifecycle-child-wake}). A worker
|
|
1960
|
-
* blocked at 202, or still holding a stream, is NOT concluded — its own wake edges drive it, not this.
|
|
1961
|
-
* The parent reads the child's deliverable from its own log (the {§worker-scheme-collect} delta) on
|
|
1962
|
-
* resume — control edge here, never an injected prompt. Recurses up via the parent's own drain-exit. */
|
|
1963
|
-
async #onDrainExit(workspaceId, workerId, systemPrompt) {
|
|
1964
|
-
const slept = await this.#db.drain_find_slept_loop.get({ worker_id: workerId });
|
|
1965
|
-
if (slept !== undefined)
|
|
1966
|
-
return; // parked at 202 — not concluded, the worker is still alive
|
|
1967
|
-
const openSubs = await this.#db.find_open_subscriptions_for_worker.all({ worker_id: workerId });
|
|
1968
|
-
if (openSubs.length > 0)
|
|
1969
|
-
return; // a stream still runs — its conclusion re-evaluates, not this exit
|
|
1970
|
-
const parent = await this.#db.worker_parent_id.get({ worker_id: workerId });
|
|
1971
|
-
if (parent?.parent_worker_id == null)
|
|
1972
|
-
return; // a root worker — nobody to wake
|
|
1973
|
-
await this.#settleCompletionWake(workspaceId, parent.parent_worker_id, systemPrompt);
|
|
1974
|
-
}
|
|
1975
1378
|
// {§methods-event-subscribe} — subscriber failures are transport-local:
|
|
1976
1379
|
// log them at this boundary and never re-enter engine control flow.
|
|
1977
1380
|
#emitTo(workspaceId, method, params) {
|