@plurnk/plurnk-service 1.6.0 → 1.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. package/.env.defaults +7 -21
  2. package/INSTALL.md +18 -17
  3. package/README.md +1 -1
  4. package/SPEC.md +125 -115
  5. package/dist/build-info.json +1 -1
  6. package/dist/core/ChannelWrite.sql +3 -3
  7. package/dist/core/CoreSchemeServices.d.ts +1 -1
  8. package/dist/core/CoreSchemeServices.d.ts.map +1 -1
  9. package/dist/core/CoreSchemeServices.js +1 -1
  10. package/dist/core/CoreSchemeServices.js.map +1 -1
  11. package/dist/core/Dispatcher.d.ts +2 -2
  12. package/dist/core/Dispatcher.d.ts.map +1 -1
  13. package/dist/core/Dispatcher.js +79 -1115
  14. package/dist/core/Dispatcher.js.map +1 -1
  15. package/dist/core/Engine.d.ts +9 -33
  16. package/dist/core/Engine.d.ts.map +1 -1
  17. package/dist/core/Engine.js +58 -1656
  18. package/dist/core/Engine.js.map +1 -1
  19. package/dist/core/Engine.sql +46 -32
  20. package/dist/core/LogBody.d.ts +1 -0
  21. package/dist/core/LogBody.d.ts.map +1 -1
  22. package/dist/core/LogBody.js +4 -0
  23. package/dist/core/LogBody.js.map +1 -1
  24. package/dist/core/ModelCall.d.ts +2 -2
  25. package/dist/core/ModelCall.d.ts.map +1 -1
  26. package/dist/core/ModelCall.js +4 -2
  27. package/dist/core/ModelCall.js.map +1 -1
  28. package/dist/core/PacketBuilder.d.ts +11 -22
  29. package/dist/core/PacketBuilder.d.ts.map +1 -1
  30. package/dist/core/PacketBuilder.js +66 -168
  31. package/dist/core/PacketBuilder.js.map +1 -1
  32. package/dist/core/ProblemLog.d.ts +1 -1
  33. package/dist/core/ProblemLog.d.ts.map +1 -1
  34. package/dist/core/ProblemLog.js +14 -3
  35. package/dist/core/ProblemLog.js.map +1 -1
  36. package/dist/core/ProposalLifecycle.d.ts +2 -2
  37. package/dist/core/ProposalLifecycle.d.ts.map +1 -1
  38. package/dist/core/ProposalLifecycle.js +14 -5
  39. package/dist/core/ProposalLifecycle.js.map +1 -1
  40. package/dist/core/ProviderInstantiate.js +2 -2
  41. package/dist/core/ProviderInstantiate.js.map +1 -1
  42. package/dist/core/ResourceMutations.d.ts +73 -0
  43. package/dist/core/ResourceMutations.d.ts.map +1 -0
  44. package/dist/core/ResourceMutations.js +1142 -0
  45. package/dist/core/ResourceMutations.js.map +1 -0
  46. package/dist/core/SchemeRegistry.js +3 -3
  47. package/dist/core/SchemeRegistry.js.map +1 -1
  48. package/dist/core/StoredPacket.d.ts +2 -2
  49. package/dist/core/StoredPacket.js +7 -7
  50. package/dist/core/TurnRunner.d.ts +72 -0
  51. package/dist/core/TurnRunner.d.ts.map +1 -0
  52. package/dist/core/TurnRunner.js +1730 -0
  53. package/dist/core/TurnRunner.js.map +1 -0
  54. package/dist/core/caps/DbChannelCaps.js +4 -4
  55. package/dist/core/caps/DbChannelCaps.js.map +1 -1
  56. package/dist/core/content-hash.js +2 -2
  57. package/dist/core/content-hash.js.map +1 -1
  58. package/dist/core/content-weight.d.ts +2 -0
  59. package/dist/core/content-weight.d.ts.map +1 -0
  60. package/dist/core/content-weight.js +4 -0
  61. package/dist/core/content-weight.js.map +1 -0
  62. package/dist/core/content_weight.d.ts +4 -0
  63. package/dist/core/content_weight.d.ts.map +1 -0
  64. package/dist/core/content_weight.js +6 -0
  65. package/dist/core/content_weight.js.map +1 -0
  66. package/dist/core/fork.sql +5 -5
  67. package/dist/core/packet-wire.d.ts +4 -4
  68. package/dist/core/packet-wire.d.ts.map +1 -1
  69. package/dist/core/packet-wire.js +21 -21
  70. package/dist/core/packet-wire.js.map +1 -1
  71. package/dist/core/scheme-types.d.ts +1 -1
  72. package/dist/core/scheme-types.d.ts.map +1 -1
  73. package/dist/digest/Digest.js +1 -1
  74. package/dist/digest/Digest.js.map +1 -1
  75. package/dist/schemes/Exec.d.ts.map +1 -1
  76. package/dist/schemes/Exec.js +26 -10
  77. package/dist/schemes/Exec.js.map +1 -1
  78. package/dist/schemes/ExecOutputScheme.js +1 -1
  79. package/dist/schemes/ExecOutputScheme.js.map +1 -1
  80. package/dist/schemes/File.d.ts.map +1 -1
  81. package/dist/schemes/File.js +1 -1
  82. package/dist/schemes/File.js.map +1 -1
  83. package/dist/schemes/Log.d.ts.map +1 -1
  84. package/dist/schemes/Log.js +2 -2
  85. package/dist/schemes/Log.sql +2 -2
  86. package/dist/schemes/Worker.js +2 -2
  87. package/dist/schemes/Worker.js.map +1 -1
  88. package/dist/schemes/_entry-crud.js +4 -4
  89. package/dist/schemes/_entry-crud.js.map +1 -1
  90. package/dist/schemes/_entry-crud.sql +2 -2
  91. package/dist/schemes/_entry-find.d.ts +5 -5
  92. package/dist/schemes/_entry-find.d.ts.map +1 -1
  93. package/dist/schemes/_entry-find.js +29 -28
  94. package/dist/schemes/_entry-find.js.map +1 -1
  95. package/dist/schemes/_entry-manifest.js +4 -4
  96. package/dist/schemes/_entry-manifest.js.map +1 -1
  97. package/dist/schemes/_entry-ops.d.ts.map +1 -1
  98. package/dist/schemes/_entry-ops.js +3 -3
  99. package/dist/schemes/_entry-ops.js.map +1 -1
  100. package/dist/schemes/_entry-ops.sql +3 -3
  101. package/dist/server/Daemon.d.ts +7 -42
  102. package/dist/server/Daemon.d.ts.map +1 -1
  103. package/dist/server/Daemon.js +124 -887
  104. package/dist/server/Daemon.js.map +1 -1
  105. package/dist/server/DrainSupervisor.d.ts +117 -0
  106. package/dist/server/DrainSupervisor.d.ts.map +1 -0
  107. package/dist/server/DrainSupervisor.js +802 -0
  108. package/dist/server/DrainSupervisor.js.map +1 -0
  109. package/dist/server/lifecycle-recovery.sql +3 -0
  110. package/dist/server/logEntry.d.ts +1 -1
  111. package/dist/server/logEntry.js +1 -1
  112. package/dist/server/seam-entry-read.sql +2 -2
  113. package/dist/service.js +1 -1
  114. package/dist/service.js.map +1 -1
  115. package/migrations/001_schema.sql +20 -10
  116. package/package.json +33 -33
  117. package/dist/core/ruler_count.d.ts +0 -4
  118. package/dist/core/ruler_count.d.ts.map +0 -1
  119. package/dist/core/ruler_count.js +0 -7
  120. package/dist/core/ruler_count.js.map +0 -1
  121. package/dist/core/token-ruler.d.ts +0 -2
  122. package/dist/core/token-ruler.d.ts.map +0 -1
  123. package/dist/core/token-ruler.js +0 -5
  124. package/dist/core/token-ruler.js.map +0 -1
@@ -2,16 +2,13 @@
2
2
  // the transport-free plugin-module seam ({§rpc}).
3
3
  import { readFile } from "node:fs/promises";
4
4
  import { resolve, dirname } from "node:path";
5
- import { setTimeout as delay } from "node:timers/promises";
6
- import { execPollBackoffMs } from "./exec-poll-backoff.js";
7
- import ChannelWrite from "../core/ChannelWrite.js";
8
- import { Paths } from "../index.js";
5
+ import Paths from "../Paths.js";
9
6
  import Engine from "../core/Engine.js";
10
7
  import ExecutorRegistry from "../core/ExecutorRegistry.js";
11
8
  import SchemeRegistry from "../core/SchemeRegistry.js";
12
9
  import { Mimetypes } from "@plurnk/plurnk-mimetypes";
13
10
  import { RuntimeDeclaration } from "@plurnk/plurnk-execs";
14
- import { aggregateProviderAccounting } from "@plurnk/plurnk-providers";
11
+ import DrainSupervisor from "./DrainSupervisor.js";
15
12
  import { parsePath, Validator, } from "@plurnk/plurnk-contracts";
16
13
  import LogEntry from "./logEntry.js";
17
14
  import Envelope from "./envelope.js";
@@ -23,19 +20,15 @@ import Fork from "../core/fork.js";
23
20
  import WorkerName from "../core/WorkerName.js";
24
21
  import LoopLifecycle from "../core/LoopLifecycle.js";
25
22
  import { promptLoopPrefix } from "../core/plurnk-uri.js";
26
- import { rulerCount } from "../core/token-ruler.js";
23
+ import { contentWeight } from "../core/content-weight.js";
27
24
  import { parseAliasesFromEnv, resolveActiveAlias } from "@plurnk/plurnk-providers";
28
25
  import ProviderInstantiate from "../core/ProviderInstantiate.js";
29
26
  import { resolveLoopAlias } from "./loop-model.js";
30
- import { DEFAULT_LOOP_FLAGS } from "../core/scheme-types.js";
31
27
  import LoopFlagsReader from "../core/LoopFlagsReader.js";
32
28
  import Results, { OperationFailureError } from "../core/results.js";
33
29
  import WorkspaceGate from "../core/WorkspaceGate.js";
34
30
  import BranchBatches from "./BranchBatches.js";
35
- import ErrorDetail from "../core/ErrorDetail.js";
36
31
  import { observed, observedSync } from "../observe/spans.js";
37
- import { LOOP_TERMINALS, recordCounter } from "../observe/metrics.js";
38
- import { readOptimisticSettlementMs } from "../core/optimistic-settlement.js";
39
32
  const clientActionFailure = (error) => {
40
33
  if (error instanceof OperationFailureError)
41
34
  return error.result;
@@ -53,6 +46,7 @@ export default class Daemon {
53
46
  #workspaceGate;
54
47
  #branchBatches;
55
48
  #lifecycle;
49
+ #drains;
56
50
  #schemes;
57
51
  #mimetypes;
58
52
  #ownsMimetypes;
@@ -68,40 +62,6 @@ export default class Daemon {
68
62
  // module (plurnk-agui) subscribes and fans out to its OWN clients; core emits, never owns
69
63
  // client transport or connection state.
70
64
  #eventSubscribers = new Set();
71
- // Worker-level drain registry. At most one drain per worker. The stored object
72
- // is the drain's identity handle: start/exit compare it by reference so a
73
- // drain exiting never clobbers a successor that raced in, and a loop
74
- // enqueued during teardown is never stranded. A drain is a pure queue
75
- // consumer (claim → run → exit on empty queue); streams live independently
76
- // (subscriptions + Exec.idle), and a concluding stream routes through
77
- // inject() like any other loop source.
78
- #activeDrains = new Map();
79
- #drainExitTasks = new Set();
80
- // Per-worker cancellation scope. Loops AND the streams they spawn (execs)
81
- // share this signal, so loop.cancel / shutdown abort it once and every
82
- // in-flight subscription tears down — even a spawn that registers AFTER the
83
- // cancel self-aborts against the already-aborted signal (no race). Outlives
84
- // any single (ephemeral) drain; replaced with a fresh controller once
85
- // aborted so a later runLoop request isn't born cancelled.
86
- #workerAborts = new Map();
87
- // grammar 0.74.20 EXEC `<T,P>` — per-worker hibernation poll-wake timer. When a loop parks at
88
- // a park with a polled stream, a timer fires every P seconds to resume it ({§exec-poll}). One
89
- // per worker (the tightest cadence); cleared/replaced on each park and on cancel.
90
- #parkTimers = new Map();
91
- #pollTimers = new Map();
92
- #pollBackoff = new Map(); // {§exec-poll} — backoff step per worker
93
- // Per-worker drain-transition lock — see #withDrainLock (R4 / {§worker-lifecycle-single-drain}).
94
- #drainLocks = new Map();
95
- // {§worker-lifecycle-child-wake} — workers owed a wake: a child/stream conclusion fired while the worker was
96
- // mid-turn (not yet slept), so #wakeParkedWorker could not resume it. A child-worker conclusion is a
97
- // BOUNDED, lossless wake (a worker always concludes), so a hibernation awaiting one MUST return —
98
- // never deadlock. The drain honors the owed wake at the worker's next park, closing the conclude-
99
- // before-park race. (Only a live exec stream, unbounded absent a timeout, may hold a park open.)
100
- #owedWakes = new Set();
101
- // {§worker-optimistic-settlement} — one non-sliding completion-wake batch
102
- // per worker. Durable conclusions and client events land before this gate;
103
- // only the parked loop's provider-dispatch requeue waits.
104
- #completionWakeGates = new Map();
105
65
  constructor({ db, schemes, mimetypes, provider, nodeModulesPath, }) {
106
66
  this.#db = db;
107
67
  this.#lifecycle = new LoopLifecycle(db);
@@ -149,7 +109,7 @@ export default class Daemon {
149
109
  }))?.id;
150
110
  if (workerId === undefined)
151
111
  throw new Error("Branch worker insert returned no row");
152
- const loopId = await this.#enqueueFreshLoop({
112
+ const loopId = await this.#drains.enqueueFreshLoop({
153
113
  workerId,
154
114
  prompt,
155
115
  providerSpec,
@@ -160,7 +120,7 @@ export default class Daemon {
160
120
  },
161
121
  startChild: async (workspaceId, workerId, loopId) => {
162
122
  const systemPrompt = await readFile(Paths.instructionsSystem, "utf8");
163
- const started = await this.#ensureDrain({ workspaceId, workerId, systemPrompt });
123
+ const started = await this.#drains.ensureDrain({ workspaceId, workerId, systemPrompt });
164
124
  if (started === null)
165
125
  throw new Error(`Branch worker ${workerId} already has a live drain`);
166
126
  const result = await started.firstLoopPromise;
@@ -171,7 +131,7 @@ export default class Daemon {
171
131
  },
172
132
  wakeParent: async (workspaceId, workerId) => {
173
133
  const systemPrompt = await readFile(Paths.instructionsSystem, "utf8");
174
- await this.#settleCompletionWake(workspaceId, workerId, systemPrompt, false);
134
+ await this.#drains.settleCompletionWake(workspaceId, workerId, systemPrompt, false);
175
135
  },
176
136
  notify: (workspaceId, payload) => {
177
137
  this.#broadcast({ workspaceId }, "workspace/branch-batch", payload);
@@ -179,13 +139,12 @@ export default class Daemon {
179
139
  });
180
140
  this.#engine = new Engine({
181
141
  db, schemes: this.#schemes, mimetypes: this.#mimetypes,
182
- // {§tokenomics-agnostic-ruler} — the ONE model-facing token ruler (chars/2), NOT the
183
- // boot provider: token accounting is workspace-wide across many concurrent models, so
184
- // the write-time + catalog counts must be model-independent. Exact per-model counting
185
- // lives only at the packet-materialization fit-gate.
186
- tokenize: rulerCount,
142
+ // {§tokenomics-agnostic-ruler} — stored and catalog curation weights
143
+ // are workspace-wide across concurrent models, so they remain
144
+ // model-independent. Request-shaped token facts stay provider-owned.
145
+ weigh: contentWeight,
187
146
  streamEventNotify: (workspaceId, event) => this.notifyStreamEvent(workspaceId, event),
188
- wakeWorkerNotify: (payload) => { void this.#handleWakeWorker(payload); },
147
+ wakeWorkerNotify: (payload) => this.#drains.notifyWakeWorker(payload),
189
148
  // worker:// loop-start primitive — spawn/fork/irc deliver through
190
149
  // Daemon.inject (active sister → fold; idle → enqueue + drain). The
191
150
  // daemon owns provider + the law-file system prompt; the worker scheme
@@ -218,10 +177,58 @@ export default class Daemon {
218
177
  workspaceTurnCompleted: async ({ turnId }) => this.#branchBatches.sealTurn(turnId),
219
178
  // worker:// KILL (terminate) — cancel the addressed worker subtree and
220
179
  // tear down its held streams before the operation completes.
221
- cancelWorker: async (workerId, reason) => this.#cancelWorkerTree(workerId, reason),
222
- cancelDescendants: async (workerId, reason) => this.#cancelTree(workerId, reason, false),
180
+ cancelWorker: async (workerId, reason) => this.#drains.cancelWorkerTree(workerId, reason),
181
+ cancelDescendants: async (workerId, reason) => this.#drains.cancelDescendants(workerId, reason),
223
182
  noticeNotify: (workspaceId, payload) => this.notifyNotice(workspaceId, payload),
224
183
  });
184
+ this.#drains = new DrainSupervisor({
185
+ db,
186
+ lifecycle: this.#lifecycle,
187
+ injectPrompt: (workerId, prompt, openPaths) => this.#engine.inject(workerId, prompt, openPaths),
188
+ assertInjectionCompatibility: async ({ workerId, loopId, providerSpec, childProviderSpec, turnCeiling, flags, }) => {
189
+ await this.#assertFoldPosture(workerId, flags, loopId);
190
+ await this.#assertLoopProvider(loopId, providerSpec);
191
+ if (childProviderSpec !== undefined) {
192
+ await this.#assertLoopChildProvider(loopId, childProviderSpec);
193
+ }
194
+ await this.#assertLoopMaxTurns(loopId, turnCeiling?.source === "explicit" ? turnCeiling.effective : undefined);
195
+ },
196
+ reconcilePrompts: (workerId, endedLoopId) => this.#reconcileOrphanedPrompts(workerId, endedLoopId),
197
+ runLoop: async ({ workspaceId, workerId, loopId, maxTurns, prompt, systemPrompt, signal, onDispatch, }) => {
198
+ const { provider, childProvider } = await this.#providersForLoop(loopId);
199
+ return this.#engine.runLoop({
200
+ provider,
201
+ childProvider,
202
+ workspaceId,
203
+ workerId,
204
+ loopId,
205
+ maxTurns,
206
+ messages: [
207
+ { role: "system", content: systemPrompt },
208
+ { role: "user", content: prompt },
209
+ ],
210
+ origin: "model",
211
+ signal,
212
+ onDispatch,
213
+ });
214
+ },
215
+ loopUsage: (loopId) => this.#engine.loopUsage(loopId),
216
+ loopAttributions: (loopId) => this.#engine.loopAttributions(loopId),
217
+ takeParkDeadline: (loopId) => {
218
+ const deadline = this.#engine.parkDeadlines.get(loopId);
219
+ this.#engine.parkDeadlines.delete(loopId);
220
+ return deadline;
221
+ },
222
+ settleSearch: (target, status) => this.#engine.searchGate.settle(target, status),
223
+ cancelSubscription: (subscriptionId) => this.#engine.cancelSubscription(subscriptionId),
224
+ hasActiveStreams: (workerId) => this.#workerHasActiveStreams(workerId),
225
+ readSystemPrompt: () => readFile(Paths.instructionsSystem, "utf8"),
226
+ emitLogEntry: async (workspaceId, logEntryId) => {
227
+ const entry = await LogEntry.fetchLogEntry(this.#db, logEntryId);
228
+ this.#broadcast({ workspaceId }, "log/entry", { entry });
229
+ },
230
+ emit: (workspaceId, method, params) => this.#broadcast({ workspaceId }, method, params),
231
+ });
225
232
  // Wire proposal-pending events to the loop/proposal WS notification.
226
233
  // Sessionid scopes the broadcast to clients on the same workspace.
227
234
  this.#engine.onProposalPending((event) => {
@@ -615,9 +622,7 @@ export default class Daemon {
615
622
  const isActive = active !== null && active.alias === a.alias;
616
623
  return {
617
624
  alias: a.alias, provider: a.provider, model: a.model, active: isActive,
618
- // The same effective model-facing budget loop usage reports, including
619
- // optional virtual pressure; known for the active alias, null elsewhere.
620
- promptBudget: isActive && this.#provider !== null ? this.#engine.promptBudgetFor(this.#provider) : null,
625
+ inputCapacity: isActive && this.#provider !== null ? this.#provider.inputCapacity : null,
621
626
  };
622
627
  }),
623
628
  };
@@ -860,7 +865,7 @@ export default class Daemon {
860
865
  contentOffset: c.contentOffset,
861
866
  contentLength: c.contentLength,
862
867
  mimetype: c.mimetype,
863
- tokens: c.tokens,
868
+ weight: c.weight,
864
869
  state: c.state,
865
870
  };
866
871
  }
@@ -981,6 +986,7 @@ export default class Daemon {
981
986
  if (this.#started)
982
987
  throw new Error("daemon already started");
983
988
  this.#started = true;
989
+ this.#drains.start();
984
990
  // Mimetypes owns its own discovery scan over @plurnk/plurnk-mimetypes-*
985
991
  // packages; pre-warm it so first index render doesn't pay the cost.
986
992
  await this.#mimetypes.ready();
@@ -1034,12 +1040,12 @@ export default class Daemon {
1034
1040
  await this.#branchBatches.recover();
1035
1041
  const orphanSources = await this.#db.recovery_orphan_prompt_sources.all({});
1036
1042
  for (const source of orphanSources) {
1037
- await this.#reconcileOrphanedPrompts(source.worker_id, source.loop_id);
1043
+ await this.#drains.reconcileOrphanedPrompts(source.worker_id, source.loop_id);
1038
1044
  }
1039
1045
  const systemPrompt = await readFile(Paths.instructionsSystem, "utf8");
1040
1046
  const queued = await this.#db.recovery_queued_workers.all({});
1041
1047
  for (const row of queued) {
1042
- const started = await this.#ensureDrain({
1048
+ const started = await this.#drains.ensureDrain({
1043
1049
  workspaceId: row.workspace_id,
1044
1050
  workerId: row.worker_id,
1045
1051
  systemPrompt,
@@ -1050,7 +1056,7 @@ export default class Daemon {
1050
1056
  }
1051
1057
  const parked = await this.#db.recovery_parked_workers.all({});
1052
1058
  for (const row of parked) {
1053
- await this.#schedulePollWake(row.workspace_id, row.worker_id, systemPrompt);
1059
+ await this.#drains.schedulePollWake(row.workspace_id, row.worker_id, systemPrompt);
1054
1060
  }
1055
1061
  }
1056
1062
  async stop() {
@@ -1064,8 +1070,8 @@ export default class Daemon {
1064
1070
  .toReversed()
1065
1071
  .map((module) => Promise.resolve().then(() => module.close())));
1066
1072
  this.#moduleClosers = [];
1067
- // Drain order: (1) abort in-flight loops via #activeDrains so
1068
- // strike paths don't keep going, (2) await each drain's promise
1073
+ // Drain order: (1) tell the supervisor to abort worker scopes so
1074
+ // strike paths don't keep going, (2) await its active drains
1069
1075
  // to completion, (3) drain streaming schemes' background work
1070
1076
  // (exec spawn cleanup, channel writes). Only THEN close the DB
1071
1077
  // upstream — drain queries hit the DB right up until they exit.
@@ -1080,24 +1086,9 @@ export default class Daemon {
1080
1086
  this.#engine.cancelAllProposals("daemon_stopping");
1081
1087
  this.#engine.cancelDerivations(derivationAbort);
1082
1088
  this.#branchBatches.beginStop();
1083
- for (const scope of this.#workerAborts.values()) {
1084
- if (!scope.signal.aborted)
1085
- scope.abort("daemon_stopping");
1086
- }
1087
- for (const t of this.#pollTimers.values())
1088
- clearTimeout(t); // drop pending hibernation poll-wakes
1089
- this.#pollBackoff.clear();
1090
- this.#pollTimers.clear();
1091
- // Cancel park-deadline timers before DB close; otherwise a late #wakeParkedWorker would run after
1092
- // stop/db-close if left pending — an unhandled rejection (SqlRite closed) that abnormally
1093
- // exits the worker under load. Symmetric with the poll-wakes above; both must be reaped.
1094
- for (const t of this.#parkTimers.values())
1095
- clearTimeout(t);
1096
- this.#parkTimers.clear();
1089
+ this.#drains.beginStop("daemon_stopping");
1097
1090
  await this.#branchBatches.idle();
1098
- const drainPromises = [...this.#activeDrains.values()].map((d) => d.promise);
1099
- await Promise.allSettled(drainPromises);
1100
- await Promise.allSettled([...this.#drainExitTasks]);
1091
+ const [drainResult] = await Promise.allSettled([this.#drains.idle()]);
1101
1092
  const closeResults = await moduleClose;
1102
1093
  const [streamingResult] = await Promise.allSettled([this.#drainStreamingSchemes()]);
1103
1094
  const [derivationResult] = await Promise.allSettled([
@@ -1107,7 +1098,19 @@ export default class Daemon {
1107
1098
  ? await Promise.allSettled([this.#mimetypes.dispose()])
1108
1099
  : [];
1109
1100
  const [schemeResult] = await Promise.allSettled([this.#schemes.close()]);
1110
- const closeErrors = [...closeResults, streamingResult, derivationResult, ...mimetypeResults, schemeResult]
1101
+ // Streaming and scheme closure are the last producers of synchronous
1102
+ // conclusion notifications. Join the supervisor-owned async tails only
1103
+ // after those producers settle, before the caller may close SQLite.
1104
+ const [wakeResult] = await Promise.allSettled([this.#drains.idle()]);
1105
+ const closeErrors = [
1106
+ ...closeResults,
1107
+ drainResult,
1108
+ streamingResult,
1109
+ derivationResult,
1110
+ ...mimetypeResults,
1111
+ schemeResult,
1112
+ wakeResult,
1113
+ ]
1111
1114
  .filter((result) => result.status === "rejected")
1112
1115
  .flatMap((result) => result.reason instanceof AggregateError
1113
1116
  ? [...result.reason.errors]
@@ -1136,18 +1139,6 @@ export default class Daemon {
1136
1139
  notifyNotice(workspaceId, payload) {
1137
1140
  this.#broadcast({ workspaceId }, "notice/event", payload);
1138
1141
  }
1139
- /**
1140
- * Inject a prompt into a worker. Two paths:
1141
- * - Active drain: writes the next prompt:///<loop>/<N> entry via
1142
- * Engine.inject. The current loop publishes it at its next
1143
- * turn. Returns immediately with {action: "injected_next_turn"}.
1144
- * - No active drain: enqueues a fresh loop with the prompt at
1145
- * status=100, starts a drain. Returns the drain promise so the
1146
- * caller can await full completion.
1147
- *
1148
- * Both `runLoop` and wake-on-completion go through this method
1149
- * ({§actor-boundary-passive-wake}).
1150
- */
1151
1142
  // {§methods-loop-run-fold-consistency} — a folded prompt cannot reconfigure its loop.
1152
1143
  async #assertFoldPosture(workerId, flags, loopId) {
1153
1144
  if (flags === undefined || Object.keys(flags).length === 0)
@@ -1168,810 +1159,56 @@ export default class Daemon {
1168
1159
  });
1169
1160
  }
1170
1161
  }
1171
- async inject(args) {
1172
- const { workspaceId, workerId, prompt } = args;
1173
- // Active loop (status=102)? Fold the wake/prompt into its next turn.
1174
- // engine.inject returns null when no loop is currently executing, so
1175
- // we enqueue a fresh loop below and ensure a drain claims it.
1176
- if (this.#activeDrains.has(workerId)) {
1177
- const active = await this.#db.drain_current_loop_for_worker.get({ worker_id: workerId });
1178
- if (active !== undefined) {
1179
- await this.#assertFoldPosture(workerId, args.flags, active.id); // compare with the exact durable loop
1180
- await this.#assertLoopProvider(active.id, args.providerSpec);
1181
- if (args.childProviderSpec !== undefined)
1182
- await this.#assertLoopChildProvider(active.id, args.childProviderSpec);
1183
- await this.#assertLoopMaxTurns(active.id, args.turnCeiling?.source === "explicit" ? args.turnCeiling.effective : undefined);
1184
- }
1185
- const result = await this.#engine.inject(workerId, prompt, args.openPaths ?? []);
1186
- if (result !== null) {
1187
- return { action: "injected_next_turn", loopId: result.loopId, turnSeq: result.turnSeq };
1188
- }
1189
- }
1190
- // {§worker-lifecycle-wake-requeue-not-terminal} — a worker parked at 202 resumes that loop in place:
1191
- // is a wake edge like a stream/child conclusion, not a fresh loop that orphans the parked one
1192
- // (which would leave the worker non-quiescent forever). engine.inject writes the message as the
1193
- // slept loop's next-turn prompt (the directed message — distinct from the env door, which
1194
- // resumes promptless); then re-queue + drain it. {§worker-lifecycle-wake-liveness}.
1195
- if (!this.#activeDrains.has(workerId)) {
1196
- const slept = await this.#db.drain_find_slept_loop.get({ worker_id: workerId });
1197
- if (slept !== undefined) {
1198
- await this.#assertFoldPosture(workerId, args.flags, slept.id); // resume drops nothing silently
1199
- await this.#assertLoopProvider(slept.id, args.providerSpec);
1200
- if (args.childProviderSpec !== undefined)
1201
- await this.#assertLoopChildProvider(slept.id, args.childProviderSpec);
1202
- await this.#assertLoopMaxTurns(slept.id, args.turnCeiling?.source === "explicit" ? args.turnCeiling.effective : undefined);
1203
- const injected = await this.#engine.inject(workerId, prompt, args.openPaths ?? []);
1204
- await this.#lifecycle.wake(slept.id);
1205
- const started = await this.#ensureDrain({
1206
- workspaceId, workerId, systemPrompt: args.systemPrompt,
1207
- });
1208
- return { action: "injected_next_turn", loopId: slept.id, ...(injected?.turnSeq !== undefined ? { turnSeq: injected.turnSeq } : {}), ...(started ?? {}) };
1209
- }
1210
- }
1211
- const loopId = await this.#enqueueFreshLoop({
1212
- workerId,
1213
- prompt,
1214
- providerSpec: args.providerSpec,
1215
- childProviderSpec: args.childProviderSpec ?? null,
1216
- maxTurns: args.turnCeiling?.effective,
1217
- flags: args.flags,
1218
- openPaths: args.openPaths,
1219
- });
1220
- // Guarantee a drain claims the loop we just enqueued. #ensureDrain runs its
1221
- // check-and-start UNDER the per-worker drain lock ({§worker-lifecycle-single-drain}),
1222
- // serialized against a draining sibling's teardown relinquish so the two can't
1223
- // both register a drain (R4). A live drain re-claims the loop in its own
1224
- // iteration or its lock-held exit re-claim, so it's never stranded.
1225
- // firstLoopPromise is present only when THIS call started the drain — runLoop
1226
- // keys its fast-path response on that.
1227
- const started = await this.#ensureDrain({
1228
- workspaceId, workerId, systemPrompt: args.systemPrompt,
1229
- });
1230
- return { action: "enqueued_new_loop", loopId, ...(started ?? {}) };
1231
- }
1232
- async #enqueueFreshLoop(args) {
1233
- // {§worker-lifecycle-single-drain}: sequence allocation and insertion
1234
- // are one queue mutation; another accepted prompt cannot claim the gap.
1235
- return this.#withDrainLock(args.workerId, async () => {
1236
- const seqRow = await this.#db.loop_run_next_sequence.get({
1237
- worker_id: args.workerId,
1238
- });
1239
- if (seqRow === undefined)
1240
- throw new Error("enqueueFreshLoop: next-sequence query returned no row");
1241
- const loopRow = await this.#db.drain_enqueue_loop.get({
1242
- worker_id: args.workerId,
1243
- sequence: seqRow.next,
1244
- prompt: args.prompt,
1245
- provider_spec: JSON.stringify(args.providerSpec),
1246
- child_provider_spec: JSON.stringify(args.childProviderSpec),
1247
- max_turns: args.maxTurns ?? Number(process.env.PLURNK_SERVICE_MAX_TURNS ?? "50"),
1248
- });
1249
- if (loopRow === undefined)
1250
- throw new Error("enqueueFreshLoop: loop enqueue returned no row");
1251
- if (args.flags !== undefined) {
1252
- await this.#db.engine_set_loop_flags.run({
1253
- loop_id: loopRow.id,
1254
- flags: JSON.stringify({ ...DEFAULT_LOOP_FLAGS, ...args.flags }),
1255
- });
1256
- }
1257
- if (args.openPaths !== undefined && args.openPaths.length > 0) {
1258
- await this.#db.engine_set_loop_open_paths.run({
1259
- loop_id: loopRow.id,
1260
- open_paths: JSON.stringify(args.openPaths),
1261
- });
1262
- }
1263
- return loopRow.id;
1264
- });
1162
+ inject(args) {
1163
+ return this.#drains.inject(args);
1265
1164
  }
1266
- /**
1267
- * Start a drain for the given worker. The drain claims queued loops via
1268
- * drain_claim_next_loop (atomic 100→102 flip), executes each via
1269
- * Engine.runLoop, and re-checks. Stream-aware: when the queue is empty
1270
- * but the worker has active subscriptions, the drain parks on a
1271
- * #drainPokes signal — wake-on-completion → inject() wakes it. Drain
1272
- * exits when queue is empty AND no active subscriptions remain.
1273
- *
1274
- * Returns both `firstLoopPromise` (resolves once the first loop the
1275
- * drain processes completes — used by runLoop to give the caller a
1276
- * fast response containing their loop's result) and `drainPromise`
1277
- * (resolves only when the whole drain finishes, queue+subs settled).
1278
- */
1279
- #startDrain(opts) {
1280
- const { workspaceId, workerId, systemPrompt } = opts;
1281
- // The drain runs under the worker's cancellation scope (shared with the
1282
- // execs its loops spawn), so loop.cancel/shutdown abort it as a unit.
1283
- const controller = this.#workerSignal(workerId);
1284
- const handle = {
1285
- controller, promise: Promise.resolve(),
1286
- };
1287
- let resolveFirst = () => { };
1288
- let rejectFirst = () => { };
1289
- const firstLoopPromise = new Promise((res, rej) => {
1290
- resolveFirst = res;
1291
- rejectFirst = rej;
1292
- });
1293
- let firstSettled = false;
1294
- const claim = () => this.#db.drain_claim_next_loop.get({ worker_id: workerId });
1295
- const drainPromise = (async () => {
1296
- let loopsDrained = 0;
1297
- let lastResult = null;
1298
- let currentLoopId = null; // the loop being drained — for abort→499 settlement
1299
- try {
1300
- while (true) {
1301
- controller.signal.throwIfAborted();
1302
- let loopRow = await claim();
1303
- if (loopRow === undefined) {
1304
- // Queue empty → teardown UNDER the per-worker drain lock (R4 / I1),
1305
- // serialized against #ensureDrain so a concurrent inject can't
1306
- // start a 2nd drain in the gap. Re-claim while holding the lock;
1307
- // relinquish the registry slot only if it's empty too. A loop
1308
- // that raced in is returned and run — we stay registered, so
1309
- // there's no transient delete for #ensureDrain to catch.
1310
- loopRow = await this.#withDrainLock(workerId, async () => {
1311
- const claimed = await claim();
1312
- if (claimed === undefined && this.#activeDrains.get(workerId) === handle) {
1313
- this.#activeDrains.delete(workerId);
1314
- }
1315
- return claimed;
1316
- });
1317
- if (loopRow === undefined)
1318
- break;
1319
- }
1320
- currentLoopId = loopRow.id;
1321
- // {§methods-loop-run-model} — provider identity belongs to the claimed loop, not the
1322
- // drain that happened to claim it. A drain can consume multiple
1323
- // queued loops; resolve each durable selection at this boundary.
1324
- const { provider, childProvider } = await this.#providersForLoop(loopRow.id);
1325
- const onDispatch = (logEntryId) => {
1326
- // {§methods-event-subscribe} — a log-broadcast failure must never crash the drain.
1327
- void (async () => {
1328
- const entry = await LogEntry.fetchLogEntry(this.#db, logEntryId);
1329
- this.#broadcast({ workspaceId }, "log/entry", { entry });
1330
- })().catch((e) => console.error("log/entry broadcast failed:", e instanceof Error ? e.message : String(e)));
1331
- };
1332
- const result = await observed(// {§observability-boundary}
1333
- "loop.run", { workspaceId, workerId, "loop.id": loopRow.id }, async (span) => {
1334
- const loopResult = await this.#engine.runLoop({
1335
- provider, childProvider, workspaceId, workerId, loopId: loopRow.id, maxTurns: loopRow.max_turns,
1336
- messages: [
1337
- { role: "system", content: systemPrompt },
1338
- { role: "user", content: loopRow.prompt },
1339
- ],
1340
- origin: "model",
1341
- onDispatch,
1342
- signal: controller.signal,
1343
- });
1344
- span.setAttribute("status", loopResult.result.status);
1345
- recordCounter(LOOP_TERMINALS, { status: loopResult.result.status });
1346
- return loopResult;
1347
- });
1348
- if (result.result.status === 202) {
1349
- // The loop slept via SEND signal 202 — suspended, not terminated. Leave it at 202
1350
- // (resumable); no loop/terminated, no orphan-reconcile. A stream conclusion
1351
- // (#handleWakeWorker) re-queues it; and if it holds a polled stream, a poll timer
1352
- // wakes it every P to inspect ({§exec-poll}). {§worker-lifecycle-wake-liveness}.
1353
- void this.#schedulePollWake(workspaceId, workerId, systemPrompt).catch((err) => console.error("poll-wake scheduling failed:", err instanceof Error ? err.message : String(err)));
1354
- // {§send-premature-terminate}/scoped SEND signal 202 — the park deadline:
1355
- // dispatcher recorded the marker's seconds; a bounded park is woken at T
1356
- // regardless of arrivals, so a park always has a next turn. -1 (indefinite:
1357
- // the butler, a [300] ask) schedules nothing — irc/inject/conclusions wake it.
1358
- // In-memory: a daemon restart drops pending deadlines.
1359
- if (currentLoopId !== null) {
1360
- const deadline = this.#engine.parkDeadlines.get(currentLoopId);
1361
- this.#engine.parkDeadlines.delete(currentLoopId);
1362
- const prior = this.#parkTimers.get(workerId);
1363
- if (prior !== undefined) {
1364
- clearTimeout(prior);
1365
- this.#parkTimers.delete(workerId);
1366
- }
1367
- if (deadline !== undefined && deadline > 0) {
1368
- const t = setTimeout(() => {
1369
- this.#parkTimers.delete(workerId);
1370
- void this.#wakeParkedWorker(workspaceId, workerId, systemPrompt).catch((err) => console.error("park-deadline wake failed:", err instanceof Error ? err.message : String(err)));
1371
- }, deadline * 1000);
1372
- t.unref();
1373
- this.#parkTimers.set(workerId, t);
1374
- }
1375
- }
1376
- // Honor an OWED wake ({§worker-lifecycle-child-wake}): a child/stream concluded while
1377
- // this worker was mid-turn, before it slept — resume in place rather than park blind,
1378
- // so a worker hibernation always returns. The loop is 202 here; reset to
1379
- // claimable and the drain re-runs it on the next claim below.
1380
- if (this.#owedWakes.delete(workerId)) {
1381
- await this.#lifecycle.wake(loopRow.id);
1382
- currentLoopId = null;
1383
- continue;
1384
- }
1385
- // The loop is blocked at 202 on a live obligation ({§wait-obligation-matrix});
1386
- // that obligation's conclusion is its wake edge (the owed-wake above covers the
1387
- // conclude-before-block race). An idle wait never reaches here — it concluded at dispatch.
1388
- currentLoopId = null;
1389
- continue;
1390
- }
1391
- this.#owedWakes.delete(workerId); // the loop concluded (non-202) — no park to honor a held wake at
1392
- const [usage, attributions, turnIds] = await Promise.all([
1393
- this.#engine.loopUsage(loopRow.id),
1394
- this.#engine.loopAttributions(loopRow.id),
1395
- this.#lifecycle.turnIds(loopRow.id),
1396
- ]);
1397
- this.#broadcast({ workspaceId }, "loop/terminated", {
1398
- workerId,
1399
- loopId: loopRow.id,
1400
- result: result.result,
1401
- hitMaxTurns: result.hitMaxTurns,
1402
- turnIds,
1403
- usage,
1404
- attributions,
1405
- });
1406
- loopsDrained++;
1407
- const loopResult = {
1408
- loopId: loopRow.id,
1409
- turnIds,
1410
- result: result.result,
1411
- hitMaxTurns: result.hitMaxTurns,
1412
- usage,
1413
- attributions,
1414
- };
1415
- lastResult = loopResult;
1416
- if (!firstSettled) {
1417
- firstSettled = true;
1418
- resolveFirst(loopResult);
1419
- }
1420
- // A next-turn prompt this loop ended before consuming (a
1421
- // wake conclusion or a runLoop-while-active prompt) is promoted to
1422
- // a fresh queued loop so it's never silently dropped.
1423
- await this.#reconcileOrphanedPrompts(workerId, loopRow.id);
1424
- currentLoopId = null;
1425
- }
1426
- }
1427
- catch (err) {
1428
- if (controller.signal.aborted) {
1429
- // {§methods-loop-cancel} — loop.cancel / shutdown aborted the live drain. A cancellation
1430
- // is the loop's TERMINAL state (499), delivered via loop/terminated (runLoop no
1431
- // longer blocks to return it). A genuine error rejects firstLoopPromise.
1432
- let usage = {
1433
- accounting: aggregateProviderAccounting([]),
1434
- contextTokens: null,
1435
- promptBudget: null,
1436
- meta: {},
1437
- };
1438
- let attributions = [];
1439
- const message = ErrorDetail.preview(controller.signal.reason ?? "user_cancelled")
1440
- || "no reason was supplied";
1441
- if (currentLoopId !== null) {
1442
- // {§methods-loop-cancel}/{§worker-lifecycle-terminal-result} —
1443
- // persist the exact 499 cancellation result before broadcasting it.
1444
- const cancelled = await this.#lifecycle.finish(currentLoopId, Results.failure("lifecycle:cancel", "loop-cancelled", 499, `The loop was cancelled: ${message}.`, {}, {
1445
- reason: message,
1446
- stage: "loop",
1447
- retryable: false,
1448
- }), { terminatedBy: "cancel" });
1449
- [usage, attributions] = await Promise.all([
1450
- this.#engine.loopUsage(currentLoopId),
1451
- this.#engine.loopAttributions(currentLoopId),
1452
- ]);
1453
- if (cancelled !== null) {
1454
- this.#broadcast({ workspaceId }, "loop/terminated", {
1455
- workerId,
1456
- loopId: currentLoopId,
1457
- result: cancelled,
1458
- hitMaxTurns: false,
1459
- turnIds: await this.#lifecycle.turnIds(currentLoopId),
1460
- usage,
1461
- attributions,
1462
- });
1463
- }
1464
- }
1465
- if (!firstSettled) {
1466
- firstSettled = true;
1467
- resolveFirst({
1468
- loopId: currentLoopId ?? 0,
1469
- turnIds: [],
1470
- result: currentLoopId === null
1471
- ? Results.failure("lifecycle:cancel", "loop-cancelled", 499, `The loop was cancelled: ${message}.`, {}, {
1472
- reason: message,
1473
- stage: "loop",
1474
- retryable: false,
1475
- })
1476
- : await this.#lifecycle.result(currentLoopId)
1477
- ?? Results.failure("lifecycle:cancel", "loop-cancelled", 499, `The loop was cancelled: ${message}.`, {}, {
1478
- reason: message,
1479
- stage: "loop",
1480
- retryable: false,
1481
- }),
1482
- hitMaxTurns: false,
1483
- usage,
1484
- });
1485
- }
1486
- }
1487
- else {
1488
- // {§worker-lifecycle-terminal-result} — a non-abort drain
1489
- // failure becomes an exact durable 500 and terminal notification;
1490
- // daemon diagnostics retain the complete caught error.
1491
- console.error(`drain error (workspace ${workspaceId}, worker ${workerId}, loop ${currentLoopId ?? "?"}):`, err);
1492
- if (currentLoopId !== null) {
1493
- const failure = err instanceof OperationFailureError
1494
- ? err.result
1495
- : Results.failure("daemon:drain", "loop-threw", 500, "The loop failed outside its operation result contract.", {}, {
1496
- stage: "loop",
1497
- retryable: false,
1498
- });
1499
- const settled = await this.#lifecycle.finish(currentLoopId, failure)
1500
- ?? await this.#lifecycle.result(currentLoopId);
1501
- if (settled === null) {
1502
- throw new Error(`drain could not settle loop ${currentLoopId}`, { cause: err });
1503
- }
1504
- const [usage, attributions] = await Promise.all([
1505
- this.#engine.loopUsage(currentLoopId),
1506
- this.#engine.loopAttributions(currentLoopId),
1507
- ]);
1508
- this.#broadcast({ workspaceId }, "loop/terminated", {
1509
- workerId,
1510
- loopId: currentLoopId,
1511
- result: settled,
1512
- hitMaxTurns: false,
1513
- turnIds: await this.#lifecycle.turnIds(currentLoopId),
1514
- usage,
1515
- attributions,
1516
- });
1517
- }
1518
- if (!firstSettled) {
1519
- firstSettled = true;
1520
- rejectFirst(err);
1521
- }
1522
- }
1523
- throw err;
1524
- }
1525
- finally {
1526
- if (!firstSettled) {
1527
- firstSettled = true;
1528
- rejectFirst(new Error("drain exited without producing a result"));
1529
- }
1530
- if (this.#activeDrains.get(workerId) === handle)
1531
- this.#activeDrains.delete(workerId);
1532
- }
1533
- return { loopsDrained, lastResult };
1534
- })();
1535
- handle.promise = drainPromise;
1536
- this.#activeDrains.set(workerId, handle);
1537
- // Topology join ({§worker-loop-lifecycle}): when this drain exits having CONCLUDED the worker, wake its parent
1538
- // if parked. Runs after the drain fully tears down (settled promise) so the quiescence check sees
1539
- // final state; speculative (#onDrainExit no-ops unless the worker concluded AND the parent is parked).
1540
- const drainExitTask = drainPromise.then(() => this.#onDrainExit(workspaceId, workerId, systemPrompt), () => this.#onDrainExit(workspaceId, workerId, systemPrompt));
1541
- this.#drainExitTasks.add(drainExitTask);
1542
- void drainExitTask.catch((err) => {
1543
- console.error(`parent wake after worker ${workerId} settlement failed:`, err);
1544
- }).finally(() => {
1545
- this.#drainExitTasks.delete(drainExitTask);
1546
- });
1547
- // Swallow unhandled rejections (drain aborts with no awaiter); the
1548
- // error already surfaced via firstLoopPromise or was logged inside.
1549
- drainPromise.catch(() => { });
1550
- firstLoopPromise.catch(() => { });
1551
- return { firstLoopPromise, drainPromise };
1552
- }
1553
- // Per-worker drain-transition lock (R4 / {§worker-lifecycle-single-drain}). #ensureDrain's
1554
- // start and a drain's teardown relinquish both run under it, serialized, so the two
1555
- // can't interleave and register two drains for one worker. The critical section is the
1556
- // registry decision only (never a loop's work) — a sub-ms hop at drain boundaries.
1557
- // A promise-chain mutex: each caller awaits the prior holder; the tail self-prunes
1558
- // when idle so the Map stays bounded to workers mid-transition.
1559
- #withDrainLock(workerId, fn) {
1560
- const prev = this.#drainLocks.get(workerId) ?? Promise.resolve();
1561
- const run = prev.then(fn, fn);
1562
- const tail = run.catch(() => { });
1563
- this.#drainLocks.set(workerId, tail);
1564
- void tail.then(() => { if (this.#drainLocks.get(workerId) === tail)
1565
- this.#drainLocks.delete(workerId); });
1566
- return run;
1567
- }
1568
- // The drain guarantee, serialized per worker via #withDrainLock so it can't race a
1569
- // sibling drain's teardown relinquish into a double-drain (R4). A live drain
1570
- // (registered, NOT aborting) will claim the just-enqueued loop in its own iteration
1571
- // or its lock-held exit re-claim → return null. A registered-but-ABORTING drain is
1572
- // in teardown and won't claim, so we don't defer to it — start fresh, or the loop
1573
- // strands on a cancel/resume race (I6 no-lost-loop). Otherwise start one.
1574
- #ensureDrain(opts) {
1575
- return this.#withDrainLock(opts.workerId, async () => {
1576
- const existing = this.#activeDrains.get(opts.workerId);
1577
- if (existing !== undefined && !existing.controller.signal.aborted)
1578
- return null;
1579
- return this.#startDrain(opts);
1580
- });
1581
- }
1582
- // After a loop terminates, promote every next-turn frame it never consumed
1583
- // into one source-keyed queued loop. The first frame occupies the loop seed;
1584
- // later frames retain separate prompt entries and publish in the same turn.
1585
- // Re-entry and boot recovery complete that same queued identity.
1165
+ // Durable prompt promotion remains daemon policy; DrainSupervisor invokes
1166
+ // it under the same worker lock as enqueue and drain teardown.
1586
1167
  async #reconcileOrphanedPrompts(workerId, endedLoopId) {
1587
- await this.#withDrainLock(workerId, async () => {
1588
- const endedSeq = (await this.#db.engine_loop_sequence.get({ loop_id: endedLoopId }))?.sequence ?? endedLoopId;
1589
- const prefix = promptLoopPrefix(endedSeq);
1590
- const frames = await this.#db.drain_orphaned_prompts_for_loop.all({ loop_id: endedLoopId, owner_id: workerId, pattern: `${prefix}%`, prefix_len: prefix.length });
1591
- const first = frames[0];
1592
- if (first === undefined)
1593
- return;
1594
- const seqRow = await this.#db.loop_run_next_sequence.get({ worker_id: workerId });
1595
- if (seqRow === undefined)
1596
- throw new Error("reconcileOrphanedPrompts: next-sequence query returned no row");
1597
- const recovery = await this.#db.drain_enqueue_orphan_recovery_loop.get({
1598
- worker_id: workerId,
1599
- sequence: seqRow.next,
1600
- prompt: first.body,
1601
- flags: first.flags,
1602
- provider_spec: first.provider_spec,
1603
- child_provider_spec: first.child_provider_spec,
1604
- max_turns: first.max_turns,
1605
- open_paths: first.open_paths ?? "[]",
1606
- orphan_source_loop_id: endedLoopId,
1607
- });
1608
- if (recovery === undefined)
1609
- throw new Error("reconcileOrphanedPrompts: enqueue returned no row");
1610
- if (recovery.status !== 100)
1611
- return;
1612
- const moved = await this.#db.drain_rehome_orphaned_prompt_frames.all({
1613
- owner_id: workerId,
1614
- source_loop_id: endedLoopId,
1615
- source_pattern: `${prefix}%`,
1616
- source_prefix_len: prefix.length,
1617
- target_prefix: promptLoopPrefix(recovery.sequence),
1618
- });
1619
- if (moved.length !== frames.length) {
1620
- throw new Error(`reconcileOrphanedPrompts: expected to re-home ${frames.length} frames, moved ${moved.length}`);
1621
- }
1168
+ const endedSeq = (await this.#db.engine_loop_sequence.get({ loop_id: endedLoopId }))?.sequence ?? endedLoopId;
1169
+ const prefix = promptLoopPrefix(endedSeq);
1170
+ const frames = await this.#db.drain_orphaned_prompts_for_loop.all({ loop_id: endedLoopId, owner_id: workerId, pattern: `${prefix}%`, prefix_len: prefix.length });
1171
+ const first = frames[0];
1172
+ if (first === undefined)
1173
+ return;
1174
+ const seqRow = await this.#db.loop_run_next_sequence.get({ worker_id: workerId });
1175
+ if (seqRow === undefined)
1176
+ throw new Error("reconcileOrphanedPrompts: next-sequence query returned no row");
1177
+ const recovery = await this.#db.drain_enqueue_orphan_recovery_loop.get({
1178
+ worker_id: workerId,
1179
+ sequence: seqRow.next,
1180
+ prompt: first.body,
1181
+ flags: first.flags,
1182
+ provider_spec: first.provider_spec,
1183
+ child_provider_spec: first.child_provider_spec,
1184
+ max_turns: first.max_turns,
1185
+ open_paths: first.open_paths ?? "[]",
1186
+ orphan_source_loop_id: endedLoopId,
1622
1187
  });
1623
- }
1624
- // The worker's cancellation scope — lazily created, and replaced once aborted
1625
- // so a later runLoop gets a live signal. The drain and the execs its loops
1626
- // spawn all run under it.
1627
- #workerSignal(workerId) {
1628
- const existing = this.#workerAborts.get(workerId);
1629
- if (existing !== undefined && !existing.signal.aborted)
1630
- return existing;
1631
- const fresh = new AbortController();
1632
- this.#workerAborts.set(workerId, fresh);
1633
- return fresh;
1634
- }
1635
- async #cancelTree(workerId, reason, includeRoot) {
1636
- const cancelled = await this.#lifecycle.cancelTree(workerId, reason, includeRoot);
1637
- for (const targetWorkerId of cancelled.workerIds) {
1638
- const pollTimer = this.#pollTimers.get(targetWorkerId);
1639
- if (pollTimer !== undefined) {
1640
- clearTimeout(pollTimer);
1641
- this.#pollTimers.delete(targetWorkerId);
1642
- }
1643
- const parkTimer = this.#parkTimers.get(targetWorkerId);
1644
- if (parkTimer !== undefined) {
1645
- clearTimeout(parkTimer);
1646
- this.#parkTimers.delete(targetWorkerId);
1647
- }
1648
- this.#pollBackoff.delete(targetWorkerId);
1649
- this.#owedWakes.delete(targetWorkerId);
1650
- const scope = this.#workerAborts.get(targetWorkerId);
1651
- if (scope !== undefined && !scope.signal.aborted)
1652
- scope.abort(reason);
1653
- }
1654
- await Promise.all(cancelled.workerIds.map(async (targetWorkerId) => this.#reapWorkerStreams(targetWorkerId)));
1655
- for (const { loopId, workerId: targetWorkerId, result } of cancelled.loops) {
1656
- const row = await this.#db.drain_get_worker_workspace.get({ worker_id: targetWorkerId });
1657
- if (row === undefined)
1658
- continue;
1659
- const [usage, attributions] = await Promise.all([
1660
- this.#engine.loopUsage(loopId),
1661
- this.#engine.loopAttributions(loopId),
1662
- ]);
1663
- this.#broadcast({ workspaceId: row.workspace_id }, "loop/terminated", {
1664
- workerId: targetWorkerId,
1665
- loopId,
1666
- result,
1667
- hitMaxTurns: false,
1668
- turnIds: await this.#lifecycle.turnIds(loopId),
1669
- usage,
1670
- attributions,
1671
- });
1188
+ if (recovery === undefined)
1189
+ throw new Error("reconcileOrphanedPrompts: enqueue returned no row");
1190
+ if (recovery.status !== 100)
1191
+ return;
1192
+ const moved = await this.#db.drain_rehome_orphaned_prompt_frames.all({
1193
+ owner_id: workerId,
1194
+ source_loop_id: endedLoopId,
1195
+ source_pattern: `${prefix}%`,
1196
+ source_prefix_len: prefix.length,
1197
+ target_prefix: promptLoopPrefix(recovery.sequence),
1198
+ });
1199
+ if (moved.length !== frames.length) {
1200
+ throw new Error(`reconcileOrphanedPrompts: expected to re-home ${frames.length} frames, moved ${moved.length}`);
1672
1201
  }
1673
1202
  }
1674
- async #cancelWorkerTree(workerId, reason) {
1675
- await this.#cancelTree(workerId, reason, true);
1676
- }
1677
- /**
1678
- * Cancel the worker's in-flight work (loop.cancel). One abort, one scope: the
1679
- * worker signal stops the running loop's turn generation AND tears down every
1680
- * stream linked to it — a background exec that outlived its loop, or even a
1681
- * spawn that registers after this abort (it self-aborts against the aborted
1682
- * signal). Returns cancelled iff there was process-local work; durable
1683
- * unresolved loops in the worker tree are terminalized independently.
1684
- */
1685
1203
  cancelDrain(workerId, reason = "user_cancelled") {
1686
- const hadDrain = this.#activeDrains.has(workerId);
1687
- const hadWork = hadDrain || this.#workerHasActiveStreams(workerId);
1688
- // A cancel is deliberate — kill any pending hibernation poll-wake so it can't resurrect the worker.
1689
- const pollTimer = this.#pollTimers.get(workerId);
1690
- if (pollTimer !== undefined) {
1691
- clearTimeout(pollTimer);
1692
- this.#pollTimers.delete(workerId);
1693
- }
1694
- // Stop the active drain's turn-generation (its loop closes 499). The worker
1695
- // signal is the optimization path — the fast, listener-driven reap.
1696
- // Durable structured cancellation: one recursive transition claims the
1697
- // worker and every unresolved descendant, then reaps each process-local scope.
1698
- void this.#cancelWorkerTree(workerId, reason).catch((err) => {
1699
- console.error(`cancelTree(${workerId}) failed:`, err);
1700
- });
1701
- return hadWork;
1204
+ return this.#drains.cancel(workerId, reason);
1702
1205
  }
1703
- // Does the worker have an in-flight stream (a background exec)? Used only for
1704
- // loop.cancel's cancelled=true/false answer; the teardown itself rides the
1705
- // worker signal. Duck-typed like #drainStreamingSchemes.
1206
+ // Process-local stream activity is supplied to the drain owner through the
1207
+ // scheme registry rather than exposing that registry across the boundary.
1706
1208
  #workerHasActiveStreams(workerId) {
1707
1209
  const exec = this.#schemes.get("exec");
1708
1210
  return exec?.hasActiveSpawns?.(workerId) ?? false;
1709
1211
  }
1710
- // The contract-routed reap ({§worker-lifecycle-total-reap}): durable rows enumerate
1711
- // every open subscription; the live registry invokes its exact callable owner.
1712
- // The worker signal is only the fast path. An exec mid-spawn or a background exec
1713
- // from a past loop is caught regardless of listener timing. Idempotent — a stream
1714
- // the signal already reaped shares the same registry cancellation.
1715
- async #reapWorkerStreams(workerId) {
1716
- const open = await ChannelWrite.findOpenSubscriptionsForWorker(this.#db, workerId);
1717
- await Promise.all(open.map(({ id }) => this.#engine.cancelSubscription(id)));
1718
- }
1719
- /**
1720
- * Wake-on-completion handler. Streaming schemes call this when a
1721
- * subscription closes. A parked loop resumes in place; an active loop
1722
- * observes the channel transition at its next turn boundary. No synthetic
1723
- * prompt or replacement loop is created.
1724
- *
1725
- * Skipped on result.status=499 (aborted): the model already knows about
1726
- * its own SEND signal 499, and a forcefully-cancelled loop's spawn-abort must
1727
- * not resurrect the worker.
1728
- */
1729
- async #handleWakeWorker(payload) {
1730
- const { entryOwnerId, ...wake } = payload;
1731
- const conclusion = { ...wake, workerId: entryOwnerId };
1732
- // {§search-gate} — settle the dedup registration: promote on a 200 conclusion, drop on
1733
- // failure (a dead search must never serve as a duplicate). No-op for non-search streams.
1734
- this.#engine.searchGate.settle(payload.target.replace(/^[a-z+.-]+:\/\//, "/").replace(/^\/+/, "/"), payload.result.status);
1735
- // Aborted streams don't wake — the abort was deliberate.
1736
- if (payload.result.status === 499) {
1737
- this.#broadcast({ workspaceId: payload.workspaceId }, "stream/concluded", {
1738
- ...conclusion, wakeAction: "skipped-aborted",
1739
- });
1740
- return;
1741
- }
1742
- // No resurrection ({§worker-lifecycle-no-resurrection}): a non-499 completion whose
1743
- // worker was cancelled (idle + its scope aborted) must not start a fresh drain —
1744
- // the cancel was deliberate. The deliverable is already in the channel/log and
1745
- // surfaces as a `collect` environment delta ({§env-delta}) if the worker is read or
1746
- // resumed; we just don't inject a turn. (An active worker folds the wake into its
1747
- // next turn via inject below; a resumed worker is active, never aborted, so it is
1748
- // unaffected.)
1749
- const scope = this.#workerAborts.get(payload.workerId);
1750
- if (scope?.signal.aborted === true && !this.#activeDrains.has(payload.workerId)) {
1751
- this.#broadcast({ workspaceId: payload.workspaceId }, "stream/concluded", {
1752
- ...conclusion, wakeAction: "skipped-cancelled",
1753
- });
1754
- return;
1755
- }
1756
- try {
1757
- const systemPrompt = await readFile(Paths.instructionsSystem, "utf8");
1758
- // A slept (202) loop means the worker parked via SEND signal 202 → resume it in place: re-queue
1759
- // it (202→100) so the drain re-claims and CONTINUES it (seq>1 → no re-foist). Checked
1760
- // FIRST: the slept status is the worker's true disposition regardless of a draining
1761
- // sibling mid-teardown (the #ensureDrain lock serializes the re-claim). No fresh loop,
1762
- // no summary-as-prompt — the resumed loop reads the concluded stream's own state from
1763
- // the manifest. {§worker-lifecycle-wake-liveness}.
1764
- const slept = await this.#db.drain_find_slept_loop.get({ worker_id: payload.workerId });
1765
- if (slept !== undefined) {
1766
- // {§worker-optimistic-settlement} — publish this conclusion now,
1767
- // then let the worker-local gate coalesce only the provider
1768
- // dispatch. Concurrent stream/child callbacks join that one gate.
1769
- void this.#settleCompletionWake(payload.workspaceId, payload.workerId, systemPrompt, false).catch((err) => {
1770
- console.error("completion wake settlement failed:", err instanceof Error ? err.message : String(err));
1771
- });
1772
- this.#broadcast({ workspaceId: payload.workspaceId }, "stream/concluded", {
1773
- ...conclusion, wakeAction: "resumed-loop", wakeLoopId: slept.id,
1774
- });
1775
- return;
1776
- }
1777
- // No slept loop. A live loop surfaces the concluded stream ambiently via the
1778
- // environment-observation injector ({§exec-stream}) on its next turn — there is no prompt
1779
- // to inject and NO task to overwrite. The obsolete "automated environment update"
1780
- // synthesis (which clobbered the model's actual goal) is retired; just tell the client.
1781
- if (this.#activeDrains.has(payload.workerId)) {
1782
- this.#broadcast({ workspaceId: payload.workspaceId }, "stream/concluded", {
1783
- ...conclusion, wakeAction: "no-op-active-loop",
1784
- });
1785
- return;
1786
- }
1787
- // No slept loop, no active drain — nothing to resume (e.g. a SEND-200-done worker whose
1788
- // streams were swept). Surface the conclusion without opening a loop.
1789
- this.#broadcast({ workspaceId: payload.workspaceId }, "stream/concluded", {
1790
- ...conclusion, wakeAction: "no-loop",
1791
- });
1792
- }
1793
- catch (err) {
1794
- console.error("wake-on-completion setup failed:", err instanceof Error ? err.message : String(err));
1795
- }
1796
- }
1797
- /**
1798
- * grammar 0.74.20 EXEC `<T,P>` — schedule a hibernation poll-wake. Called when a loop parks at
1799
- * a park; if the worker holds an open polled stream, arm a timer for its tightest cadence P that
1800
- * resumes the slept loop so the model inspects progress. While the loop is ACTIVE there is no
1801
- * poll work — ambient folded stream deltas already surface progress ({§exec-stream}); the wake
1802
- * matters only across hibernation. A wake-edge-less 202 (no polled stream) gets no timer. {§exec-poll}
1803
- */
1804
- async #schedulePollWake(workspaceId, workerId, systemPrompt) {
1805
- const existing = this.#pollTimers.get(workerId);
1806
- if (existing !== undefined) {
1807
- clearTimeout(existing);
1808
- this.#pollTimers.delete(workerId);
1809
- }
1810
- const row = await this.#db.drain_worker_min_poll.get({ worker_id: workerId });
1811
- if ((row?.open_count ?? 0) === 0) {
1812
- this.#pollBackoff.delete(workerId);
1813
- return;
1814
- }
1815
- const pollSec = row?.poll_seconds ?? null;
1816
- // {§exec-poll} — a positive explicit cadence wins, zero opts out,
1817
- // and an absent cadence uses the worker's exponential-backoff step.
1818
- let delayMs;
1819
- if (pollSec !== null && pollSec > 0) {
1820
- this.#pollBackoff.delete(workerId);
1821
- delayMs = pollSec * 1000;
1822
- }
1823
- else if (pollSec === 0) {
1824
- this.#pollBackoff.delete(workerId);
1825
- return; // explicit opt-out
1826
- }
1827
- else {
1828
- // An open stream without an explicit cadence uses the stream polling floor.
1829
- // Child joins never enter this branch: durable child settlement is their only wake edge.
1830
- const base = Number(process.env.PLURNK_SERVICE_EXEC_POLL_SEC ?? "60");
1831
- const turns = Number(process.env.PLURNK_SERVICE_EXEC_POLL_TURNS ?? "8");
1832
- const step = this.#pollBackoff.get(workerId) ?? 0;
1833
- delayMs = execPollBackoffMs(step, base, turns);
1834
- this.#pollBackoff.set(workerId, step + 1);
1835
- }
1836
- // Floored by the optimistic settlement cap so a `<…,1>` cannot wake a
1837
- // parked loop faster than the preceding turn's settlement scale.
1838
- const optimisticWaitMs = readOptimisticSettlementMs();
1839
- const timer = setTimeout(() => {
1840
- this.#pollTimers.delete(workerId);
1841
- void this.#wakeParkedWorker(workspaceId, workerId, systemPrompt);
1842
- }, Math.max(delayMs, optimisticWaitMs));
1843
- timer.unref();
1844
- this.#pollTimers.set(workerId, timer);
1845
- }
1846
- /** Resume `workerId`'s slept (202) loop in place — the same 202→100 resume #handleWakeWorker uses, minus a
1847
- * wake payload. The shared wake primitive: a poll cadence ({§exec-poll}), a watched stream concluding,
1848
- * or a child worker finishing ({§worker-loop-lifecycle} topology join) all call this. A no-op if the worker was
1849
- * cancelled or isn't actually parked (no slept loop) — so calling it speculatively is safe. */
1850
- async #wakeParkedWorker(workspaceId, workerId, systemPrompt, oweIfActive = true) {
1851
- if (!this.#started)
1852
- return;
1853
- const scope = this.#workerAborts.get(workerId);
1854
- if (scope?.signal.aborted === true && !this.#activeDrains.has(workerId))
1855
- return; // cancelled — no resurrection
1856
- const slept = await this.#db.drain_find_slept_loop.get({ worker_id: workerId });
1857
- if (slept === undefined) {
1858
- // Not parked. If a drain is still ACTIVE, the worker is mid-turn and about to park — the
1859
- // conclusion that fired this wake arrived before the 202 committed (the conclude-before-park
1860
- // race). Owe the wake: the drain honors it at park so a worker hibernation never deadlocks.
1861
- // (No active drain → already concluded/running; nothing to wake.)
1862
- if (this.#activeDrains.has(workerId)) {
1863
- if (oweIfActive)
1864
- this.#owedWakes.add(workerId);
1865
- }
1866
- return;
1867
- }
1868
- const woke = await this.#lifecycle.wake(slept.id);
1869
- if (!woke)
1870
- return;
1871
- const started = await this.#ensureDrain({
1872
- workspaceId, workerId, systemPrompt,
1873
- });
1874
- started?.drainPromise?.catch((err) => {
1875
- if (this.#started) {
1876
- console.error("wake-parked resume drain failed:", err instanceof Error ? err.message : String(err));
1877
- }
1878
- });
1879
- }
1880
- async #workerHasLiveObligation(workerId) {
1881
- const [openSubscriptions, liveChild] = await Promise.all([
1882
- this.#db.find_open_subscriptions_for_worker.all({ worker_id: workerId }),
1883
- this.#db.engine_worker_has_live_child.get({ worker_id: workerId }),
1884
- ]);
1885
- return openSubscriptions.length > 0 || liveChild !== undefined;
1886
- }
1887
- #settleCompletionWake(workspaceId, workerId, systemPrompt, oweIfActive = true) {
1888
- const existing = this.#completionWakeGates.get(workerId);
1889
- if (existing !== undefined) {
1890
- existing.conclusions++;
1891
- existing.poke.resolve();
1892
- return existing.promise;
1893
- }
1894
- const completed = Promise.withResolvers();
1895
- const gate = {
1896
- conclusions: 1,
1897
- poke: Promise.withResolvers(),
1898
- promise: completed.promise,
1899
- };
1900
- this.#completionWakeGates.set(workerId, gate);
1901
- void this.#runCompletionWake(workspaceId, workerId, systemPrompt, oweIfActive, gate).then(completed.resolve, completed.reject).finally(() => {
1902
- if (this.#completionWakeGates.get(workerId) === gate) {
1903
- this.#completionWakeGates.delete(workerId);
1904
- }
1905
- });
1906
- return gate.promise;
1907
- }
1908
- async #runCompletionWake(workspaceId, workerId, systemPrompt, oweIfActive, gate) {
1909
- const slept = await this.#db.drain_find_slept_loop.get({ worker_id: workerId });
1910
- if (slept === undefined) {
1911
- this.#releaseCompletionWake(workerId, gate);
1912
- return this.#wakeParkedWorker(workspaceId, workerId, systemPrompt, oweIfActive);
1913
- }
1914
- const timeoutMs = readOptimisticSettlementMs();
1915
- if (timeoutMs === 0 || !(await this.#workerHasLiveObligation(workerId))) {
1916
- this.#releaseCompletionWake(workerId, gate);
1917
- return this.#wakeParkedWorker(workspaceId, workerId, systemPrompt, false);
1918
- }
1919
- return observed("worker.wake.settlement", { "window.ms": timeoutMs }, async (span) => {
1920
- const startedAt = performance.now();
1921
- const signal = this.#workerAborts.get(workerId)?.signal;
1922
- const deadline = delay(timeoutMs, "deadline", { signal, ref: false })
1923
- .catch((cause) => {
1924
- if (signal?.aborted === true)
1925
- return "cancelled";
1926
- throw cause;
1927
- });
1928
- let release = "quiescent";
1929
- while (await this.#workerHasLiveObligation(workerId)) {
1930
- const poke = gate.poke;
1931
- const outcome = await Promise.race([
1932
- poke.promise.then(() => "arrival"),
1933
- deadline,
1934
- ]);
1935
- if (outcome === "arrival") {
1936
- if (gate.poke === poke)
1937
- gate.poke = Promise.withResolvers();
1938
- continue;
1939
- }
1940
- release = outcome;
1941
- break;
1942
- }
1943
- span.setAttribute("release", release);
1944
- span.setAttribute("conclusions", gate.conclusions);
1945
- span.setAttribute("elapsed.ms", Math.round(performance.now() - startedAt));
1946
- this.#releaseCompletionWake(workerId, gate);
1947
- if (release === "cancelled")
1948
- return;
1949
- return this.#wakeParkedWorker(workspaceId, workerId, systemPrompt, false);
1950
- });
1951
- }
1952
- #releaseCompletionWake(workerId, gate) {
1953
- if (this.#completionWakeGates.get(workerId) === gate) {
1954
- this.#completionWakeGates.delete(workerId);
1955
- }
1956
- }
1957
- /** A worker's drain exited. If the worker truly CONCLUDED — no 202-blocked loop, no open stream — then
1958
- * wake its PARENT in place if the parent is blocked on the join (the structured-concurrency join — a
1959
- * child finishing is the wake edge for a parent that waited on it, {§worker-lifecycle-child-wake}). A worker
1960
- * blocked at 202, or still holding a stream, is NOT concluded — its own wake edges drive it, not this.
1961
- * The parent reads the child's deliverable from its own log (the {§worker-scheme-collect} delta) on
1962
- * resume — control edge here, never an injected prompt. Recurses up via the parent's own drain-exit. */
1963
- async #onDrainExit(workspaceId, workerId, systemPrompt) {
1964
- const slept = await this.#db.drain_find_slept_loop.get({ worker_id: workerId });
1965
- if (slept !== undefined)
1966
- return; // parked at 202 — not concluded, the worker is still alive
1967
- const openSubs = await this.#db.find_open_subscriptions_for_worker.all({ worker_id: workerId });
1968
- if (openSubs.length > 0)
1969
- return; // a stream still runs — its conclusion re-evaluates, not this exit
1970
- const parent = await this.#db.worker_parent_id.get({ worker_id: workerId });
1971
- if (parent?.parent_worker_id == null)
1972
- return; // a root worker — nobody to wake
1973
- await this.#settleCompletionWake(workspaceId, parent.parent_worker_id, systemPrompt);
1974
- }
1975
1212
  // {§methods-event-subscribe} — subscriber failures are transport-local:
1976
1213
  // log them at this boundary and never re-enter engine control flow.
1977
1214
  #emitTo(workspaceId, method, params) {