@north-light/crouter 0.3.281 → 0.3.283
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/client.d.ts +19 -3
- package/dist/api/client.js +34 -0
- package/dist/api/dto/broker-ops.d.ts +11 -0
- package/dist/api/dto/broker.d.ts +3 -8
- package/dist/api/dto/common.d.ts +1 -1
- package/dist/api/dto/messages.d.ts +2 -2
- package/dist/api/dto/node-outcomes.d.ts +80 -0
- package/dist/api/dto/node-outcomes.js +2 -0
- package/dist/api/dto/nodes.d.ts +9 -0
- package/dist/api/dto/profiles.d.ts +5 -0
- package/dist/api/dto/reports.d.ts +45 -0
- package/dist/api/index.d.ts +1 -0
- package/dist/api/index.js +1 -0
- package/dist/api/plugin-manifest-schema.d.ts +208 -0
- package/dist/api/plugin-manifest-schema.js +23 -0
- package/dist/api/routes.d.ts +7 -0
- package/dist/api/routes.js +7 -0
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js +24 -3
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +31 -4
- package/dist/cli.js +8 -2
- package/dist/clients/attach/overlays/graph.d.ts +5 -0
- package/dist/clients/attach/overlays/graph.js +65 -47
- package/dist/clients/attach/overlays/pickers.js +8 -9
- package/dist/clients/attach/render/card-presentation.js +1 -1
- package/dist/clients/attach/viewer.js +1286 -1260
- package/dist/commands/api-client.js +22 -23
- package/dist/commands/node/create.js +38 -3
- package/dist/commands/node/inspect.js +3 -7
- package/dist/commands/node/message.js +3 -3
- package/dist/commands/node/outcome.d.ts +2 -0
- package/dist/commands/node/outcome.js +134 -0
- package/dist/commands/node.js +2 -1
- package/dist/commands/pkg/browse/catalog.js +1 -1
- package/dist/commands/pkg/plugin-manage.d.ts +46 -1
- package/dist/commands/pkg/plugin-manage.js +170 -20
- package/dist/commands/profile/pause.js +17 -14
- package/dist/commands/push.js +102 -1
- package/dist/commands/sys/__tests__/setup-core.test.js +4 -1
- package/dist/commands/sys/panels/broker-limits-panel.js +7 -1
- package/dist/commands/sys/panels/models-panel.js +2 -2
- package/dist/commands/sys/settings-shell.d.ts +1 -1
- package/dist/commands/sys/settings-shell.js +6 -0
- package/dist/commands/sys/settings.js +3 -3
- package/dist/core/__tests__/fixtures/fake-engine.d.ts +2 -0
- package/dist/core/__tests__/fixtures/fake-engine.js +42 -4
- package/dist/core/__tests__/helpers/harness.d.ts +1 -0
- package/dist/core/__tests__/helpers/harness.js +9 -0
- package/dist/core/__tests__/integration/plugin-revalidate.test.d.ts +1 -0
- package/dist/core/__tests__/integration/plugin-revalidate.test.js +355 -0
- package/dist/core/__tests__/integration/profile-pause.test.d.ts +1 -0
- package/dist/core/__tests__/integration/profile-pause.test.js +115 -0
- package/dist/core/__tests__/node-outcome.test.d.ts +1 -0
- package/dist/core/__tests__/node-outcome.test.js +184 -0
- package/dist/core/__tests__/revive-parked-fresh.test.js +13 -20
- package/dist/core/__tests__/seam/broker-provider-retry.test.js +6 -2
- package/dist/core/__tests__/seam/dormancy-release.test.js +3 -3
- package/dist/core/__tests__/seam/node-outcome-delivery.test.d.ts +1 -0
- package/dist/core/__tests__/seam/node-outcome-delivery.test.js +153 -0
- package/dist/core/bash-jobs.d.ts +6 -0
- package/dist/core/bash-jobs.js +22 -0
- package/dist/core/canvas/canvas.d.ts +41 -3
- package/dist/core/canvas/canvas.js +153 -39
- package/dist/core/canvas/crons.js +2 -1
- package/dist/core/canvas/index.d.ts +1 -0
- package/dist/core/canvas/index.js +1 -0
- package/dist/core/canvas/migrations.js +47 -0
- package/dist/core/canvas/node-outcome-deliveries.d.ts +80 -0
- package/dist/core/canvas/node-outcome-deliveries.js +163 -0
- package/dist/core/canvas/pid.d.ts +7 -0
- package/dist/core/canvas/pid.js +17 -0
- package/dist/core/canvas/render-source.js +2 -2
- package/dist/core/canvas/types.d.ts +42 -1
- package/dist/core/canvas/types.js +2 -1
- package/dist/core/command-manifests/registry.d.ts +3 -1
- package/dist/core/command-manifests/schema.d.ts +2 -75
- package/dist/core/command-manifests/schema.js +5 -0
- package/dist/core/command-plugins/compose.js +1 -1
- package/dist/core/command-plugins/revalidate.d.ts +18 -0
- package/dist/core/command-plugins/revalidate.js +152 -0
- package/dist/core/command-plugins/transport/http-fetch.d.ts +29 -4
- package/dist/core/command-plugins/transport/http-fetch.js +18 -8
- package/dist/core/command-plugins/transport/http-invoke.js +7 -0
- package/dist/core/command.d.ts +8 -0
- package/dist/core/command.js +19 -3
- package/dist/core/config.d.ts +4 -1
- package/dist/core/config.js +25 -1
- package/dist/core/exclusive-lock.d.ts +12 -0
- package/dist/core/exclusive-lock.js +29 -0
- package/dist/core/feed/feed.d.ts +8 -0
- package/dist/core/feed/feed.js +25 -4
- package/dist/core/feed/inbox.d.ts +5 -0
- package/dist/core/feed/inbox.js +13 -0
- package/dist/core/installed-plugins.js +31 -0
- package/dist/core/keybindings/catalog.d.ts +1 -1
- package/dist/core/keybindings/catalog.js +3 -0
- package/dist/core/manifest-recovery.d.ts +15 -0
- package/dist/core/manifest-recovery.js +72 -0
- package/dist/core/manifest-stale.d.ts +18 -0
- package/dist/core/manifest-stale.js +39 -0
- package/dist/core/plugin-swap-lock.d.ts +9 -0
- package/dist/core/plugin-swap-lock.js +31 -0
- package/dist/core/profiles/manifest.d.ts +3 -1
- package/dist/core/profiles/manifest.js +6 -1
- package/dist/core/runtime/broker/daemon-ops.d.ts +3 -1
- package/dist/core/runtime/broker/daemon-ops.js +6 -0
- package/dist/core/runtime/broker/engine-drive.d.ts +2 -3
- package/dist/core/runtime/broker/engine-drive.js +74 -27
- package/dist/core/runtime/broker/frame-dispatch.js +1 -0
- package/dist/core/runtime/broker/read-ops.js +14 -0
- package/dist/core/runtime/broker/rebind.js +0 -7
- package/dist/core/runtime/broker-protocol.d.ts +1 -6
- package/dist/core/runtime/fleet.d.ts +7 -0
- package/dist/core/runtime/headless-pi.d.ts +13 -0
- package/dist/core/runtime/headless-pi.js +10 -2
- package/dist/core/runtime/lifecycle.d.ts +6 -1
- package/dist/core/runtime/lifecycle.js +50 -1
- package/dist/core/runtime/nodes.d.ts +5 -1
- package/dist/core/runtime/nodes.js +2 -1
- package/dist/core/runtime/outcome-document.d.ts +70 -0
- package/dist/core/runtime/outcome-document.js +97 -0
- package/dist/core/runtime/revive.js +16 -21
- package/dist/core/runtime/spawn.d.ts +5 -1
- package/dist/core/runtime/spawn.js +18 -7
- package/dist/core/runtime/stamp/channel.d.ts +3 -5
- package/dist/core/runtime/stamp/channel.js +3 -10
- package/dist/core/runtime/stop-guard.js +2 -2
- package/dist/core/runtime/structured-output.d.ts +24 -3
- package/dist/core/runtime/structured-output.js +42 -3
- package/dist/core/runtime/tmux-bindings.js +4 -0
- package/dist/core/runtime/warm-pool.d.ts +5 -2
- package/dist/core/runtime/warm-pool.js +5 -5
- package/dist/core/subscription-state.js +7 -5
- package/dist/core/substrate/render.js +10 -10
- package/dist/core/user-settings.d.ts +18 -1
- package/dist/core/user-settings.js +37 -0
- package/dist/daemon/__tests__/node-deadline-lane.test.d.ts +1 -0
- package/dist/daemon/__tests__/node-deadline-lane.test.js +76 -0
- package/dist/daemon/__tests__/node-outcome-birth-invariants.test.d.ts +1 -0
- package/dist/daemon/__tests__/node-outcome-birth-invariants.test.js +122 -0
- package/dist/daemon/__tests__/node-outcome-delivery.test.d.ts +1 -0
- package/dist/daemon/__tests__/node-outcome-delivery.test.js +86 -0
- package/dist/daemon/api/__tests__/broker-settle-park.test.js +86 -18
- package/dist/daemon/api/__tests__/profile-launch-gates.test.js +10 -1
- package/dist/daemon/api/__tests__/seam/node-outcome-api.test.d.ts +1 -0
- package/dist/daemon/api/__tests__/seam/node-outcome-api.test.js +159 -0
- package/dist/daemon/api/__tests__/submit-result.test.d.ts +1 -0
- package/dist/daemon/api/__tests__/submit-result.test.js +214 -0
- package/dist/daemon/api/handlers/broker-ops.js +53 -24
- package/dist/daemon/api/handlers/canvas.js +2 -2
- package/dist/daemon/api/handlers/messages.js +12 -2
- package/dist/daemon/api/handlers/node-outcomes.d.ts +2 -0
- package/dist/daemon/api/handlers/node-outcomes.js +136 -0
- package/dist/daemon/api/handlers/nodes.js +67 -1
- package/dist/daemon/api/handlers/profiles.js +53 -1
- package/dist/daemon/api/handlers/reports.js +197 -33
- package/dist/daemon/api/map.d.ts +7 -2
- package/dist/daemon/api/map.js +78 -4
- package/dist/daemon/api/server.js +2 -0
- package/dist/daemon/crtrd.js +12 -0
- package/dist/daemon/fleet.d.ts +4 -8
- package/dist/daemon/fleet.js +48 -9
- package/dist/daemon/messaging/node-message.js +2 -1
- package/dist/daemon/node-outcome/deliver-outcome.d.ts +14 -0
- package/dist/daemon/node-outcome/deliver-outcome.js +155 -0
- package/dist/daemon/park-activity.d.ts +9 -0
- package/dist/daemon/park-activity.js +30 -0
- package/dist/daemon/park-pending.d.ts +10 -5
- package/dist/daemon/park-pending.js +18 -8
- package/dist/daemon/reconcilers/bash-deadline.js +6 -0
- package/dist/daemon/reconcilers/broker-supervision.d.ts +3 -7
- package/dist/daemon/reconcilers/broker-supervision.js +29 -33
- package/dist/daemon/reconcilers/node-deadline.d.ts +5 -0
- package/dist/daemon/reconcilers/node-deadline.js +82 -0
- package/dist/daemon/reconcilers/node-lifecycle/freeze-lane.js +9 -1
- package/dist/daemon/reconcilers/node-lifecycle/tick.js +12 -3
- package/dist/daemon/reconcilers/node-outcome-delivery-lane.d.ts +10 -0
- package/dist/daemon/reconcilers/node-outcome-delivery-lane.js +41 -0
- package/dist/pi-extensions/__tests__/canvas-structured-output.test.js +66 -66
- package/dist/pi-extensions/canvas-structured-output.d.ts +9 -5
- package/dist/pi-extensions/canvas-structured-output.js +23 -297
- package/dist/shared/env.d.ts +3 -0
- package/dist/shared/env.js +6 -0
- package/dist/shared/generated-context.d.ts +3 -9
- package/dist/shared/generated-context.js +8 -16
- package/dist/types.d.ts +19 -0
- package/dist/types.js +1 -0
- package/package.json +9 -8
- package/runtime.lock.json +227 -383
|
@@ -26,17 +26,15 @@ import { formatCard, PARK_SUMMARY_PROMPT } from '../../shared/generated-context.
|
|
|
26
26
|
import { brokerThresholdsForDaemon } from '../../core/runtime/fleet.js';
|
|
27
27
|
import { readConfig } from '../../core/config.js';
|
|
28
28
|
import { beginPendingPark, consumePendingPark, pendingParkIds, pendingParkStartedAt } from '../park-pending.js';
|
|
29
|
+
import { shouldParkAfterSummary } from '../park-activity.js';
|
|
29
30
|
import { applyParkEvent, parkEvent } from './live-obligation.js';
|
|
30
|
-
import { envNoDaemonAutostart,
|
|
31
|
+
import { envNoDaemonAutostart, envTestUnattendedParkMs, parkSummaryGraceMs, envUnattendedParkMs, } from '../../shared/env.js';
|
|
31
32
|
const execFileAsync = promisify(execFile);
|
|
32
33
|
export const YIELD_STALL_GRACE_MS = 3 * 60_000;
|
|
33
34
|
const KILL_ESCALATE_MS = 20_000;
|
|
34
35
|
export const WEDGE_QUIET_MS = 20 * 60_000;
|
|
35
36
|
export const WEDGE_CPU_MAX_PERCENT = 2;
|
|
36
37
|
const OVER_CAP_WARN_THROTTLE_MS = 5 * 60_000;
|
|
37
|
-
// Longer than a summary turn and shorter than the unattended clock itself, so a
|
|
38
|
-
// stuck parking turn cannot hold a broker for a whole extra clock period.
|
|
39
|
-
const DEFAULT_PARK_SUMMARY_GRACE_MS = 5 * 60_000;
|
|
40
38
|
export function unattendedParkMsForDaemon() {
|
|
41
39
|
// The `CRTR_UNATTENDED_PARK_MS` override wins whenever set, autostart or not.
|
|
42
40
|
const envOverride = envUnattendedParkMs();
|
|
@@ -50,7 +48,7 @@ export function unattendedParkMsForDaemon() {
|
|
|
50
48
|
return envTestUnattendedParkMs() ?? configured;
|
|
51
49
|
}
|
|
52
50
|
export function parkSummaryGraceMsForDaemon() {
|
|
53
|
-
return
|
|
51
|
+
return parkSummaryGraceMs();
|
|
54
52
|
}
|
|
55
53
|
export function yieldStallVerdict(piPidAlive, intent, busy, stalledFor) {
|
|
56
54
|
if (piPidAlive !== true)
|
|
@@ -199,9 +197,7 @@ export class BrokerSupervisionReconciler {
|
|
|
199
197
|
}
|
|
200
198
|
/** Every condition that resets the unattended clock EXCEPT `isBusy` — the
|
|
201
199
|
* node is unavailable to park, attended, or has work waiting. Short-circuits
|
|
202
|
-
* in cost order so the common tick does no file reads it can avoid.
|
|
203
|
-
* Evaluated for a pending park too, where a busy engine is precisely what is
|
|
204
|
-
* expected (it is writing the summary) and every other hit aborts. */
|
|
200
|
+
* in cost order so the common tick does no file reads it can avoid. */
|
|
205
201
|
parkResetCondition(id, row, livePid, now) {
|
|
206
202
|
if (row.pi_session_id == null
|
|
207
203
|
|| row.pi_pid !== livePid
|
|
@@ -239,26 +235,21 @@ export class BrokerSupervisionReconciler {
|
|
|
239
235
|
return;
|
|
240
236
|
}
|
|
241
237
|
const reset = this.parkResetCondition(id, row, livePid, now);
|
|
242
|
-
// A parking turn
|
|
243
|
-
//
|
|
238
|
+
// A shadow parking turn owns this marker until it completes or runs out of
|
|
239
|
+
// grace. Both paths take the same main-engine activity verdict before
|
|
240
|
+
// terminalizing, so input during the summary only keeps the node live.
|
|
244
241
|
const parkingSince = pendingParkStartedAt(id);
|
|
245
242
|
if (parkingSince !== undefined) {
|
|
246
|
-
if (reset) {
|
|
247
|
-
// Something worth attending to arrived (or the node became unavailable)
|
|
248
|
-
// during the summary turn. The node stays live and the park consequence
|
|
249
|
-
// is discarded; its clock restarts from scratch.
|
|
250
|
-
consumePendingPark(id);
|
|
251
|
-
this.unattendedSince.delete(id);
|
|
252
|
-
return;
|
|
253
|
-
}
|
|
254
243
|
if (now - parkingSince < parkSummaryGraceMsForDaemon())
|
|
255
244
|
return;
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
// BEFORE the host dies (the crash-safety invariant).
|
|
259
|
-
if (!consumePendingPark(id))
|
|
245
|
+
const pending = consumePendingPark(id);
|
|
246
|
+
if (pending === undefined)
|
|
260
247
|
return;
|
|
261
248
|
this.unattendedSince.delete(id);
|
|
249
|
+
if (!shouldParkAfterSummary(id, row, pending))
|
|
250
|
+
return;
|
|
251
|
+
// The degraded path has no report. Status flips BEFORE the host dies (the
|
|
252
|
+
// crash-safety invariant).
|
|
262
253
|
emitEvent({
|
|
263
254
|
level: 'warn',
|
|
264
255
|
event: 'broker.park.summary_expired',
|
|
@@ -299,26 +290,21 @@ export class BrokerSupervisionReconciler {
|
|
|
299
290
|
headlessBrokerHost.teardown(id);
|
|
300
291
|
return;
|
|
301
292
|
}
|
|
302
|
-
// A resident
|
|
303
|
-
//
|
|
304
|
-
// broker settlement, which re-evaluates this verdict once that turn ends.
|
|
293
|
+
// A resident gets one isolated turn to leave a report behind. Its completion
|
|
294
|
+
// re-evaluates this verdict before terminalizing the node.
|
|
305
295
|
if (!ctx.lifecycle.acceptsDetachedWork())
|
|
306
296
|
return;
|
|
307
297
|
beginPendingPark(id, now);
|
|
308
298
|
ctx.lifecycle.registerDetached(this.requestParkSummary(id));
|
|
309
299
|
}
|
|
310
300
|
/** Deliver the parking prompt over the live broker's serialized frame loop.
|
|
311
|
-
*
|
|
312
|
-
*
|
|
313
|
-
* `steer` route means a turn started between the idle gate and the frame, so
|
|
314
|
-
* the node is working again and its clock simply restarts. */
|
|
301
|
+
* An unreachable broker falls through to degraded parking; either prompt route
|
|
302
|
+
* starts the same isolated turn, so a concurrent main turn stays passive. */
|
|
315
303
|
async requestParkSummary(id) {
|
|
316
304
|
try {
|
|
317
305
|
const route = await deliverLive(id, formatCard('park', {}, PARK_SUMMARY_PROMPT), {
|
|
318
306
|
origin: { type: 'runtime', kind: 'park' },
|
|
319
307
|
});
|
|
320
|
-
if (route === 'steer' && consumePendingPark(id))
|
|
321
|
-
return;
|
|
322
308
|
emitEvent({
|
|
323
309
|
level: 'info',
|
|
324
310
|
event: 'broker.park.summary_requested',
|
|
@@ -327,7 +313,7 @@ export class BrokerSupervisionReconciler {
|
|
|
327
313
|
});
|
|
328
314
|
}
|
|
329
315
|
catch (err) {
|
|
330
|
-
if (
|
|
316
|
+
if (consumePendingPark(id) === undefined)
|
|
331
317
|
return;
|
|
332
318
|
if (err instanceof BrokerUnreachableError) {
|
|
333
319
|
applyParkEvent(id, 'park');
|
|
@@ -599,7 +585,17 @@ export class BrokerSupervisionReconciler {
|
|
|
599
585
|
`Error: ${firstLine}`;
|
|
600
586
|
if (terminalOverflow) {
|
|
601
587
|
try {
|
|
602
|
-
transition(id, 'crash'
|
|
588
|
+
transition(id, 'crash', {
|
|
589
|
+
reason: 'context_overflow',
|
|
590
|
+
outcome: {
|
|
591
|
+
detail: {
|
|
592
|
+
schema: 'crtr.node-outcome-detail/v1',
|
|
593
|
+
message: fault.message,
|
|
594
|
+
fault_kind: 'context-overflow',
|
|
595
|
+
error_class: 'context_overflow',
|
|
596
|
+
},
|
|
597
|
+
},
|
|
598
|
+
});
|
|
603
599
|
headlessBrokerHost.teardown(id);
|
|
604
600
|
}
|
|
605
601
|
catch {
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { dueNodeDeadlines, subscribersOf, } from '../../core/canvas/index.js';
|
|
2
|
+
import { emitEvent } from '../../core/events/emit.js';
|
|
3
|
+
import { isProfilePaused } from '../../core/profiles/manifest.js';
|
|
4
|
+
import { operationIdContext } from '../../core/events/operation-id.js';
|
|
5
|
+
import { errorClassFromFault } from '../../core/fault-classifier.js';
|
|
6
|
+
import { fanDoctrineWake } from '../../core/runtime/close.js';
|
|
7
|
+
import { readFault } from '../../core/runtime/fault.js';
|
|
8
|
+
import { requestBrokerTeardown } from '../../core/runtime/host.js';
|
|
9
|
+
import { transition } from '../../core/runtime/lifecycle.js';
|
|
10
|
+
/** Enforce absolute spawn deadlines independently of the tick's live-row scan. */
|
|
11
|
+
export class NodeDeadlineReconciler {
|
|
12
|
+
run(now) {
|
|
13
|
+
let rows;
|
|
14
|
+
try {
|
|
15
|
+
rows = dueNodeDeadlines(new Date(now).toISOString());
|
|
16
|
+
}
|
|
17
|
+
catch (error) {
|
|
18
|
+
emitEvent({ level: 'error', event: 'node_deadline.scan_failed', error });
|
|
19
|
+
return;
|
|
20
|
+
}
|
|
21
|
+
for (const row of rows) {
|
|
22
|
+
if (isProfilePaused(row.profile_id))
|
|
23
|
+
continue;
|
|
24
|
+
operationIdContext.fresh(() => this.expire(row, now));
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
expire(row, now) {
|
|
28
|
+
const deadlineAt = row.deadline_at;
|
|
29
|
+
if (deadlineAt === null || deadlineAt === undefined)
|
|
30
|
+
return;
|
|
31
|
+
const fault = readFault(row.node_id);
|
|
32
|
+
const lastFatalFault = fault?.retry.disposition === 'fatal' ? fault : null;
|
|
33
|
+
let terminal;
|
|
34
|
+
try {
|
|
35
|
+
terminal = transition(row.node_id, 'cancel', {
|
|
36
|
+
reason: 'deadline_exceeded',
|
|
37
|
+
outcome: {
|
|
38
|
+
detail: {
|
|
39
|
+
schema: 'crtr.node-outcome-detail/v1',
|
|
40
|
+
deadline: { deadline_at: deadlineAt, elapsed_ms: now - Date.parse(deadlineAt) },
|
|
41
|
+
...(lastFatalFault === null ? {} : {
|
|
42
|
+
message: lastFatalFault.message,
|
|
43
|
+
fault_kind: lastFatalFault.kind,
|
|
44
|
+
error_class: errorClassFromFault(lastFatalFault).class,
|
|
45
|
+
}),
|
|
46
|
+
},
|
|
47
|
+
},
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
catch (error) {
|
|
51
|
+
emitEvent({ level: 'error', event: 'node_deadline.terminalize_failed', node_id: row.node_id, error, fields: { deadline_at: deadlineAt } });
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
// cancel's requireNoFinalReport gate lets a same-instant canonical final win;
|
|
55
|
+
// only the terminal write that latched this deadline outcome may reap the host.
|
|
56
|
+
if (terminal.status !== 'canceled' || terminal.outcome_reason !== 'deadline_exceeded')
|
|
57
|
+
return;
|
|
58
|
+
try {
|
|
59
|
+
requestBrokerTeardown(row.node_id);
|
|
60
|
+
}
|
|
61
|
+
catch (error) {
|
|
62
|
+
emitEvent({ level: 'error', event: 'node_deadline.teardown_failed', node_id: row.node_id, error, fields: { deadline_at: deadlineAt } });
|
|
63
|
+
}
|
|
64
|
+
try {
|
|
65
|
+
fanDoctrineWake(row.node_id, subscribersOf(row.node_id), 'child-deadline-exceeded', { reason: 'child-deadline-exceeded', child: row.node_id });
|
|
66
|
+
}
|
|
67
|
+
catch (error) {
|
|
68
|
+
emitEvent({ level: 'error', event: 'node_deadline.notify_failed', node_id: row.node_id, error, fields: { deadline_at: deadlineAt } });
|
|
69
|
+
}
|
|
70
|
+
try {
|
|
71
|
+
emitEvent({
|
|
72
|
+
level: 'error',
|
|
73
|
+
event: 'node_deadline.expired',
|
|
74
|
+
node_id: row.node_id,
|
|
75
|
+
fields: { deadline_at: deadlineAt, elapsed_ms: now - Date.parse(deadlineAt) },
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
catch {
|
|
79
|
+
// Event emission must not make a committed deadline outcome retry teardown.
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
}
|
|
@@ -134,7 +134,15 @@ export class FreezeLaneReconciler {
|
|
|
134
134
|
max_attempts: IDLE_RELEASE_RELAUNCH_MAX_ATTEMPTS,
|
|
135
135
|
},
|
|
136
136
|
});
|
|
137
|
-
transition(id, 'crash'
|
|
137
|
+
transition(id, 'crash', {
|
|
138
|
+
reason: 'launch_failed',
|
|
139
|
+
outcome: {
|
|
140
|
+
detail: {
|
|
141
|
+
schema: 'crtr.node-outcome-detail/v1',
|
|
142
|
+
respawn_failures: attempts,
|
|
143
|
+
},
|
|
144
|
+
},
|
|
145
|
+
});
|
|
138
146
|
fanDoctrineWake(id, subscribersOf(id), `Child crashed — ${fullName(row)} (${id}) failed to relaunch after ${attempts} attempts while trying to ` +
|
|
139
147
|
`deliver a pending inbox message (no delivery progress across any attempt), and is now dead. It stays ` +
|
|
140
148
|
`dead until you revive it — \`crtr node lifecycle revive ${id}\`.`, { reason: 'child-crashed', child: id });
|
|
@@ -69,9 +69,14 @@ export class NodeLifecycleReconciler {
|
|
|
69
69
|
candidates.push({ row, enact: 'thaw' });
|
|
70
70
|
return;
|
|
71
71
|
}
|
|
72
|
-
// 1. A fleet handle IS execution
|
|
72
|
+
// 1. A fleet handle IS execution, except profile pause must stop a broker
|
|
73
|
+
// that registered after pause's fast scan. The fleet records this exact
|
|
74
|
+
// interruption before SIGTERM so resume cannot classify it as a crash.
|
|
73
75
|
if (ctx.fleet.has(id)) {
|
|
74
|
-
|
|
76
|
+
if (isProfilePaused(row.profile_id))
|
|
77
|
+
ctx.fleet.interruptForProfilePause(id);
|
|
78
|
+
else
|
|
79
|
+
this.thawMark(row);
|
|
75
80
|
return;
|
|
76
81
|
}
|
|
77
82
|
// A paused profile launches nothing, and its stuck-delivery episode is over
|
|
@@ -117,7 +122,11 @@ export class NodeLifecycleReconciler {
|
|
|
117
122
|
// mode is delay, never a wrong kill.
|
|
118
123
|
if (recordedPidLiveness(row.pi_pid, row.pi_pid_identity) !== 'dead')
|
|
119
124
|
return;
|
|
120
|
-
|
|
125
|
+
// A pause interruption needs two passes after resume: the first consumes
|
|
126
|
+
// its marker without failure or launch, and retaining the PID lets the
|
|
127
|
+
// second recognize the same dead row for ordinary recovery.
|
|
128
|
+
if (!ctx.fleet.hasProfilePauseInterruption(id))
|
|
129
|
+
clearRecordedPid(id);
|
|
121
130
|
// 7. The invariant-5 policy table, on the same inputs an exit event gives
|
|
122
131
|
// it. 8. A verdict that needs no broker is enacted right here, because it
|
|
123
132
|
// costs no slot; a respawn becomes a candidate for the freeze lane.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { DetachedWorkLifecycle } from './broker-supervision.js';
|
|
2
|
+
/** One daemon-life identity for guarded outcome-delivery claims. */
|
|
3
|
+
export declare const outcomeDeliveryDaemonInstanceId: `${string}-${string}-${string}-${string}-${string}`;
|
|
4
|
+
export interface NodeOutcomeDeliveryLaneContext {
|
|
5
|
+
lifecycle: DetachedWorkLifecycle;
|
|
6
|
+
}
|
|
7
|
+
/** Admit due outcome deliveries without making the daemon's serial tick await subprocesses. */
|
|
8
|
+
export declare class NodeOutcomeDeliveryLaneReconciler {
|
|
9
|
+
run(now: number, ctx: NodeOutcomeDeliveryLaneContext): void;
|
|
10
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { claimOutcomeDelivery, dueOutcomeDeliveries } from '../../core/canvas/node-outcome-deliveries.js';
|
|
3
|
+
import { emitEvent } from '../../core/events/emit.js';
|
|
4
|
+
import { operationIdContext } from '../../core/events/operation-id.js';
|
|
5
|
+
import { deliverOutcome } from '../node-outcome/deliver-outcome.js';
|
|
6
|
+
/** One daemon-life identity for guarded outcome-delivery claims. */
|
|
7
|
+
export const outcomeDeliveryDaemonInstanceId = randomUUID();
|
|
8
|
+
/** Admit due outcome deliveries without making the daemon's serial tick await subprocesses. */
|
|
9
|
+
export class NodeOutcomeDeliveryLaneReconciler {
|
|
10
|
+
run(now, ctx) {
|
|
11
|
+
if (!ctx.lifecycle.acceptsDetachedWork())
|
|
12
|
+
return;
|
|
13
|
+
let due;
|
|
14
|
+
try {
|
|
15
|
+
due = dueOutcomeDeliveries(now);
|
|
16
|
+
}
|
|
17
|
+
catch (error) {
|
|
18
|
+
operationIdContext.fresh(() => {
|
|
19
|
+
emitEvent({ level: 'error', event: 'node_outcome.delivery.scan_failed', error });
|
|
20
|
+
});
|
|
21
|
+
return;
|
|
22
|
+
}
|
|
23
|
+
for (const candidate of due) {
|
|
24
|
+
try {
|
|
25
|
+
const claimed = claimOutcomeDelivery(candidate.nodeId, now, outcomeDeliveryDaemonInstanceId);
|
|
26
|
+
if (claimed === null)
|
|
27
|
+
continue;
|
|
28
|
+
ctx.lifecycle.registerDetached(deliverOutcome(claimed, outcomeDeliveryDaemonInstanceId).catch((error) => {
|
|
29
|
+
operationIdContext.fresh(() => {
|
|
30
|
+
emitEvent({ level: 'error', event: 'node_outcome.delivery.failed', error, fields: { node_id: claimed.nodeId } });
|
|
31
|
+
});
|
|
32
|
+
}));
|
|
33
|
+
}
|
|
34
|
+
catch (error) {
|
|
35
|
+
operationIdContext.fresh(() => {
|
|
36
|
+
emitEvent({ level: 'error', event: 'node_outcome.delivery.claim_failed', error, fields: { node_id: candidate.nodeId } });
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
@@ -1,22 +1,33 @@
|
|
|
1
1
|
// Run with: node --import tsx/esm --test src/pi-extensions/__tests__/canvas-structured-output.test.ts
|
|
2
2
|
//
|
|
3
|
-
//
|
|
4
|
-
|
|
3
|
+
// The extension injects a pending structured-output request at the END of the
|
|
4
|
+
// system prompt each agent run; submission itself is `crtr push result`
|
|
5
|
+
// (server-side). These cases pin the injection contract: terminal vs one-off
|
|
6
|
+
// wording, no injection for absent/invalid requests, and per-run re-reads.
|
|
7
|
+
import { after, before, beforeEach, test } from 'node:test';
|
|
5
8
|
import assert from 'node:assert/strict';
|
|
6
|
-
import { mkdtempSync, rmSync } from 'node:fs';
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import { convertResponsesTools } from '@earendil-works/pi-ai/api/openai-responses-shared';
|
|
10
|
-
import { writeOutputSchema } from '../../core/runtime/structured-output.js';
|
|
9
|
+
import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from 'node:fs';
|
|
10
|
+
import { dirname } from 'node:path';
|
|
11
|
+
import { outputSchemaPath, removeOutputSchema, writeOutputSchema } from '../../core/runtime/structured-output.js';
|
|
11
12
|
import registerCanvasStructuredOutput from '../canvas-structured-output.js';
|
|
13
|
+
const NODE = 'structured-output-test';
|
|
14
|
+
const PROMPT = 'BASE SYSTEM PROMPT';
|
|
12
15
|
const previous = {
|
|
13
16
|
home: process.env['CRTR_HOME'],
|
|
14
17
|
nodeId: process.env['CRTR_NODE_ID'],
|
|
15
18
|
};
|
|
16
19
|
let home;
|
|
20
|
+
before(() => {
|
|
21
|
+
// macOS limits Unix-socket paths to 104 bytes; keep the home short.
|
|
22
|
+
home = mkdtempSync('/tmp/crtr-so-');
|
|
23
|
+
process.env['CRTR_HOME'] = home;
|
|
24
|
+
process.env['CRTR_NODE_ID'] = NODE;
|
|
25
|
+
});
|
|
26
|
+
beforeEach(() => {
|
|
27
|
+
rmSync(home, { recursive: true, force: true });
|
|
28
|
+
});
|
|
17
29
|
after(() => {
|
|
18
|
-
|
|
19
|
-
rmSync(home, { recursive: true, force: true });
|
|
30
|
+
rmSync(home, { recursive: true, force: true });
|
|
20
31
|
if (previous.home === undefined)
|
|
21
32
|
delete process.env['CRTR_HOME'];
|
|
22
33
|
else
|
|
@@ -26,63 +37,52 @@ after(() => {
|
|
|
26
37
|
else
|
|
27
38
|
process.env['CRTR_NODE_ID'] = previous.nodeId;
|
|
28
39
|
});
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
const schema = {
|
|
34
|
-
type: 'object',
|
|
35
|
-
properties: {
|
|
36
|
-
status: { type: 'string' },
|
|
37
|
-
commit: { type: 'string' },
|
|
38
|
-
verification: { type: 'string' },
|
|
39
|
-
blocker: { type: 'string' },
|
|
40
|
-
details: {
|
|
41
|
-
type: 'object',
|
|
42
|
-
properties: {
|
|
43
|
-
summary: { type: 'string' },
|
|
44
|
-
},
|
|
45
|
-
required: ['summary'],
|
|
46
|
-
},
|
|
47
|
-
tags: {
|
|
48
|
-
type: 'array',
|
|
49
|
-
items: { type: 'string' },
|
|
50
|
-
minItems: 2,
|
|
51
|
-
maxItems: 5,
|
|
52
|
-
},
|
|
53
|
-
},
|
|
54
|
-
required: ['status', 'verification', 'details', 'tags'],
|
|
55
|
-
};
|
|
56
|
-
writeOutputSchema('structured-output-test', 'oneoff', schema);
|
|
57
|
-
let definition;
|
|
40
|
+
/** Register the extension against a stub pi and return a run(): the prompt the
|
|
41
|
+
* model would see for one agent run. */
|
|
42
|
+
function harness() {
|
|
43
|
+
let handler;
|
|
58
44
|
registerCanvasStructuredOutput({
|
|
59
|
-
|
|
60
|
-
definition = tool;
|
|
61
|
-
},
|
|
62
|
-
on() { },
|
|
63
|
-
});
|
|
64
|
-
assert.ok(definition, 'the pending request registers submit');
|
|
65
|
-
const parameters = definition['parameters'];
|
|
66
|
-
assert.deepEqual(parameters.required, ['status', 'commit', 'verification', 'blocker', 'details', 'tags']);
|
|
67
|
-
assert.deepEqual(parameters.properties.commit, { anyOf: [{ type: 'string' }, { type: 'null' }] });
|
|
68
|
-
assert.deepEqual(parameters.properties.blocker, { anyOf: [{ type: 'string' }, { type: 'null' }] });
|
|
69
|
-
assert.equal(parameters.additionalProperties, false);
|
|
70
|
-
assert.equal(parameters.properties.details.additionalProperties, false);
|
|
71
|
-
assert.equal(parameters.properties.tags['minItems'], undefined);
|
|
72
|
-
assert.equal(parameters.properties.tags['maxItems'], undefined);
|
|
73
|
-
assert.match(parameters.properties.tags['description'], /minItems: 2/);
|
|
74
|
-
assert.match(parameters.properties.tags['description'], /maxItems: 5/);
|
|
75
|
-
assert.deepEqual(schema.required, ['status', 'verification', 'details', 'tags'], 'the stored request keeps its original constraints');
|
|
76
|
-
const [providerTool] = convertResponsesTools([definition], {
|
|
77
|
-
strict: false,
|
|
78
|
-
supportsStrictMode: true,
|
|
45
|
+
on: (_event, h) => { handler = h; },
|
|
79
46
|
});
|
|
80
|
-
assert.
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
47
|
+
assert.ok(handler, 'extension registered a before_agent_start handler');
|
|
48
|
+
return (prompt = PROMPT) => handler({ systemPrompt: prompt })?.systemPrompt ?? prompt;
|
|
49
|
+
}
|
|
50
|
+
test('terminal request appends schema, push result, and a final-push prohibition at the end', () => {
|
|
51
|
+
const run = harness();
|
|
52
|
+
writeOutputSchema(NODE, 'terminal', { type: 'object', properties: { verdict: { type: 'string' } } });
|
|
53
|
+
const prompt = run();
|
|
54
|
+
assert.ok(prompt.startsWith(PROMPT), 'base prompt preserved');
|
|
55
|
+
const block = prompt.slice(PROMPT.length);
|
|
56
|
+
assert.match(block, /crtr push result/);
|
|
57
|
+
assert.match(block, /"verdict"/);
|
|
58
|
+
assert.match(block, /do NOT call `crtr push final`/i);
|
|
59
|
+
assert.ok(prompt.trimEnd().endsWith('</structured-output-request>'), 'block sits at the end of the prompt');
|
|
60
|
+
});
|
|
61
|
+
test('one-off request appends continue-after-success wording', () => {
|
|
62
|
+
const run = harness();
|
|
63
|
+
writeOutputSchema(NODE, 'oneoff', { type: 'object' });
|
|
64
|
+
const prompt = run();
|
|
65
|
+
const block = prompt.slice(PROMPT.length);
|
|
66
|
+
assert.match(block, /crtr push result/);
|
|
67
|
+
assert.match(block, /continue working/);
|
|
68
|
+
assert.doesNotMatch(block, /push final/);
|
|
69
|
+
});
|
|
70
|
+
test('absent request leaves the prompt untouched', () => {
|
|
71
|
+
const run = harness();
|
|
72
|
+
assert.equal(run(), PROMPT);
|
|
73
|
+
});
|
|
74
|
+
test('invalid request file leaves the prompt untouched (stop guard owns the error)', () => {
|
|
75
|
+
const run = harness();
|
|
76
|
+
const path = outputSchemaPath(NODE);
|
|
77
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
78
|
+
writeFileSync(path, 'not json', 'utf8');
|
|
79
|
+
assert.equal(run(), PROMPT);
|
|
80
|
+
});
|
|
81
|
+
test('request state is re-read every run, never cached', () => {
|
|
82
|
+
const run = harness();
|
|
83
|
+
assert.equal(run(), PROMPT);
|
|
84
|
+
writeOutputSchema(NODE, 'oneoff', { type: 'object' });
|
|
85
|
+
assert.match(run(), /crtr push result/);
|
|
86
|
+
removeOutputSchema(NODE);
|
|
87
|
+
assert.equal(run(), PROMPT);
|
|
88
88
|
});
|
|
@@ -1,9 +1,13 @@
|
|
|
1
|
-
type
|
|
1
|
+
import { type StructuredOutputRequest } from '../core/runtime/structured-output.js';
|
|
2
|
+
interface BeforeAgentStartEventLike {
|
|
3
|
+
systemPrompt: string;
|
|
4
|
+
}
|
|
2
5
|
interface PiLike {
|
|
3
|
-
on: (event:
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
setActiveTools?: (tools: string[]) => void;
|
|
6
|
+
on: (event: 'before_agent_start', handler: (event: BeforeAgentStartEventLike) => {
|
|
7
|
+
systemPrompt: string;
|
|
8
|
+
} | void) => void;
|
|
7
9
|
}
|
|
10
|
+
/** Render the prompt block for one valid request. Exported for the test suite. */
|
|
11
|
+
export declare function structuredOutputPromptBlock(request: StructuredOutputRequest): string;
|
|
8
12
|
export declare function registerCanvasStructuredOutput(pi: PiLike): void;
|
|
9
13
|
export default registerCanvasStructuredOutput;
|