@north-light/crouter 0.3.233 → 0.3.235
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/client.js +16 -15
- package/dist/api/dto/nodes.d.ts +11 -0
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/__tests__/integration/provider-rotation.test.ts +1 -1
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/claude-plugin-commands.ts +14 -17
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/crtr-commands/index.ts +7 -11
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/frontmatter-rules/index.ts +8 -8
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/memory-slash-commands.ts +10 -11
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/pi-shell-runner.ts +2 -2
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.d.ts +3 -3
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js +7 -7
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +8 -8
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/statusline.ts +6 -6
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/sysprompt-window.ts +3 -3
- package/dist/builtin-pi-packages/pi-crtr-extensions/lib/subscription-state.ts +1 -1
- package/dist/clients/attach/chrome/ticket-panel.js +15 -2
- package/dist/clients/attach/overlays/graph.js +0 -3
- package/dist/clients/attach/overlays/mcp.js +1 -2
- package/dist/clients/attach/render/chat-view.js +0 -12
- package/dist/clients/attach/render/crtr-output.js +0 -2
- package/dist/clients/attach/render/tool-calls.js +0 -6
- package/dist/clients/attach/slash/dispatch.js +8 -11
- package/dist/clients/attach/viewer.js +535 -537
- package/dist/clients/conversation/projection.js +0 -2
- package/dist/clients/inbox/controller.js +22 -3
- package/dist/clients/inbox/review/companion-pane.js +0 -4
- package/dist/clients/inbox/review-adapter.js +0 -7
- package/dist/clients/inbox/tui/slots.d.ts +3 -3
- package/dist/clients/inbox/tui/types.d.ts +2 -2
- package/dist/commands/api-client.js +0 -8
- package/dist/commands/attention.js +1 -9
- package/dist/commands/canvas-browse.js +2 -1
- package/dist/commands/canvas-config.js +0 -10
- package/dist/commands/canvas-history.js +2 -1
- package/dist/commands/cron.js +10 -23
- package/dist/commands/dashboard.js +0 -2
- package/dist/commands/human/feedback.js +2 -1
- package/dist/commands/human/prompts.js +2 -5
- package/dist/commands/human/queue.js +2 -7
- package/dist/commands/human/request.js +0 -14
- package/dist/commands/human/review.js +2 -1
- package/dist/commands/human.js +1 -1
- package/dist/commands/memory/__tests__/command-selector-and-mutation-guards.test.js +4 -4
- package/dist/commands/memory/__tests__/memory-list-and-lint-ignore.test.d.ts +1 -0
- package/dist/commands/memory/__tests__/memory-list-and-lint-ignore.test.js +96 -0
- package/dist/commands/memory/delete.js +1 -1
- package/dist/commands/memory/edit.js +6 -3
- package/dist/commands/memory/find.js +2 -2
- package/dist/commands/memory/lint.js +2 -1
- package/dist/commands/memory/list.d.ts +4 -0
- package/dist/commands/memory/list.js +20 -14
- package/dist/commands/memory/move.js +1 -1
- package/dist/commands/memory/read.d.ts +2 -7
- package/dist/commands/memory/read.js +5 -16
- package/dist/commands/memory/shared.d.ts +1 -0
- package/dist/commands/memory/shared.js +17 -8
- package/dist/commands/memory/write.js +8 -4
- package/dist/commands/node/bash.js +2 -1
- package/dist/commands/node/create.js +2 -1
- package/dist/commands/node/inspect.js +11 -10
- package/dist/commands/node/lifecycle.js +5 -13
- package/dist/commands/node/message.js +11 -6
- package/dist/commands/node/subscription.js +7 -2
- package/dist/commands/node/wait.js +9 -4
- package/dist/commands/node-context.js +2 -1
- package/dist/commands/node-lifecycle-revive.js +0 -2
- package/dist/commands/node-worktree.js +2 -1
- package/dist/commands/pkg/browse/actions.d.ts +1 -1
- package/dist/commands/pkg/browse/actions.js +4 -4
- package/dist/commands/pkg/browse/catalog.d.ts +1 -1
- package/dist/commands/pkg/browse/catalog.js +15 -15
- package/dist/commands/pkg/browse/command-view.d.ts +1 -1
- package/dist/commands/pkg/browse/command-view.js +3 -3
- package/dist/commands/pkg/browse/detail-panel.d.ts +1 -1
- package/dist/commands/pkg/browse/detail-panel.js +8 -8
- package/dist/commands/pkg/browse/doc-view.d.ts +1 -1
- package/dist/commands/pkg/browse/doc-view.js +3 -3
- package/dist/commands/pkg/browse/list-panel.d.ts +1 -1
- package/dist/commands/pkg/browse/list-panel.js +3 -3
- package/dist/commands/pkg/browse/model.d.ts +3 -3
- package/dist/commands/pkg/browse/shell.d.ts +2 -2
- package/dist/commands/pkg/browse/shell.js +13 -13
- package/dist/commands/pkg/plugin-manage.js +3 -2
- package/dist/commands/pkg/shared.js +0 -2
- package/dist/commands/pkg.js +1 -0
- package/dist/commands/profile/delete.js +2 -1
- package/dist/commands/profile/rename.js +1 -1
- package/dist/commands/push.js +2 -3
- package/dist/commands/revive.js +0 -2
- package/dist/commands/surface/node/placement.js +2 -1
- package/dist/commands/surface-edit.js +2 -1
- package/dist/commands/sys/config.js +1 -4
- package/dist/commands/sys/daemon.js +0 -10
- package/dist/commands/sys/doctor.js +3 -2
- package/dist/commands/sys/panels/broker-limits-panel.d.ts +2 -2
- package/dist/commands/sys/panels/broker-limits-panel.js +4 -4
- package/dist/commands/sys/panels/dependencies-panel.d.ts +2 -2
- package/dist/commands/sys/panels/dependencies-panel.js +2 -2
- package/dist/commands/sys/panels/keybindings-panel.d.ts +1 -1
- package/dist/commands/sys/panels/keybindings-panel.js +5 -5
- package/dist/commands/sys/panels/models-panel.d.ts +4 -4
- package/dist/commands/sys/panels/models-panel.js +11 -10
- package/dist/commands/sys/panels/plugins-panel.d.ts +2 -2
- package/dist/commands/sys/panels/plugins-panel.js +2 -2
- package/dist/commands/sys/panels/prefs-panel.d.ts +2 -2
- package/dist/commands/sys/panels/prefs-panel.js +4 -4
- package/dist/commands/sys/panels/profiles-panel.d.ts +1 -1
- package/dist/commands/sys/panels/profiles-panel.js +14 -13
- package/dist/commands/sys/panels/provider-panel.d.ts +3 -3
- package/dist/commands/sys/panels/provider-panel.js +8 -8
- package/dist/commands/sys/settings-shell.d.ts +3 -3
- package/dist/commands/sys/settings-shell.js +15 -15
- package/dist/commands/sys/setup-settings-runner.d.ts +3 -3
- package/dist/commands/sys/setup-settings-runner.js +5 -5
- package/dist/commands/sys/setup-wizard.d.ts +4 -4
- package/dist/commands/sys/setup-wizard.js +8 -8
- package/dist/commands/sys/support/prepare.js +4 -4
- package/dist/commands/sys/support/submit.js +2 -2
- package/dist/commands/sys/sync-deps.js +0 -2
- package/dist/commands/sys/sync-project-guidance.js +2 -1
- package/dist/commands/sys/sysprompt.js +2 -1
- package/dist/commands/sys.js +1 -0
- package/dist/core/__tests__/boot.test.js +0 -8
- package/dist/core/__tests__/broker-extension-canvas-db-boundary.test.js +23 -1
- package/dist/core/__tests__/canvas-db-boundary.test.d.ts +1 -0
- package/dist/core/__tests__/canvas-db-boundary.test.js +46 -0
- package/dist/core/__tests__/canvas.test.js +1 -1
- package/dist/core/__tests__/child-death-wake.test.js +0 -2
- package/dist/core/__tests__/cli-help-contract.test.d.ts +1 -0
- package/dist/core/__tests__/cli-help-contract.test.js +150 -0
- package/dist/core/__tests__/cron-armed-context-failure.test.d.ts +1 -0
- package/dist/core/__tests__/cron-armed-context-failure.test.js +145 -0
- package/dist/core/__tests__/daemon-wedge.test.js +0 -4
- package/dist/core/__tests__/dead-node-policy-table.test.js +0 -4
- package/dist/core/__tests__/fixtures/fake-engine.js +4 -14
- package/dist/core/__tests__/fixtures/fake-pi-host.js +5 -5
- package/dist/core/__tests__/focuses.test.js +0 -6
- package/dist/core/__tests__/fs-utils.test.d.ts +1 -0
- package/dist/core/__tests__/fs-utils.test.js +50 -0
- package/dist/core/__tests__/helpers/harness.js +1 -1
- package/dist/core/__tests__/inline-memory-refs.test.js +0 -2
- package/dist/core/__tests__/integration/broker-fork-seam.test.js +0 -2
- package/dist/core/__tests__/integration/broker-sdk-wiring.test.js +3 -16
- package/dist/core/__tests__/integration/command-plugins.test.js +21 -24
- package/dist/core/__tests__/integration/deferred-no-wake.test.js +0 -6
- package/dist/core/__tests__/integration/flagship-lifecycle.test.js +0 -26
- package/dist/core/__tests__/integration/live-mutation-verbs.test.js +0 -4
- package/dist/core/__tests__/integration/live-mutation.test.js +1 -5
- package/dist/core/__tests__/integration/refresh-stall-recycle.test.js +1 -5
- package/dist/core/__tests__/integration/revive.test.js +0 -12
- package/dist/core/__tests__/integration/subscription-delivery.test.js +0 -6
- package/dist/core/__tests__/integration/tmux-surface.test.js +24 -8
- package/dist/core/__tests__/integration/worktree-reap.test.js +0 -6
- package/dist/core/__tests__/lifecycle.test.js +0 -18
- package/dist/core/__tests__/memory-resolver-precedence.test.js +1 -1
- package/dist/core/__tests__/migration.test.js +2 -2
- package/dist/core/__tests__/on-read-dedup-resume.test.js +3 -3
- package/dist/core/__tests__/pid-identity-match.test.js +1 -1
- package/dist/core/__tests__/plugin-kinds.test.js +1 -1
- package/dist/core/__tests__/plugin-page-components.test.js +1 -1
- package/dist/core/__tests__/preview-registry-sync.test.js +42 -1
- package/dist/core/__tests__/respawn-throttle.test.js +0 -4
- package/dist/core/__tests__/revive-all.test.js +5 -5
- package/dist/core/__tests__/revive-parked-fresh.test.js +62 -11
- package/dist/core/__tests__/seam/broker-attach-multiclient.test.js +0 -8
- package/dist/core/__tests__/seam/broker-attach-stream.test.js +0 -8
- package/dist/core/__tests__/seam/broker-crash-teardown.test.js +0 -2
- package/dist/core/__tests__/tmux-driver-boundary.test.d.ts +1 -0
- package/dist/core/__tests__/tmux-driver-boundary.test.js +50 -0
- package/dist/core/__tests__/watchdog-abort-arms-retry.test.js +11 -6
- package/dist/core/activity-signal.d.ts +7 -0
- package/dist/core/activity-signal.js +38 -0
- package/dist/core/auto-update.js +2 -1
- package/dist/core/bootstrap.js +2 -1
- package/dist/core/canvas/__tests__/remote-canvas-source.test.js +1 -1
- package/dist/core/canvas/__tests__/remote-transport.test.js +1 -1
- package/dist/core/canvas/__tests__/render-remote.test.js +1 -1
- package/dist/core/canvas/browse/app.js +3 -2
- package/dist/core/canvas/browse/model.d.ts +1 -1
- package/dist/core/canvas/browse/model.js +0 -4
- package/dist/core/canvas/canvas.d.ts +22 -0
- package/dist/core/canvas/canvas.js +61 -19
- package/dist/core/canvas/crons.js +0 -6
- package/dist/core/canvas/db.d.ts +14 -20
- package/dist/core/canvas/db.js +30 -1332
- package/dist/core/canvas/focuses.js +0 -4
- package/dist/core/canvas/history.js +2 -1
- package/dist/core/canvas/meta-file.js +18 -27
- package/dist/core/canvas/migrations.d.ts +20 -0
- package/dist/core/canvas/migrations.js +1339 -0
- package/dist/core/canvas/nav-model.js +1 -11
- package/dist/core/canvas/nav-render.js +0 -8
- package/dist/core/canvas/remote-canvas-source.d.ts +8 -0
- package/dist/core/canvas/remote-canvas-source.js +9 -3
- package/dist/core/canvas/render-source.js +31 -13
- package/dist/core/command-manifests/manifest.js +0 -10
- package/dist/core/command-manifests/registry.d.ts +2 -0
- package/dist/core/command-manifests/schema.js +0 -4
- package/dist/core/command-plugins/compose.js +4 -2
- package/dist/core/command-plugins/discovery.d.ts +1 -0
- package/dist/core/command-plugins/discovery.js +1 -1
- package/dist/core/command-plugins/transport/http-invoke.js +0 -14
- package/dist/core/command.js +2 -6
- package/dist/core/config.js +2 -1
- package/dist/core/fault-classifier.js +8 -0
- package/dist/core/feed/inbox.d.ts +10 -0
- package/dist/core/feed/inbox.js +18 -12
- package/dist/core/fs-utils.d.ts +9 -2
- package/dist/core/fs-utils.js +7 -15
- package/dist/core/help.js +0 -8
- package/dist/core/host-exports/export.d.ts +2 -2
- package/dist/core/host-exports/export.js +9 -12
- package/dist/core/human/answer.d.ts +2 -2
- package/dist/core/human/convention.d.ts +2 -5
- package/dist/core/human/convention.js +3 -13
- package/dist/core/human/page-schema.d.ts +2 -2
- package/dist/core/human/page-schema.js +2 -2
- package/dist/core/human/root.js +6 -23
- package/dist/core/human/scan.js +24 -7
- package/dist/core/human/tickets.d.ts +4 -1
- package/dist/core/human/tickets.js +20 -6
- package/dist/core/io.js +0 -12
- package/dist/core/memory/identity.js +0 -4
- package/dist/core/memory/inline-ref-inventory.js +3 -2
- package/dist/core/memory/lint.js +3 -6
- package/dist/core/memory-resolver.d.ts +5 -0
- package/dist/core/memory-resolver.js +14 -12
- package/dist/core/pagination.d.ts +2 -0
- package/dist/core/pagination.js +2 -3
- package/dist/core/preview-registry.d.ts +14 -0
- package/dist/core/preview-registry.js +2 -4
- package/dist/core/preview-result-path.js +2 -1
- package/dist/core/profiles/manifest.d.ts +37 -5
- package/dist/core/profiles/manifest.js +44 -24
- package/dist/core/profiles/select.js +0 -4
- package/dist/core/review/signal.js +4 -23
- package/dist/core/review/ticket-filter.d.ts +3 -3
- package/dist/core/review/ticket-filter.js +4 -9
- package/dist/core/runtime/__tests__/engine-route.test.js +1 -1
- package/dist/core/runtime/__tests__/spawn-env.test.js +0 -8
- package/dist/core/runtime/bearings-render.d.ts +56 -0
- package/dist/core/runtime/bearings-render.js +241 -0
- package/dist/core/runtime/bearings.d.ts +2 -17
- package/dist/core/runtime/bearings.js +12 -198
- package/dist/core/runtime/broker/auth-reload.d.ts +39 -0
- package/dist/core/runtime/broker/auth-reload.js +149 -0
- package/dist/core/runtime/broker/engine-drive.d.ts +26 -0
- package/dist/core/runtime/broker/engine-drive.js +257 -0
- package/dist/core/runtime/broker/engine-routing.d.ts +143 -0
- package/dist/core/runtime/broker/engine-routing.js +211 -0
- package/dist/core/runtime/broker/event-projection.js +3 -2
- package/dist/core/runtime/broker/fault-retry.js +11 -4
- package/dist/core/runtime/broker/frame-client.d.ts +38 -0
- package/dist/core/runtime/broker/frame-client.js +129 -0
- package/dist/core/runtime/broker/frame-dispatch.d.ts +12 -189
- package/dist/core/runtime/broker/frame-dispatch.js +89 -1476
- package/dist/core/runtime/broker/frame-engine-control.d.ts +20 -0
- package/dist/core/runtime/broker/frame-engine-control.js +231 -0
- package/dist/core/runtime/broker/frame-memory-refs.d.ts +19 -0
- package/dist/core/runtime/broker/frame-memory-refs.js +41 -0
- package/dist/core/runtime/broker/frame-model-query.d.ts +13 -0
- package/dist/core/runtime/broker/frame-model-query.js +69 -0
- package/dist/core/runtime/broker/frame-read.d.ts +19 -0
- package/dist/core/runtime/broker/frame-read.js +116 -0
- package/dist/core/runtime/broker/frame-replies.d.ts +24 -0
- package/dist/core/runtime/broker/frame-replies.js +64 -0
- package/dist/core/runtime/broker/frame-session.d.ts +34 -0
- package/dist/core/runtime/broker/frame-session.js +203 -0
- package/dist/core/runtime/broker/rebind.js +50 -0
- package/dist/core/runtime/broker/turn-ignition.d.ts +42 -0
- package/dist/core/runtime/broker/turn-ignition.js +74 -0
- package/dist/core/runtime/broker-extension-render.js +9 -85
- package/dist/core/runtime/broker-protocol.js +0 -4
- package/dist/core/runtime/broker-sdk.js +2 -1
- package/dist/core/runtime/broker.d.ts +1 -1
- package/dist/core/runtime/broker.js +5 -23
- package/dist/core/runtime/fault.js +3 -2
- package/dist/core/runtime/headless-pi.js +3 -2
- package/dist/core/runtime/host.js +2 -3
- package/dist/core/runtime/kickoff.js +0 -4
- package/dist/core/runtime/launch-target.js +2 -1
- package/dist/core/runtime/launch.d.ts +2 -2
- package/dist/core/runtime/launch.js +20 -10
- package/dist/core/runtime/lifecycle.js +17 -26
- package/dist/core/runtime/managed-provider-cooling.d.ts +1 -1
- package/dist/core/runtime/naming.js +0 -2
- package/dist/core/runtime/nodes.js +2 -1
- package/dist/core/runtime/persona.js +0 -6
- package/dist/core/runtime/placement-tmux.js +2 -5
- package/dist/core/runtime/placement.js +4 -11
- package/dist/core/runtime/recap.js +4 -3
- package/dist/core/runtime/reopen.js +4 -7
- package/dist/core/runtime/reset.js +0 -4
- package/dist/core/runtime/resume-root.js +0 -2
- package/dist/core/runtime/revive.js +17 -13
- package/dist/core/runtime/session-list-cache.js +2 -2
- package/dist/core/runtime/shell-expansion.js +0 -2
- package/dist/core/runtime/spawn.js +0 -2
- package/dist/core/runtime/stop-guard.js +28 -16
- package/dist/core/runtime/stream-watchdog.d.ts +0 -1
- package/dist/core/runtime/stream-watchdog.js +13 -18
- package/dist/core/runtime/tmux-bindings.js +0 -2
- package/dist/core/runtime/tmux-driver.d.ts +12 -0
- package/dist/core/runtime/tmux-driver.js +18 -13
- package/dist/core/runtime/warm-pool.js +0 -4
- package/dist/core/scope.js +4 -3
- package/dist/core/spawn.js +2 -3
- package/dist/core/subscription-state.js +10 -17
- package/dist/core/substrate/__tests__/surface-match-pre-command.test.js +3 -3
- package/dist/core/substrate/frontmatter-validation.d.ts +1 -0
- package/dist/core/substrate/frontmatter-validation.js +1 -1
- package/dist/core/substrate/render.js +0 -12
- package/dist/core/substrate/schema.js +0 -12
- package/dist/core/substrate/surface-match.js +0 -2
- package/dist/core/termrender/display.js +2 -1
- package/dist/core/tui/page-host.d.ts +3 -3
- package/dist/core/tui/page-host.js +4 -4
- package/dist/core/tui/panel.js +3 -3
- package/dist/core/wake.js +0 -12
- package/dist/core/worktree.js +0 -2
- package/dist/daemon/__tests__/reconciler-signature.test.d.ts +1 -0
- package/dist/daemon/__tests__/reconciler-signature.test.js +36 -0
- package/dist/daemon/api/__tests__/error-seam.test.js +27 -0
- package/dist/daemon/api/__tests__/seam/api-server.test.js +2 -16
- package/dist/daemon/api/__tests__/seam/b10-attach-modelauth.test.js +3 -11
- package/dist/daemon/api/__tests__/seam/leaf-api-parity.test.js +0 -6
- package/dist/daemon/api/handlers/attach.js +2 -1
- package/dist/daemon/api/handlers/broker-ops.js +3 -6
- package/dist/daemon/api/handlers/canvas.js +12 -18
- package/dist/daemon/api/handlers/feedback-comments.js +0 -8
- package/dist/daemon/api/handlers/health.js +3 -2
- package/dist/daemon/api/handlers/human-requests.js +0 -10
- package/dist/daemon/api/handlers/human.js +1 -5
- package/dist/daemon/api/handlers/inbox.js +0 -18
- package/dist/daemon/api/handlers/memory.js +1 -1
- package/dist/daemon/api/handlers/messages.js +16 -20
- package/dist/daemon/api/handlers/nodes.js +23 -22
- package/dist/daemon/api/handlers/reports.js +1 -1
- package/dist/daemon/api/handlers/reviews.js +1 -6
- package/dist/daemon/api/handlers/worktree.js +6 -22
- package/dist/daemon/api/map.d.ts +4 -2
- package/dist/daemon/api/map.js +22 -8
- package/dist/daemon/api/router.js +0 -4
- package/dist/daemon/api/server.js +2 -5
- package/dist/daemon/boot-migrations.d.ts +14 -0
- package/dist/daemon/boot-migrations.js +31 -0
- package/dist/daemon/cron/armed-context.d.ts +21 -0
- package/dist/daemon/cron/armed-context.js +100 -0
- package/dist/daemon/cron/capture.d.ts +20 -0
- package/dist/daemon/cron/capture.js +45 -0
- package/dist/daemon/cron/disposition.d.ts +19 -0
- package/dist/daemon/cron/disposition.js +52 -0
- package/dist/daemon/cron/escalation.d.ts +12 -0
- package/dist/daemon/cron/escalation.js +124 -0
- package/dist/daemon/cron/held.d.ts +4 -0
- package/dist/daemon/cron/held.js +20 -0
- package/dist/daemon/cron/lease-recovery.d.ts +3 -0
- package/dist/daemon/cron/lease-recovery.js +119 -0
- package/dist/daemon/cron/live-runs.d.ts +11 -0
- package/dist/daemon/cron/live-runs.js +23 -0
- package/dist/daemon/cron/passes.d.ts +11 -0
- package/dist/daemon/cron/passes.js +78 -0
- package/dist/daemon/cron/settle.d.ts +28 -0
- package/dist/daemon/cron/settle.js +37 -0
- package/dist/daemon/cron/sinks.d.ts +2 -0
- package/dist/daemon/cron/sinks.js +100 -0
- package/dist/daemon/cron-run.d.ts +5 -41
- package/dist/daemon/cron-run.js +193 -882
- package/dist/daemon/crtrd-cli.js +2 -1
- package/dist/daemon/crtrd.js +22 -22
- package/dist/daemon/fleet.js +0 -4
- package/dist/daemon/human/deliver-action.js +2 -1
- package/dist/daemon/human/sweep.js +15 -4
- package/dist/daemon/manage.d.ts +2 -2
- package/dist/daemon/manage.js +4 -13
- package/dist/daemon/pidfile.js +2 -2
- package/dist/daemon/reconcilers/broker-supervision.js +27 -30
- package/dist/daemon/reconcilers/dormant-inbox.d.ts +1 -1
- package/dist/daemon/reconcilers/dormant-inbox.js +2 -1
- package/dist/daemon/reconcilers/live-obligation.js +2 -2
- package/dist/daemon/reconcilers/pending-review-submit.d.ts +1 -1
- package/dist/daemon/reconcilers/pending-review-submit.js +2 -1
- package/dist/daemon/reconcilers/storage-maintenance.d.ts +2 -1
- package/dist/daemon/reconcilers/storage-maintenance.js +2 -1
- package/dist/daemon/review/companion.js +2 -2
- package/dist/daemon/review/sweep.js +8 -3
- package/dist/migrations/001-surfaces-frontmatter.js +0 -6
- package/dist/migrations/corpus.js +2 -5
- package/dist/migrations/registry.js +15 -2
- package/dist/pi-extensions/__tests__/canvas-stophook-agentend.test.js +0 -4
- package/dist/pi-extensions/broker-local.js +2 -1
- package/dist/pi-extensions/canvas-bash-valve.js +7 -20
- package/dist/pi-extensions/canvas-context-intro.js +2 -7
- package/dist/pi-extensions/canvas-doc-substrate.js +2 -3
- package/dist/pi-extensions/canvas-goal-capture.js +2 -1
- package/dist/pi-extensions/canvas-inbox-watcher.js +5 -12
- package/dist/pi-extensions/canvas-passive-context.js +2 -5
- package/dist/pi-extensions/canvas-preview-result.js +2 -1
- package/dist/pi-extensions/canvas-prompt-scrub.js +2 -1
- package/dist/pi-extensions/canvas-recap.js +3 -16
- package/dist/pi-extensions/canvas-review-boundary.js +2 -5
- package/dist/pi-extensions/canvas-stophook.js +2 -26
- package/dist/pi-extensions/canvas-structured-output.js +2 -1
- package/dist/pi-extensions/canvas-tool-guide.js +2 -1
- package/dist/shared/__tests__/env-boundary.test.d.ts +1 -0
- package/dist/shared/__tests__/env-boundary.test.js +93 -0
- package/dist/shared/env.d.ts +68 -0
- package/dist/shared/env.js +171 -0
- package/package.json +3 -1
- package/runtime.lock.json +2 -2
- package/dist/core/canvas/render.d.ts +0 -47
- package/dist/core/canvas/render.js +0 -245
package/dist/daemon/cron-run.js
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
|
-
// src/daemon/cron-run.ts — the daemon-owned cron run executor
|
|
2
|
-
// spawn/capture/timeout/lease mechanics
|
|
3
|
-
//
|
|
4
|
-
//
|
|
1
|
+
// src/daemon/cron-run.ts — the daemon-owned cron run executor and the entry
|
|
2
|
+
// surface of the cron lane: subprocess spawn/capture/timeout/lease mechanics
|
|
3
|
+
// for due `crons` rows (the cron-spec scheduler), and the tick pass that fires
|
|
4
|
+
// them. The collaborators it composes live in `cron/`: arm-time context,
|
|
5
|
+
// output capture, the live-run registry, the pre-run passes and capacity
|
|
6
|
+
// admission, stale-lease recovery, the owed-gate decision, delivery sinks,
|
|
7
|
+
// escalation, output disposition, and the one settle tail.
|
|
5
8
|
//
|
|
6
9
|
// A cron RUN is the enactment — there is no separate action to fire —
|
|
7
10
|
// so a recurrence advances `fire_at` BEFORE the run (crash-safe: a crash
|
|
@@ -21,6 +24,8 @@
|
|
|
21
24
|
// disposition except `silent` escalates: pause + spawn — pausing is the
|
|
22
25
|
// natural dedup (a paused row never refires). An out-of-band run (`cron run`)
|
|
23
26
|
// never escalates or pauses: the invoker sees the result directly.
|
|
27
|
+
// Those rules are enforced in `cron/disposition.ts` and `cron/settle.ts`; this
|
|
28
|
+
// file only decides WHICH outcome a closed child settled into.
|
|
24
29
|
//
|
|
25
30
|
// Fire-and-forget contract: runDueCrons
|
|
26
31
|
// NEVER awaits a subprocess in the tick — it kicks each child off
|
|
@@ -29,589 +34,165 @@
|
|
|
29
34
|
// record/lease/consume transition is written. The daemon tick registers those
|
|
30
35
|
// promises for graceful-teardown drain but never blocks on a slow run.
|
|
31
36
|
import { spawn } from 'node:child_process';
|
|
32
|
-
import { createHash,
|
|
33
|
-
import {
|
|
34
|
-
import {
|
|
35
|
-
import {
|
|
36
|
-
import {
|
|
37
|
-
import {
|
|
38
|
-
import {
|
|
39
|
-
import { dueClockCrons,
|
|
40
|
-
import { getNode } from '../core/canvas/canvas.js';
|
|
41
|
-
import { fullName } from '../core/canvas/labels.js';
|
|
37
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
38
|
+
import { CRON_KILL_GRACE_MS, CRON_OUTPUT_HEAD_BYTES, EMPTY_STDOUT_SHA256, newHeadCapture, } from './cron/capture.js';
|
|
39
|
+
import { clearLiveRun, cronDaemonInstanceId, liveRunFor, registerLiveRun } from './cron/live-runs.js';
|
|
40
|
+
import { buildArmedRunContext } from './cron/armed-context.js';
|
|
41
|
+
import { decideHeldSettle, EXIT_HELD, HELD_DELIVERED } from './cron/held.js';
|
|
42
|
+
import { settleRun } from './cron/settle.js';
|
|
43
|
+
import { applyOverlapPolicies, cronLaunchesNode, emitCapacityDeferral, expireDueCrons, } from './cron/passes.js';
|
|
44
|
+
import { dueClockCrons, acquireCronRunLease, advanceCron, setCronState, } from '../core/canvas/crons.js';
|
|
42
45
|
import { killProcessGroup } from '../core/canvas/pid.js';
|
|
43
|
-
import { appendInbox } from '../core/feed/inbox.js';
|
|
44
46
|
import { emitEvent } from '../core/events/emit.js';
|
|
45
|
-
import {
|
|
46
|
-
import {
|
|
47
|
-
|
|
48
|
-
import
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
return;
|
|
79
|
-
}
|
|
80
|
-
chunks.push(chunk);
|
|
81
|
-
bytes += chunk.length;
|
|
82
|
-
},
|
|
83
|
-
decode() {
|
|
84
|
-
const text = Buffer.concat(chunks).toString('utf8');
|
|
85
|
-
return truncated ? `${text}\n[… output truncated at ${CRON_OUTPUT_HEAD_BYTES} bytes]` : text;
|
|
86
|
-
},
|
|
87
|
-
};
|
|
88
|
-
}
|
|
89
|
-
/** Bound between the run-timeout SIGTERM and a forced SIGKILL escalation, so
|
|
90
|
-
* a signal-trapping command cannot stall teardown's detached drain. */
|
|
91
|
-
const CRON_KILL_GRACE_MS = 2000;
|
|
92
|
-
/** Poll interval while waiting for a SIGTERMed stale-lease run to actually die
|
|
93
|
-
* (each probe is a `ps` call, so this is deliberately coarse). */
|
|
94
|
-
const CRON_DEATH_POLL_MS = 250;
|
|
95
|
-
/** sha256 of empty stdout — the change-detection hash a run that produced no
|
|
96
|
-
* output carries (a failed spawn, a recovered stale lease). */
|
|
97
|
-
const EMPTY_STDOUT_SHA256 = createHash('sha256').update('').digest('hex');
|
|
98
|
-
/** Lead of a `cron_runs.delivered` string written by the overlap=skip pass.
|
|
99
|
-
* Matched (not just written) — see `latestRunIsSkip`. */
|
|
100
|
-
const SKIP_MARKER = 'skipped (overlap=skip)';
|
|
101
|
-
/** EX_TEMPFAIL — the owed-gate disposition. A scheduled run exiting 75
|
|
102
|
-
* declares "this occurrence is owed but not currently eligible": the row is
|
|
103
|
-
* PARKED (held=1, occurrence not spent) instead of disposed — no failure, no
|
|
104
|
-
* escalation, no delivery, no on-change hash, no one-shot consumption. A poke
|
|
105
|
-
* (`POST /v1/crons/poke`) re-dues it now; otherwise a recurring row re-checks
|
|
106
|
-
* at its already-advanced natural slot and a held one-shot waits at its
|
|
107
|
-
* backstop (`expires_at`, else far future — poke-only). The gate itself stays
|
|
108
|
-
* in bash; the daemon only honors the exit code. */
|
|
109
|
-
const EXIT_HELD = 75;
|
|
110
|
-
/** `cron_runs.delivered` string for a held (exit-75) settlement. */
|
|
111
|
-
const HELD_DELIVERED = 'deferred (exit 75) — held for poke';
|
|
112
|
-
/** Backstop `fire_at` for a held one-shot without `--expires`: never due on
|
|
113
|
-
* the clock — the row fires only on a poke. Without this, a past-due parked
|
|
114
|
-
* one-shot would re-fire every tick forever. */
|
|
115
|
-
const HELD_ONE_SHOT_FAR_FUTURE = '9999-12-31T23:59:59.999Z';
|
|
116
|
-
/** One id per daemon process instance — distinguishes THIS process's own
|
|
117
|
-
* in-flight run leases from a stale 'running' row a prior (crashed/restarted)
|
|
118
|
-
* daemon left behind. */
|
|
119
|
-
export const cronDaemonInstanceId = randomUUID();
|
|
120
|
-
// ---------------------------------------------------------------------------
|
|
121
|
-
// Broker-capacity admission — the fleet-safety invariant the retired clock pass
|
|
122
|
-
// enforced (it reserved capacity before a due fresh revive and left an
|
|
123
|
-
// unadmitted one due for the next poll, so a fleet of simultaneous alarms was
|
|
124
|
-
// never stampeded). A cron's enactment is an opaque shell command, so the only
|
|
125
|
-
// structural signals the daemon has are the armed sink and the crtr verb the
|
|
126
|
-
// command invokes. Both misreadings are safe: a false negative behaves exactly
|
|
127
|
-
// as an ungated run does today, and a false positive only DEFERS one poll while
|
|
128
|
-
// the fleet is already at its hard cap.
|
|
129
|
-
// ---------------------------------------------------------------------------
|
|
130
|
-
/** crtr verbs that launch a broker: a node revive (single or graph-wide) and a
|
|
131
|
-
* node birth. `node message send` is deliberately absent — a message is not a
|
|
132
|
-
* launch (its wake rides the daemon's own capacity-gated inbox pass) — EXCEPT
|
|
133
|
-
* in its `--fresh` form, which is a revive wearing the message verb. */
|
|
134
|
-
const NODE_LAUNCHING_COMMAND = /\bcrtr\b[^\n]*?\bnode\s+lifecycle\s+revive\b|\bcrtr\b[^\n]*?\bnode\s+new\b|\bcrtr\b[^\n]*?\bcanvas\s+revive\b|\bcrtr\b[^\n]*?\bnode\s+message\s+send\b[^\n]*?\s--fresh\b/;
|
|
135
|
-
/** Will this cron's run consume broker capacity? True when its armed sink
|
|
136
|
-
* births a node, or its command invokes a node-launching crtr verb. A cron
|
|
137
|
-
* that merely runs a shell command or sends a message reads false and is
|
|
138
|
-
* never gated. */
|
|
139
|
-
export function cronLaunchesNode(c) {
|
|
140
|
-
if (parseSink(c.sink)?.kind === 'spawn')
|
|
141
|
-
return true;
|
|
142
|
-
return NODE_LAUNCHING_COMMAND.test(c.command);
|
|
143
|
-
}
|
|
144
|
-
const liveRuns = new Map();
|
|
145
|
-
/** Mark a cancellation as originating from the live cron run it names. Only
|
|
146
|
-
* that exact process may finish after its row is removed, so an external cancel
|
|
147
|
-
* retains its normal process-group kill behavior. */
|
|
148
|
-
export function markSelfCanceledCronRun(cronId, runId) {
|
|
149
|
-
const live = liveRuns.get(cronId);
|
|
150
|
-
if (live === undefined || live.runId !== runId)
|
|
151
|
-
return false;
|
|
152
|
-
live.selfCanceled = true;
|
|
153
|
-
return true;
|
|
154
|
-
}
|
|
155
|
-
/** The cron's profile, re-validated at delivery time and returned in canonical
|
|
156
|
-
* id form when its manifest still resolves, else null (root fallback per spec). */
|
|
157
|
-
function resolveLiveProfile(c) {
|
|
158
|
-
if (c.profile == null)
|
|
159
|
-
return null;
|
|
160
|
-
try {
|
|
161
|
-
return loadProfileManifest(c.profile).profileId;
|
|
162
|
-
}
|
|
163
|
-
catch {
|
|
164
|
-
return null;
|
|
165
|
-
}
|
|
166
|
-
}
|
|
167
|
-
function resolveLiveProfileName(c) {
|
|
168
|
-
if (c.profile == null)
|
|
169
|
-
return undefined;
|
|
170
|
-
try {
|
|
171
|
-
return loadProfileManifest(c.profile).manifest.name;
|
|
172
|
-
}
|
|
173
|
-
catch {
|
|
174
|
-
return undefined;
|
|
47
|
+
import { nextSlotAfter } from '../core/wake.js';
|
|
48
|
+
import { resolveAdmittedHostCommand } from '../core/runtime/spawn-env.js';
|
|
49
|
+
// The entry surface the daemon tick, the API handlers, and the cron tests
|
|
50
|
+
// import: the executor and tick pass below, plus the collaborators they were
|
|
51
|
+
// decomposed into.
|
|
52
|
+
export { CRON_OUTPUT_HEAD_BYTES } from './cron/capture.js';
|
|
53
|
+
export { cronDaemonInstanceId, markSelfCanceledCronRun } from './cron/live-runs.js';
|
|
54
|
+
export { cronLaunchesNode } from './cron/passes.js';
|
|
55
|
+
export { recoverStaleCronLeases } from './cron/lease-recovery.js';
|
|
56
|
+
/** Settle one closed child: choose the outcome (replaced short-circuit, owed
|
|
57
|
+
* gate, or the full disposition), then hand it to the one settle tail. */
|
|
58
|
+
async function settleClosedRun(c, run) {
|
|
59
|
+
const { runId, outOfBand, startedAtIso, finishedAtMs, code, timedOut, processError } = run;
|
|
60
|
+
// The owed-gate branch (exit 75): a peer of the wasReplaced short-circuit —
|
|
61
|
+
// skip disposition entirely (no delivery, no escalation, no pause, no
|
|
62
|
+
// on-change hash, no one-shot consumption) and settle the row's held state
|
|
63
|
+
// with the lease release instead. The exclusions are deliberate: a manual
|
|
64
|
+
// `cron run` must not change scheduling state; a timeout-killed or
|
|
65
|
+
// launch-failed run is a real failure even if a stray 75 surfaces; a replaced
|
|
66
|
+
// or self-canceled run's existing short-circuits win.
|
|
67
|
+
const heldDeferred = !outOfBand &&
|
|
68
|
+
!timedOut &&
|
|
69
|
+
processError == null &&
|
|
70
|
+
code === EXIT_HELD &&
|
|
71
|
+
!run.wasReplaced &&
|
|
72
|
+
!run.wasSelfCanceled;
|
|
73
|
+
let outcome;
|
|
74
|
+
let lease;
|
|
75
|
+
if (run.wasReplaced) {
|
|
76
|
+
// Record it, no disposition, no escalation, and leave `held` untouched
|
|
77
|
+
// (the gate is still unresolved).
|
|
78
|
+
outcome = { kind: 'stated', delivered: 'replaced (overlap=replace)' };
|
|
79
|
+
lease = 'release';
|
|
175
80
|
}
|
|
176
|
-
|
|
177
|
-
//
|
|
178
|
-
//
|
|
179
|
-
//
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
* sink is a FAILURE per spec, never a silent drop. */
|
|
183
|
-
function deliverNodeSink(c, target, stdout) {
|
|
184
|
-
const meta = getNode(target);
|
|
185
|
-
if (meta === null)
|
|
186
|
-
throw new Error(`node sink ${target} is gone`);
|
|
187
|
-
// A node parked by the unattended clock is written `done` with the park
|
|
188
|
-
// marker. Its cron is an explicit next cycle, unlike mass revival or deferred
|
|
189
|
-
// delivery, so accept precisely that parked shape. A final latch, a
|
|
190
|
-
// cancellation, or a node finished any other way remains undeliverable and
|
|
191
|
-
// takes the ordinary failure path.
|
|
192
|
-
if (hasNoNaturalCycle(meta) && !isParked(meta)) {
|
|
193
|
-
throw new Error(`node sink ${target} has no natural cycle ahead (done or finalized)`);
|
|
81
|
+
else if (heldDeferred) {
|
|
82
|
+
// The occurrence is NOT spent — even a one-shot is retained (like a paused
|
|
83
|
+
// escalated one-shot is). Park/re-due and the lease release are one write.
|
|
84
|
+
// A row canceled mid-run has nothing to settle.
|
|
85
|
+
outcome = { kind: 'stated', delivered: HELD_DELIVERED };
|
|
86
|
+
lease = decideHeldSettle(c, startedAtIso, finishedAtMs) ?? 'release';
|
|
194
87
|
}
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
88
|
+
else {
|
|
89
|
+
outcome = {
|
|
90
|
+
kind: 'disposed',
|
|
91
|
+
ctx: {
|
|
92
|
+
runId,
|
|
93
|
+
startedAtIso,
|
|
94
|
+
exitCode: timedOut ? -1 : code,
|
|
95
|
+
timedOut,
|
|
96
|
+
runTimeoutS: c.run_timeout_s,
|
|
97
|
+
processErrorMessage: processError != null ? processError.message : null,
|
|
98
|
+
stdoutHead: run.stdoutHead,
|
|
99
|
+
stdoutSha256: run.stdoutSha256,
|
|
100
|
+
stderrHead: run.stderrHead,
|
|
101
|
+
},
|
|
102
|
+
selfCanceled: run.wasSelfCanceled,
|
|
103
|
+
};
|
|
104
|
+
// An ordinary scheduled disposition settlement resolves the gate, so it
|
|
105
|
+
// unparks a held row; an out-of-band run leaves `held` untouched.
|
|
106
|
+
lease = outOfBand ? 'release' : { kind: 'clear' };
|
|
212
107
|
}
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
// PARENTLESS TERMINAL node: nobody to report to, but it still owes a final
|
|
231
|
-
// so it reaps, and it is never brought forefront. A resident root here
|
|
232
|
-
// would leave an hourly cron accreting ~24 permanently-live nodes a day
|
|
233
|
-
// (the count-based prune never deletes live nodes) and would steal the
|
|
234
|
-
// human's tmux focus on every unattended fire.
|
|
235
|
-
...(creatorAlive ? { parent: c.created_by } : { root: true, rootLifecycle: 'terminal' }),
|
|
236
|
-
readiness: 'launch',
|
|
237
|
-
profile: resolveLiveProfile(c),
|
|
238
|
-
wakeOrigin: cronWakeOrigin(c, 'node_birth'),
|
|
108
|
+
return settleRun(c, {
|
|
109
|
+
outOfBand,
|
|
110
|
+
outcome,
|
|
111
|
+
record: {
|
|
112
|
+
runId,
|
|
113
|
+
startedAtIso,
|
|
114
|
+
finishedAtMs,
|
|
115
|
+
durationMs: finishedAtMs - run.startedAtMs,
|
|
116
|
+
exitCode: timedOut ? -1 : code,
|
|
117
|
+
stdoutHead: run.stdoutHead === '' ? null : run.stdoutHead,
|
|
118
|
+
stderrHead: processError != null
|
|
119
|
+
? `run process error: ${processError.message}`.slice(0, CRON_OUTPUT_HEAD_BYTES)
|
|
120
|
+
: run.stderrHead === ''
|
|
121
|
+
? null
|
|
122
|
+
: run.stderrHead,
|
|
123
|
+
},
|
|
124
|
+
lease,
|
|
239
125
|
});
|
|
240
|
-
return res.node.node_id;
|
|
241
126
|
}
|
|
242
|
-
/**
|
|
243
|
-
* `
|
|
244
|
-
*
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
127
|
+
/** Settle a run whose child never existed — the arm-time context was corrupt,
|
|
128
|
+
* or `spawn` threw synchronously — so no lease was ever taken. Settled through
|
|
129
|
+
* the failure path so a broken cron pauses + escalates (and keeps its one-shot)
|
|
130
|
+
* instead of refiring every tick. `lead` names which of the two happened. */
|
|
131
|
+
function settleUnspawnedRun(c, run, lead, err) {
|
|
132
|
+
const finishedAtMs = Date.now();
|
|
133
|
+
const stderrHead = `${lead}: ${err.message}`.slice(0, CRON_OUTPUT_HEAD_BYTES);
|
|
134
|
+
return settleRun(c, {
|
|
135
|
+
outOfBand: run.outOfBand,
|
|
136
|
+
outcome: {
|
|
137
|
+
kind: 'disposed',
|
|
138
|
+
ctx: {
|
|
139
|
+
runId: run.runId,
|
|
140
|
+
startedAtIso: run.startedAtIso,
|
|
141
|
+
exitCode: null,
|
|
142
|
+
timedOut: false,
|
|
143
|
+
runTimeoutS: c.run_timeout_s,
|
|
144
|
+
processErrorMessage: err.message,
|
|
145
|
+
stdoutHead: '',
|
|
146
|
+
stdoutSha256: EMPTY_STDOUT_SHA256,
|
|
147
|
+
stderrHead,
|
|
258
148
|
},
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
* success; throws on an undeliverable sink (the caller's failure path). */
|
|
271
|
-
async function deliverToSink(c, stdout) {
|
|
272
|
-
const sink = parseSink(c.sink);
|
|
273
|
-
if (sink === null)
|
|
274
|
-
return 'no sink armed — nothing delivered';
|
|
275
|
-
if (sink.kind === 'node') {
|
|
276
|
-
deliverNodeSink(c, sink.node, stdout);
|
|
277
|
-
return `delivered: node ${sink.node}`;
|
|
278
|
-
}
|
|
279
|
-
if (sink.kind === 'spawn') {
|
|
280
|
-
const nodeId = await deliverSpawnSink(c, sink.spawn, stdout);
|
|
281
|
-
return `delivered: spawned ${nodeId}`;
|
|
282
|
-
}
|
|
283
|
-
deliverHumanSink(c, stdout);
|
|
284
|
-
return 'delivered: human send --inbox';
|
|
285
|
-
}
|
|
286
|
-
function failureSummary(ctx, deliveryFailure) {
|
|
287
|
-
if (deliveryFailure !== null)
|
|
288
|
-
return deliveryFailure;
|
|
289
|
-
if (ctx.timedOut)
|
|
290
|
-
return `run timed out after ${ctx.runTimeoutS}s (killed; recorded exit -1)`;
|
|
291
|
-
if (ctx.processErrorMessage !== null)
|
|
292
|
-
return `run process error: ${ctx.processErrorMessage}`;
|
|
293
|
-
return `command exited ${ctx.exitCode ?? '(no exit observed)'}`;
|
|
294
|
-
}
|
|
295
|
-
/** The escalation node's kickoff: the cron's identity/config, this failure,
|
|
296
|
-
* the run-log ring, the creating-node reference, and a mandate that names its
|
|
297
|
-
* actual options (resume after verifying / cancel / ask the human). */
|
|
298
|
-
function buildEscalationKickoff(c, ctx, deliveryFailure) {
|
|
299
|
-
const creator = c.created_by != null ? getNode(c.created_by) : null;
|
|
300
|
-
const creatorLine = c.created_by == null
|
|
301
|
-
? 'created by: (no creating node recorded)'
|
|
302
|
-
: creator !== null
|
|
303
|
-
? `created by: node ${c.created_by} ("${fullName(creator)}") — still on the canvas; consider telling it what happened`
|
|
304
|
-
: `created by: node ${c.created_by} (now gone)`;
|
|
305
|
-
const schedule = c.recur != null ? `every ${cadenceDisplay(c.recur)}` : `one-shot at ${c.fire_at}`;
|
|
306
|
-
const priorRuns = listCronRuns(c.cron_id)
|
|
307
|
-
.slice(0, 10)
|
|
308
|
-
.map((r) => {
|
|
309
|
-
const exit = r.exit_code === null ? 'no exit' : `exit ${r.exit_code}`;
|
|
310
|
-
const head = (r.stderr_head ?? r.stdout_head ?? '').split('\n')[0]?.slice(0, 120) ?? '';
|
|
311
|
-
return `- ${r.started} — ${exit}, ${r.duration_ms}ms — ${r.delivered ?? ''}${head !== '' ? ` — ${head}` : ''}`;
|
|
149
|
+
},
|
|
150
|
+
record: {
|
|
151
|
+
runId: run.runId,
|
|
152
|
+
startedAtIso: run.startedAtIso,
|
|
153
|
+
finishedAtMs,
|
|
154
|
+
durationMs: finishedAtMs - run.startedAtMs,
|
|
155
|
+
exitCode: null,
|
|
156
|
+
stdoutHead: null,
|
|
157
|
+
stderrHead,
|
|
158
|
+
},
|
|
159
|
+
lease: 'none',
|
|
312
160
|
});
|
|
313
|
-
return [
|
|
314
|
-
`A scheduled cron FAILED and has paused itself. You were spawned to deal with it.`,
|
|
315
|
-
'',
|
|
316
|
-
`- cron: "${c.name}" (${c.cron_id})`,
|
|
317
|
-
`- schedule: ${schedule}${c.tz != null ? ` (tz ${c.tz})` : ''}`,
|
|
318
|
-
`- context: cwd ${c.cwd}${c.profile != null ? `, profile ${c.profile}` : ''}`,
|
|
319
|
-
`- ${creatorLine}`,
|
|
320
|
-
`- this failure: ${failureSummary(ctx, deliveryFailure)}`,
|
|
321
|
-
'',
|
|
322
|
-
'## The command',
|
|
323
|
-
'',
|
|
324
|
-
'```bash',
|
|
325
|
-
c.command,
|
|
326
|
-
'```',
|
|
327
|
-
'',
|
|
328
|
-
'## This run',
|
|
329
|
-
'',
|
|
330
|
-
`- exit: ${ctx.timedOut ? '-1 (timeout)' : String(ctx.exitCode)}`,
|
|
331
|
-
`- stdout (head): ${ctx.stdoutHead === '' ? '(empty)' : ''}`,
|
|
332
|
-
...(ctx.stdoutHead !== '' ? ['```', ctx.stdoutHead, '```'] : []),
|
|
333
|
-
`- stderr (head): ${ctx.stderrHead === '' ? '(empty)' : ''}`,
|
|
334
|
-
...(ctx.stderrHead !== '' ? ['```', ctx.stderrHead, '```'] : []),
|
|
335
|
-
'',
|
|
336
|
-
'## Recent runs (most recent first, this failure not yet recorded)',
|
|
337
|
-
'',
|
|
338
|
-
...(priorRuns.length > 0 ? priorRuns : ['(no prior runs recorded)']),
|
|
339
|
-
'',
|
|
340
|
-
'## Your mandate',
|
|
341
|
-
'',
|
|
342
|
-
`Diagnose why it failed and decide what to do:`,
|
|
343
|
-
`- Transient or fixable: fix the environment/command context, verify with \`crtr cron run ${c.cron_id}\` (fires now, out of band, shows you the result), then \`crtr cron resume ${c.cron_id}\` to re-arm it.`,
|
|
344
|
-
`- Obsolete: \`crtr cron cancel ${c.cron_id}\`.`,
|
|
345
|
-
`- The user's call (or the fix needs a change you cannot make): \`crtr human send\`.`,
|
|
346
|
-
'',
|
|
347
|
-
'The cron stays paused until someone resumes it — nothing refires while you work.',
|
|
348
|
-
].join('\n');
|
|
349
161
|
}
|
|
350
|
-
/**
|
|
351
|
-
*
|
|
352
|
-
*
|
|
353
|
-
*
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
162
|
+
/** Settle the schedule BEFORE the run: a recurrence advances to its next slot
|
|
163
|
+
* now, so a crash mid-run never re-fires this slot (the lease guards the
|
|
164
|
+
* intra-life overlap; the advanced fire_at guards the crash window).
|
|
165
|
+
*
|
|
166
|
+
* Returns the process error to hand `close` when the recurrence cannot be
|
|
167
|
+
* parsed, else null. That row can never be scheduled again as written — but
|
|
168
|
+
* DELETING it would destroy the user's config AND its whole run history with
|
|
169
|
+
* no record and no escalation, the one outcome the spec has no vocabulary for.
|
|
170
|
+
* It takes the failure path instead, so the row PAUSES and an escalation node
|
|
171
|
+
* arrives holding the parse error. `fire_at` is deliberately left past-due: a
|
|
172
|
+
* paused row is not due, and once someone fixes the recurrence a `cron resume`
|
|
173
|
+
* fires immediately. */
|
|
174
|
+
function advanceRecurrence(c, startedAtMs) {
|
|
175
|
+
let nextFire;
|
|
357
176
|
try {
|
|
358
|
-
|
|
359
|
-
kind: 'general',
|
|
360
|
-
cwd: c.cwd,
|
|
361
|
-
prompt: buildEscalationKickoff(c, ctx, deliveryFailure),
|
|
362
|
-
// The spec's word is TERMINAL: a node that deals with this failure and
|
|
363
|
-
// reaps. With the creator gone there is nobody to parent it, but
|
|
364
|
-
// parentless must not mean resident-and-forefront — see deliverSpawnSink.
|
|
365
|
-
...(creatorAlive ? { parent: c.created_by } : { root: true, rootLifecycle: 'terminal' }),
|
|
366
|
-
readiness: 'launch',
|
|
367
|
-
profile: resolveLiveProfile(c),
|
|
368
|
-
wakeOrigin: cronWakeOrigin(c, 'node_birth'),
|
|
369
|
-
});
|
|
370
|
-
emitEvent({
|
|
371
|
-
level: 'info',
|
|
372
|
-
event: 'cron.failure.escalated',
|
|
373
|
-
fields: { cron_id: c.cron_id, name: c.name, escalation_node: res.node.node_id },
|
|
374
|
-
});
|
|
375
|
-
return `failure: cron paused, escalation node ${res.node.node_id} spawned`;
|
|
177
|
+
nextFire = nextSlotAfter(c.recur, new Date(startedAtMs));
|
|
376
178
|
}
|
|
377
179
|
catch (err) {
|
|
378
180
|
emitEvent({
|
|
379
181
|
level: 'error',
|
|
380
|
-
event: 'cron.
|
|
182
|
+
event: 'cron.recur.unparseable',
|
|
381
183
|
error: err,
|
|
382
|
-
fields: { cron_id: c.cron_id, name: c.name },
|
|
184
|
+
fields: { cron_id: c.cron_id, name: c.name, recur: c.recur },
|
|
383
185
|
});
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
async function disposeSettledRun(c, ctx, outOfBand, selfCanceled = false) {
|
|
391
|
-
const prefix = outOfBand ? 'out-of-band: ' : '';
|
|
392
|
-
// Re-read the row: last_output_hash/state must be current. An external
|
|
393
|
-
// cancellation suppresses delivery as before. A cancellation made by this
|
|
394
|
-
// exact run is different: the process has already produced its output and
|
|
395
|
-
// the documented self-cancel pattern must deliver that final observation.
|
|
396
|
-
const fresh = getCron(c.cron_id);
|
|
397
|
-
if (fresh === null && !selfCanceled)
|
|
398
|
-
return { delivered: `${prefix}cron canceled mid-run — nothing delivered`, paused: false };
|
|
399
|
-
const effective = fresh ?? c;
|
|
400
|
-
const failed = ctx.timedOut || ctx.processErrorMessage !== null || ctx.exitCode !== 0;
|
|
401
|
-
// on-change bookkeeping happens for every SCHEDULED settled run (delivered or
|
|
402
|
-
// not, failed or not) so the next comparison is against the latest
|
|
403
|
-
// observation. An out-of-band run COMPARES but never WRITES: `cron run` is a
|
|
404
|
-
// verification, and a baseline it rewrote would make the next scheduled run
|
|
405
|
-
// report "unchanged" and skip a delivery the user should have received.
|
|
406
|
-
let changed = true;
|
|
407
|
-
if (effective.on_output === 'on-change') {
|
|
408
|
-
changed = effective.last_output_hash !== ctx.stdoutSha256;
|
|
409
|
-
// The row is intentionally gone after an in-run self-cancel, so there is
|
|
410
|
-
// no baseline to persist. It cannot fire again to need one.
|
|
411
|
-
if (!outOfBand && fresh !== null)
|
|
412
|
-
setCronLastOutputHash(c.cron_id, ctx.stdoutSha256);
|
|
413
|
-
}
|
|
414
|
-
if (failed) {
|
|
415
|
-
if (effective.on_output === 'silent')
|
|
416
|
-
return { delivered: `${prefix}silent`, paused: false };
|
|
417
|
-
if (outOfBand)
|
|
418
|
-
return { delivered: `out-of-band: ${failureSummary(ctx, null)} — not escalated`, paused: false };
|
|
419
|
-
return { delivered: await escalateFailure(effective, ctx, null), paused: true };
|
|
420
|
-
}
|
|
421
|
-
switch (effective.on_output) {
|
|
422
|
-
case 'silent':
|
|
423
|
-
return { delivered: `${prefix}silent`, paused: false };
|
|
424
|
-
case 'on-failure':
|
|
425
|
-
return { delivered: `${prefix}ok — nothing to deliver (on-failure)`, paused: false };
|
|
426
|
-
case 'on-change':
|
|
427
|
-
if (!changed)
|
|
428
|
-
return { delivered: `${prefix}unchanged — not delivered (on-change)`, paused: false };
|
|
429
|
-
return deliverOrEscalate(effective, ctx, outOfBand, prefix);
|
|
430
|
-
case 'always':
|
|
431
|
-
return deliverOrEscalate(effective, ctx, outOfBand, prefix);
|
|
432
|
-
}
|
|
433
|
-
}
|
|
434
|
-
/** The shared delivering tail of `always` and a CHANGED `on-change`: push
|
|
435
|
-
* stdout to the armed sink, or take the failure path when the sink is
|
|
436
|
-
* undeliverable. */
|
|
437
|
-
async function deliverOrEscalate(c, ctx, outOfBand, prefix) {
|
|
438
|
-
if (ctx.stdoutHead.trim() === '') {
|
|
439
|
-
return { delivered: `${prefix}empty stdout — nothing delivered`, paused: false };
|
|
440
|
-
}
|
|
441
|
-
try {
|
|
442
|
-
return { delivered: `${prefix}${await deliverToSink(c, ctx.stdoutHead)}`, paused: false };
|
|
443
|
-
}
|
|
444
|
-
catch (err) {
|
|
445
|
-
const why = `delivery failed: ${err.message}`;
|
|
446
|
-
if (outOfBand)
|
|
447
|
-
return { delivered: `out-of-band: ${why} — not escalated`, paused: false };
|
|
448
|
-
return { delivered: await escalateFailure(c, ctx, why), paused: true };
|
|
186
|
+
// Pause HERE, not only via the escalation: `fire_at` was never advanced, so
|
|
187
|
+
// an on_output='silent' row (whose failure path never escalates and never
|
|
188
|
+
// pauses) would otherwise re-fire this unschedulable command on every
|
|
189
|
+
// single tick, forever.
|
|
190
|
+
setCronState(c.cron_id, 'paused');
|
|
191
|
+
return new Error(`unparseable recurrence ${JSON.stringify(c.recur)}: ${err.message} — this cron cannot be rescheduled until its recurrence is fixed (\`crtr cron cancel\` and re-arm it), so it has been paused.`);
|
|
449
192
|
}
|
|
193
|
+
advanceCron(c.cron_id, nextFire);
|
|
194
|
+
return null;
|
|
450
195
|
}
|
|
451
|
-
// ---------------------------------------------------------------------------
|
|
452
|
-
// Stale-lease recovery
|
|
453
|
-
// ---------------------------------------------------------------------------
|
|
454
|
-
/** Recover stale 'running' rows left by a PRIOR daemon instance: a pid that is
|
|
455
|
-
* not VERIFIABLY still the process this lease launched resets straight to
|
|
456
|
-
* idle; a confirmed-live process group past its own run_timeout_s is killed
|
|
457
|
-
* and settled as a timeout run through the SAME failure path a live timeout
|
|
458
|
-
* takes (pause + escalation unless silent). Rows this SAME instance owns are
|
|
459
|
-
* genuinely in flight and left alone.
|
|
460
|
-
*
|
|
461
|
-
* The verification is `verifyLeasedCronRun`, not `isPidAlive`, and it is
|
|
462
|
-
* FAIL-CLOSED: this row was written by a daemon life that has since ended, so
|
|
463
|
-
* after a host reboot its `run_pid` names a slot in a dead kernel's pid space
|
|
464
|
-
* that the OS has probably recycled, and the timeout comparison below is
|
|
465
|
-
* trivially satisfied by any downtime. Liveness alone would therefore SIGTERM
|
|
466
|
-
* a stranger's process group. Only a matching launch-time identity authorizes
|
|
467
|
-
* the signal; everything else releases the lease. */
|
|
468
|
-
export function recoverStaleCronLeases(now, options = {}) {
|
|
469
|
-
for (const c of runningCrons()) {
|
|
470
|
-
if (c.run_lease_owner === cronDaemonInstanceId)
|
|
471
|
-
continue;
|
|
472
|
-
if (!verifyLeasedCronRun(c)) {
|
|
473
|
-
settleCrashedRun(c, now, options);
|
|
474
|
-
continue;
|
|
475
|
-
}
|
|
476
|
-
const startedAtMs = c.run_started_at != null ? new Date(c.run_started_at).getTime() : now;
|
|
477
|
-
if (now - startedAtMs < c.run_timeout_s * 1000)
|
|
478
|
-
continue; // within its own window
|
|
479
|
-
if (!killLeasedCronRun(c)) {
|
|
480
|
-
// The process exited (or was re-identified) between the two probes: there
|
|
481
|
-
// is nothing to signal, so settle it as the unobserved run it now is.
|
|
482
|
-
settleCrashedRun(c, now, options);
|
|
483
|
-
continue;
|
|
484
|
-
}
|
|
485
|
-
const ctx = {
|
|
486
|
-
runId: randomUUID(),
|
|
487
|
-
startedAtIso: c.run_started_at ?? new Date(now).toISOString(),
|
|
488
|
-
exitCode: -1,
|
|
489
|
-
timedOut: true,
|
|
490
|
-
runTimeoutS: c.run_timeout_s,
|
|
491
|
-
processErrorMessage: null,
|
|
492
|
-
stdoutHead: '',
|
|
493
|
-
stdoutSha256: EMPTY_STDOUT_SHA256,
|
|
494
|
-
stderrHead: `run timed out after ${c.run_timeout_s}s (recovered stale lease from a prior daemon instance)`,
|
|
495
|
-
};
|
|
496
|
-
const settlement = (async () => {
|
|
497
|
-
await awaitLeasedRunDeath(c);
|
|
498
|
-
const disposition = await disposeSettledRun(c, ctx, false);
|
|
499
|
-
recordCronRun({
|
|
500
|
-
run_id: ctx.runId,
|
|
501
|
-
cron_id: c.cron_id,
|
|
502
|
-
started: ctx.startedAtIso,
|
|
503
|
-
finished: new Date(now).toISOString(),
|
|
504
|
-
duration_ms: now - startedAtMs,
|
|
505
|
-
exit_code: -1,
|
|
506
|
-
stdout_head: null,
|
|
507
|
-
stderr_head: ctx.stderrHead,
|
|
508
|
-
delivered: disposition.delivered,
|
|
509
|
-
});
|
|
510
|
-
// A recovered-timeout settlement took the ordinary disposition path
|
|
511
|
-
// (pause + escalation unless silent) — the gate is resolved, clear held.
|
|
512
|
-
releaseCronRunLease(c.cron_id, { kind: 'clear' });
|
|
513
|
-
if (c.recur == null && !disposition.paused)
|
|
514
|
-
consumeCron(c.cron_id);
|
|
515
|
-
})();
|
|
516
|
-
options.register?.(settlement);
|
|
517
|
-
void settlement.catch((err) => {
|
|
518
|
-
emitEvent({ level: 'error', event: 'cron.stale_lease.settle_failed', error: err, fields: { cron_id: c.cron_id } });
|
|
519
|
-
});
|
|
520
|
-
}
|
|
521
|
-
}
|
|
522
|
-
/** Wait for a SIGTERMed stale-lease run to actually die, escalating to SIGKILL
|
|
523
|
-
* if it does not — the same bound `killWithEscalation` gives a live timeout.
|
|
524
|
-
* Recovery must not record the run as finished the instant it signals: a
|
|
525
|
-
* signal-trapping command can still be running and still producing side
|
|
526
|
-
* effects. */
|
|
527
|
-
async function awaitLeasedRunDeath(c) {
|
|
528
|
-
const deadline = Date.now() + CRON_KILL_GRACE_MS;
|
|
529
|
-
while (Date.now() < deadline) {
|
|
530
|
-
await delay(CRON_DEATH_POLL_MS);
|
|
531
|
-
if (!verifyLeasedCronRun(c))
|
|
532
|
-
return; // gone
|
|
533
|
-
}
|
|
534
|
-
killLeasedCronRun(c, 'SIGKILL');
|
|
535
|
-
}
|
|
536
|
-
/** Settle a stale lease whose process can no longer be observed: the daemon
|
|
537
|
-
* life that owned this run ended (SIGKILL, host reboot) and nothing recorded
|
|
538
|
-
* the outcome. Releasing the lease and nothing else is wrong twice.
|
|
539
|
-
*
|
|
540
|
-
* 1. The run must leave a RECORD. The run log's whole justification is "it
|
|
541
|
-
* silently stopped working Tuesday"; a hole exactly where the crash
|
|
542
|
-
* happened is the one gap it cannot afford.
|
|
543
|
-
* 2. A ONE-SHOT must not RE-FIRE. A recurrence advanced `fire_at` before its
|
|
544
|
-
* run, so releasing it to idle costs at most a missed slot. A one-shot's
|
|
545
|
-
* `fire_at` is still past-due and it is deleted only after a settle that
|
|
546
|
-
* never came — so releasing it re-runs a command whose real-world effect
|
|
547
|
-
* (post the report, send the money, open the PR) may already have completed
|
|
548
|
-
* in full. It is PAUSED instead (the same dedup the failure path uses: a
|
|
549
|
-
* paused row is never due) and escalated, so a node decides whether the
|
|
550
|
-
* effect landed — `cron run` to redo it, `cron cancel` to drop it.
|
|
551
|
-
*
|
|
552
|
-
* A `silent` cron pauses without an escalation node: silent means record only,
|
|
553
|
-
* never deliver, never escalate. */
|
|
554
|
-
function settleCrashedRun(c, now, options) {
|
|
555
|
-
const oneShot = c.recur == null;
|
|
556
|
-
const startedAtMs = c.run_started_at != null ? new Date(c.run_started_at).getTime() : now;
|
|
557
|
-
const why = 'the daemon life that owned this run ended before it settled — the process is gone and its outcome was never observed';
|
|
558
|
-
const ctx = {
|
|
559
|
-
runId: randomUUID(),
|
|
560
|
-
startedAtIso: c.run_started_at ?? new Date(now).toISOString(),
|
|
561
|
-
exitCode: null,
|
|
562
|
-
timedOut: false,
|
|
563
|
-
runTimeoutS: c.run_timeout_s,
|
|
564
|
-
processErrorMessage: why,
|
|
565
|
-
stdoutHead: '',
|
|
566
|
-
stdoutSha256: EMPTY_STDOUT_SHA256,
|
|
567
|
-
stderrHead: `run outcome unobserved: ${why}`,
|
|
568
|
-
};
|
|
569
|
-
const settlement = (async () => {
|
|
570
|
-
let delivered = `crashed: ${why}${oneShot ? '' : ' — the next occurrence is unaffected'}`;
|
|
571
|
-
if (oneShot) {
|
|
572
|
-
// Stop the re-fire FIRST, so even a throwing escalation cannot leave the
|
|
573
|
-
// row active with a past-due fire_at. (escalateFailure pauses too; the
|
|
574
|
-
// second write is a harmless no-op.)
|
|
575
|
-
setCronState(c.cron_id, 'paused');
|
|
576
|
-
delivered =
|
|
577
|
-
c.on_output === 'silent'
|
|
578
|
-
? `crashed: ${why} — one-shot paused, not re-fired (silent: not escalated)`
|
|
579
|
-
: `crashed one-shot: ${await escalateFailure(c, ctx, crashedOneShotSummary(why))}`;
|
|
580
|
-
}
|
|
581
|
-
recordCronRun({
|
|
582
|
-
run_id: ctx.runId,
|
|
583
|
-
cron_id: c.cron_id,
|
|
584
|
-
started: ctx.startedAtIso,
|
|
585
|
-
finished: new Date(now).toISOString(),
|
|
586
|
-
duration_ms: Math.max(0, now - startedAtMs),
|
|
587
|
-
exit_code: null,
|
|
588
|
-
stdout_head: null,
|
|
589
|
-
stderr_head: ctx.stderrHead,
|
|
590
|
-
delivered,
|
|
591
|
-
});
|
|
592
|
-
releaseCronRunLease(c.cron_id);
|
|
593
|
-
emitEvent({
|
|
594
|
-
level: 'warn',
|
|
595
|
-
event: 'cron.run.unobserved',
|
|
596
|
-
fields: { cron_id: c.cron_id, name: c.name, one_shot: oneShot },
|
|
597
|
-
});
|
|
598
|
-
})();
|
|
599
|
-
options.register?.(settlement);
|
|
600
|
-
void settlement.catch((err) => {
|
|
601
|
-
emitEvent({ level: 'error', event: 'cron.crashed_run.settle_failed', error: err, fields: { cron_id: c.cron_id } });
|
|
602
|
-
});
|
|
603
|
-
}
|
|
604
|
-
function crashedOneShotSummary(why) {
|
|
605
|
-
return [
|
|
606
|
-
`${why}.`,
|
|
607
|
-
'This is a ONE-SHOT, so it has been PAUSED rather than released: its side effects may have completed in full,',
|
|
608
|
-
'and re-firing would enact them a second time. Determine whether the work actually landed, then either',
|
|
609
|
-
'`crtr cron run` it to redo it deliberately, or `crtr cron cancel` it.',
|
|
610
|
-
].join(' ');
|
|
611
|
-
}
|
|
612
|
-
// ---------------------------------------------------------------------------
|
|
613
|
-
// The run executor
|
|
614
|
-
// ---------------------------------------------------------------------------
|
|
615
196
|
/** Kick off ONE cron run and return its settlement promise, resolving with
|
|
616
197
|
* the settled run record (null = the idle→running lease was lost, or the row
|
|
617
198
|
* was quarantined — nothing was recorded by THIS call). The subprocess
|
|
@@ -627,57 +208,22 @@ function crashedOneShotSummary(why) {
|
|
|
627
208
|
export function executeCron(c, opts) {
|
|
628
209
|
const { outOfBand } = opts;
|
|
629
210
|
return new Promise((resolve, reject) => {
|
|
630
|
-
let extraEnv = {};
|
|
631
|
-
if (c.env_json != null) {
|
|
632
|
-
try {
|
|
633
|
-
const parsed = JSON.parse(c.env_json);
|
|
634
|
-
for (const [k, v] of Object.entries(parsed)) {
|
|
635
|
-
if (typeof v === 'string')
|
|
636
|
-
extraEnv[k] = v;
|
|
637
|
-
}
|
|
638
|
-
}
|
|
639
|
-
catch {
|
|
640
|
-
extraEnv = {}; // a malformed snapshot never blocks the run itself
|
|
641
|
-
}
|
|
642
|
-
}
|
|
643
211
|
const runId = randomUUID();
|
|
644
|
-
// The cron RUNS UNDER THE PROFILE IT WAS ARMED UNDER (re-validated here, root
|
|
645
|
-
// fallback when the manifest is gone). The spec's central promise is that
|
|
646
|
-
// "messaging a node, reviving one, birthing one is `crtr node …` inside the
|
|
647
|
-
// command" — and every one of those resolves against a profile. Passing the
|
|
648
|
-
// profile to buildOperationalEnvBase only steers `spawnEnv.allow` config
|
|
649
|
-
// resolution; the child also needs CRTR_PROFILE_ID itself (not an
|
|
650
|
-
// operational literal name, so it never crosses on its own) and the
|
|
651
|
-
// profile's own env store, exactly as buildBrokerEnv gives a broker — storing
|
|
652
|
-
// a value there IS the operator's consent for it to cross this boundary.
|
|
653
|
-
// Layering: profile store < the arm-time `--env` snapshot (an explicit
|
|
654
|
-
// choice) < crtr's own constructed CRTR_* names, which always win.
|
|
655
|
-
const profileId = resolveLiveProfile(c);
|
|
656
|
-
const operationalEnv = buildOperationalEnvBase({ targetCwd: c.cwd, targetProfileId: profileId });
|
|
657
|
-
const env = {
|
|
658
|
-
...operationalEnv,
|
|
659
|
-
...readProfileEnvVars(profileId),
|
|
660
|
-
...extraEnv,
|
|
661
|
-
...(profileId !== null ? { CRTR_PROFILE_ID: profileId } : {}),
|
|
662
|
-
CRTR_CRON_ID: c.cron_id,
|
|
663
|
-
CRTR_CRON_NAME: c.name,
|
|
664
|
-
CRTR_RUN_ID: runId,
|
|
665
|
-
};
|
|
666
|
-
// An explicit canvas home is part of the runtime's own identity, not host
|
|
667
|
-
// state a `spawnEnv.allow` list opts into: a cron command's whole promise
|
|
668
|
-
// is that `crtr node …` inside it talks to THIS daemon. Without it the
|
|
669
|
-
// child crtr resolves the default `~/.crouter/canvas`, so every crtr call
|
|
670
|
-
// in a cron body fails to reach the daemon that spawned it. Same
|
|
671
|
-
// propagation the broker/child-node boundary does (`core/runtime/nodes.ts`).
|
|
672
|
-
const canvasHomeOverride = process.env['CRTR_HOME'];
|
|
673
|
-
if (canvasHomeOverride !== undefined && canvasHomeOverride !== '') {
|
|
674
|
-
env['CRTR_HOME'] = canvasHomeOverride;
|
|
675
|
-
}
|
|
676
|
-
if (c.anchor_node != null)
|
|
677
|
-
env['CRTR_ANCHOR'] = c.anchor_node;
|
|
678
|
-
env[ADMITTED_HOST_PATH_ENV] = operationalEnv[ADMITTED_HOST_PATH_ENV];
|
|
679
212
|
const startedAtMs = Date.now();
|
|
680
213
|
const startedAtIso = new Date(startedAtMs).toISOString();
|
|
214
|
+
const unspawned = { runId, outOfBand, startedAtIso, startedAtMs };
|
|
215
|
+
// The context the run was ARMED under, built before anything is spawned: a
|
|
216
|
+
// corrupt snapshot must not execute the command under an environment or
|
|
217
|
+
// profile nobody armed. It settles through the same no-lease failure path a
|
|
218
|
+
// synchronous spawn failure takes.
|
|
219
|
+
let env;
|
|
220
|
+
try {
|
|
221
|
+
({ env } = buildArmedRunContext(c, runId));
|
|
222
|
+
}
|
|
223
|
+
catch (err) {
|
|
224
|
+
settleUnspawnedRun(c, unspawned, 'run could not be armed', err).then(resolve, reject);
|
|
225
|
+
return;
|
|
226
|
+
}
|
|
681
227
|
let child;
|
|
682
228
|
try {
|
|
683
229
|
child = spawn(resolveAdmittedHostCommand('bash', env, c.cwd), ['-c', c.command], {
|
|
@@ -688,38 +234,7 @@ export function executeCron(c, opts) {
|
|
|
688
234
|
});
|
|
689
235
|
}
|
|
690
236
|
catch (err) {
|
|
691
|
-
|
|
692
|
-
// Settle through the failure path so a broken cron pauses + escalates
|
|
693
|
-
// instead of refiring every tick.
|
|
694
|
-
const ctx = {
|
|
695
|
-
runId,
|
|
696
|
-
startedAtIso,
|
|
697
|
-
exitCode: null,
|
|
698
|
-
timedOut: false,
|
|
699
|
-
runTimeoutS: c.run_timeout_s,
|
|
700
|
-
processErrorMessage: err.message,
|
|
701
|
-
stdoutHead: '',
|
|
702
|
-
stdoutSha256: EMPTY_STDOUT_SHA256,
|
|
703
|
-
stderrHead: `run failed to spawn: ${err.message}`.slice(0, CRON_OUTPUT_HEAD_BYTES),
|
|
704
|
-
};
|
|
705
|
-
void (async () => {
|
|
706
|
-
const disposition = await disposeSettledRun(c, ctx, outOfBand);
|
|
707
|
-
const record = {
|
|
708
|
-
run_id: runId,
|
|
709
|
-
cron_id: c.cron_id,
|
|
710
|
-
started: startedAtIso,
|
|
711
|
-
finished: new Date().toISOString(),
|
|
712
|
-
duration_ms: Date.now() - startedAtMs,
|
|
713
|
-
exit_code: null,
|
|
714
|
-
stdout_head: null,
|
|
715
|
-
stderr_head: ctx.stderrHead,
|
|
716
|
-
delivered: disposition.delivered,
|
|
717
|
-
};
|
|
718
|
-
recordCronRun(record);
|
|
719
|
-
if (!outOfBand && c.recur == null && !disposition.paused)
|
|
720
|
-
consumeCron(c.cron_id);
|
|
721
|
-
resolve(record);
|
|
722
|
-
})().catch(reject);
|
|
237
|
+
settleUnspawnedRun(c, unspawned, 'run failed to spawn', err).then(resolve, reject);
|
|
723
238
|
return;
|
|
724
239
|
}
|
|
725
240
|
let settled = false;
|
|
@@ -767,121 +282,31 @@ export function executeCron(c, opts) {
|
|
|
767
282
|
clearTimeout(killTimer);
|
|
768
283
|
const stdoutHead = stdout.decode();
|
|
769
284
|
const stderrHead = stderr.decode();
|
|
770
|
-
const live =
|
|
285
|
+
const live = liveRunFor(c.cron_id);
|
|
771
286
|
const wasReplaced = live !== undefined && live.runId === runId && live.replaced;
|
|
772
287
|
const wasSelfCanceled = live !== undefined && live.runId === runId && live.selfCanceled;
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
if (wasReplaced) {
|
|
796
|
-
// overlap=replace killed this run so the next fire could start
|
|
797
|
-
// fresh: record it, no disposition, no escalation.
|
|
798
|
-
delivered = 'replaced (overlap=replace)';
|
|
799
|
-
}
|
|
800
|
-
else if (heldDeferred) {
|
|
801
|
-
delivered = HELD_DELIVERED;
|
|
802
|
-
// Re-read the row: `last_poke_at` must be CURRENT — a poke landing
|
|
803
|
-
// while this gate check was in flight is exactly the race this
|
|
804
|
-
// check closes. (A row canceled mid-run is gone; recordCronRun and
|
|
805
|
-
// the lease release below both no-op.)
|
|
806
|
-
const fresh = getCron(c.cron_id);
|
|
807
|
-
if (fresh !== null) {
|
|
808
|
-
if (fresh.state === 'active' && fresh.last_poke_at !== null && startedAtIso < fresh.last_poke_at) {
|
|
809
|
-
// A poke arrived mid-run — its unpark statement deliberately
|
|
810
|
-
// skips leased rows — so re-due instead of park: the gate
|
|
811
|
-
// re-checks next tick instead of stalling until the backstop
|
|
812
|
-
// while the very eligibility it waits for is present. Only for
|
|
813
|
-
// a row still ACTIVE: a row paused mid-run parks below instead —
|
|
814
|
-
// the poke skips paused rows on purpose while its stamp lands on
|
|
815
|
-
// every row, and a re-due would leave a past-due UNHELD row that
|
|
816
|
-
// `cron resume` fires blind even though its gate never passed.
|
|
817
|
-
heldSettle = { kind: 'redue', nowIso: new Date(finishedAtMs).toISOString() };
|
|
818
|
-
}
|
|
819
|
-
else if (c.recur != null) {
|
|
820
|
-
// Recurring: the pre-run advance already wrote the natural next
|
|
821
|
-
// slot, which IS the backstop — park and touch nothing else.
|
|
822
|
-
heldSettle = { kind: 'park' };
|
|
823
|
-
}
|
|
824
|
-
else {
|
|
825
|
-
// One-shot: park at its backstop. With --expires, expiry deletes
|
|
826
|
-
// the row unfired at that instant (bounded wait); without, the
|
|
827
|
-
// far-future fire_at means poke-only.
|
|
828
|
-
heldSettle = { kind: 'park', fireAt: fresh.expires_at ?? HELD_ONE_SHOT_FAR_FUTURE };
|
|
829
|
-
}
|
|
830
|
-
}
|
|
831
|
-
}
|
|
832
|
-
else {
|
|
833
|
-
const disposition = await disposeSettledRun(c, {
|
|
834
|
-
runId,
|
|
835
|
-
startedAtIso,
|
|
836
|
-
exitCode: timedOut ? -1 : code,
|
|
837
|
-
timedOut,
|
|
838
|
-
runTimeoutS: c.run_timeout_s,
|
|
839
|
-
processErrorMessage: processError != null ? processError.message : null,
|
|
840
|
-
stdoutHead,
|
|
841
|
-
stdoutSha256: stdoutHash.digest('hex'),
|
|
842
|
-
stderrHead,
|
|
843
|
-
}, outOfBand, wasSelfCanceled);
|
|
844
|
-
delivered = disposition.delivered;
|
|
845
|
-
paused = disposition.paused;
|
|
846
|
-
}
|
|
847
|
-
const record = {
|
|
848
|
-
run_id: runId,
|
|
849
|
-
cron_id: c.cron_id,
|
|
850
|
-
started: startedAtIso,
|
|
851
|
-
finished: new Date(finishedAtMs).toISOString(),
|
|
852
|
-
duration_ms: finishedAtMs - startedAtMs,
|
|
853
|
-
exit_code: timedOut ? -1 : code,
|
|
854
|
-
stdout_head: stdoutHead === '' ? null : stdoutHead,
|
|
855
|
-
stderr_head: processError != null
|
|
856
|
-
? `run process error: ${processError.message}`.slice(0, CRON_OUTPUT_HEAD_BYTES)
|
|
857
|
-
: stderrHead === ''
|
|
858
|
-
? null
|
|
859
|
-
: stderrHead,
|
|
860
|
-
delivered,
|
|
861
|
-
};
|
|
862
|
-
recordCronRun(record);
|
|
863
|
-
if (heldDeferred) {
|
|
864
|
-
// Held: the occurrence is NOT spent — even a one-shot is retained
|
|
865
|
-
// (like a paused escalated one-shot is). Park/re-due and the lease
|
|
866
|
-
// release are one write.
|
|
867
|
-
releaseCronRunLease(c.cron_id, heldSettle);
|
|
868
|
-
}
|
|
869
|
-
else if (!outOfBand && c.recur == null && !paused) {
|
|
870
|
-
// One-shot: the run is done, delete the row (its history cascades
|
|
871
|
-
// with it — matching one-shot consumption). A failing
|
|
872
|
-
// one-shot that PAUSED for escalation is kept, so `cron resume` can
|
|
873
|
-
// re-arm its still-due fire_at. A racing explicit cancel already
|
|
874
|
-
// removed it; consumeCron no-ops.
|
|
875
|
-
consumeCron(c.cron_id);
|
|
876
|
-
}
|
|
877
|
-
else {
|
|
878
|
-
// An ordinary scheduled disposition settlement resolves the gate,
|
|
879
|
-
// so it unparks a held row; a replaced or out-of-band run leaves
|
|
880
|
-
// `held` untouched (the gate is still unresolved).
|
|
881
|
-
releaseCronRunLease(c.cron_id, wasReplaced || outOfBand ? undefined : { kind: 'clear' });
|
|
882
|
-
}
|
|
883
|
-
resolve(record);
|
|
884
|
-
})().catch(reject);
|
|
288
|
+
clearLiveRun(c.cron_id, runId);
|
|
289
|
+
if (leaseLost) {
|
|
290
|
+
// Lost the idle→running race (explicit cancel or another daemon's
|
|
291
|
+
// lease) — whoever won owns the row's disposition; abandon silently.
|
|
292
|
+
resolve(null);
|
|
293
|
+
return;
|
|
294
|
+
}
|
|
295
|
+
settleClosedRun(c, {
|
|
296
|
+
runId,
|
|
297
|
+
outOfBand,
|
|
298
|
+
startedAtIso,
|
|
299
|
+
startedAtMs,
|
|
300
|
+
finishedAtMs: Date.now(),
|
|
301
|
+
code,
|
|
302
|
+
timedOut,
|
|
303
|
+
processError,
|
|
304
|
+
stdoutHead,
|
|
305
|
+
stderrHead,
|
|
306
|
+
stdoutSha256: stdoutHash.digest('hex'),
|
|
307
|
+
wasReplaced,
|
|
308
|
+
wasSelfCanceled,
|
|
309
|
+
}).then(resolve, reject);
|
|
885
310
|
});
|
|
886
311
|
if (child.pid == null ||
|
|
887
312
|
!acquireCronRunLease(c.cron_id, cronDaemonInstanceId, child.pid, startedAtIso, outOfBand)) {
|
|
@@ -890,7 +315,7 @@ export function executeCron(c, opts) {
|
|
|
890
315
|
leaseLost = true;
|
|
891
316
|
return; // 'close' above performs the actual settlement.
|
|
892
317
|
}
|
|
893
|
-
|
|
318
|
+
registerLiveRun(c.cron_id, {
|
|
894
319
|
runId,
|
|
895
320
|
replaced: false,
|
|
896
321
|
selfCanceled: false,
|
|
@@ -899,40 +324,16 @@ export function executeCron(c, opts) {
|
|
|
899
324
|
killWithEscalation(child.pid);
|
|
900
325
|
},
|
|
901
326
|
});
|
|
902
|
-
// Settle the schedule BEFORE the run: a recurrence advances to its next
|
|
903
|
-
// slot now, so a crash mid-run never re-fires this slot (the lease guards
|
|
904
|
-
// the intra-life overlap; the advanced fire_at guards the crash window).
|
|
905
327
|
// An out-of-band run never touches the schedule.
|
|
906
328
|
if (!outOfBand && c.recur != null) {
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
// Unparseable recurrence. This row can never be scheduled again as
|
|
913
|
-
// written — but DELETING it would destroy the user's config AND its
|
|
914
|
-
// whole run history with no record and no escalation, the
|
|
915
|
-
// one outcome the spec has no vocabulary for. Take the failure path
|
|
916
|
-
// instead: kill the just-started run and hand `close` a process error, so
|
|
917
|
-
// the row PAUSES and an escalation node arrives holding the parse error.
|
|
918
|
-
// `fire_at` is deliberately left past-due: a paused row is not due, and
|
|
919
|
-
// once someone fixes the recurrence a `cron resume` fires immediately.
|
|
920
|
-
emitEvent({
|
|
921
|
-
level: 'error',
|
|
922
|
-
event: 'cron.recur.unparseable',
|
|
923
|
-
error: err,
|
|
924
|
-
fields: { cron_id: c.cron_id, name: c.name, recur: c.recur },
|
|
925
|
-
});
|
|
926
|
-
processError = new Error(`unparseable recurrence ${JSON.stringify(c.recur)}: ${err.message} — this cron cannot be rescheduled until its recurrence is fixed (\`crtr cron cancel\` and re-arm it), so it has been paused.`);
|
|
927
|
-
// Pause HERE, not only via the escalation: `fire_at` was never advanced,
|
|
928
|
-
// so an on_output='silent' row (whose failure path never escalates and
|
|
929
|
-
// never pauses) would otherwise re-fire this unschedulable command on
|
|
930
|
-
// every single tick, forever.
|
|
931
|
-
setCronState(c.cron_id, 'paused');
|
|
329
|
+
const recurError = advanceRecurrence(c, startedAtMs);
|
|
330
|
+
if (recurError !== null) {
|
|
331
|
+
// Kill the just-started run and hand `close` a process error, so it
|
|
332
|
+
// settles through the ordinary failure path.
|
|
333
|
+
processError = recurError;
|
|
932
334
|
killWithEscalation(child.pid);
|
|
933
|
-
return;
|
|
335
|
+
return;
|
|
934
336
|
}
|
|
935
|
-
advanceCron(c.cron_id, nextFire);
|
|
936
337
|
}
|
|
937
338
|
timer = setTimeout(() => {
|
|
938
339
|
if (settled)
|
|
@@ -944,81 +345,6 @@ export function executeCron(c, opts) {
|
|
|
944
345
|
timer.unref?.();
|
|
945
346
|
});
|
|
946
347
|
}
|
|
947
|
-
// ---------------------------------------------------------------------------
|
|
948
|
-
// The per-tick passes: expiry, overlap policy, due runs
|
|
949
|
-
// ---------------------------------------------------------------------------
|
|
950
|
-
/** Delete idle crons whose clock bound has passed. A running expired row is
|
|
951
|
-
* left for a later tick (its run settles first). */
|
|
952
|
-
function expireDueCrons(nowIso) {
|
|
953
|
-
for (const c of expiredIdleCrons(nowIso)) {
|
|
954
|
-
cancelCron(c.cron_id);
|
|
955
|
-
emitEvent({
|
|
956
|
-
level: 'info',
|
|
957
|
-
event: 'cron.expired',
|
|
958
|
-
fields: { cron_id: c.cron_id, name: c.name, expires_at: c.expires_at },
|
|
959
|
-
});
|
|
960
|
-
}
|
|
961
|
-
}
|
|
962
|
-
/** Apply each recurring cron's overlap policy to a next fire that has landed
|
|
963
|
-
* while its previous run is still leased:
|
|
964
|
-
* skip — advance fire_at past the missed slot and record a skip entry
|
|
965
|
-
* (the observable evidence a fire was deliberately not started).
|
|
966
|
-
* queue — do nothing: fire_at stays past-due, so the moment the running
|
|
967
|
-
* lease releases, the next tick fires immediately. Multiple
|
|
968
|
-
* missed fires collapse structurally (fire_at holds ONE
|
|
969
|
-
* occurrence) — the spec's depth cap of 1.
|
|
970
|
-
* replace — mark THIS instance's in-flight run replaced and kill it; its
|
|
971
|
-
* close handler records the replacement (no escalation) and
|
|
972
|
-
* releases the lease, and the still-due fire_at starts fresh on
|
|
973
|
-
* the next tick. A stale other-instance lease is left to
|
|
974
|
-
* recoverStaleCronLeases. */
|
|
975
|
-
/** True when this cron's most recent run record is already a skip marker — the
|
|
976
|
-
* streak guard in the overlap=skip pass. */
|
|
977
|
-
function latestRunIsSkip(cronId) {
|
|
978
|
-
return listCronRuns(cronId)[0]?.delivered?.startsWith(SKIP_MARKER) === true;
|
|
979
|
-
}
|
|
980
|
-
function applyOverlapPolicies(now, nowIso) {
|
|
981
|
-
for (const c of dueOverlapCrons(nowIso)) {
|
|
982
|
-
if (c.overlap === 'queue')
|
|
983
|
-
continue;
|
|
984
|
-
if (c.overlap === 'skip') {
|
|
985
|
-
let nextFire;
|
|
986
|
-
try {
|
|
987
|
-
nextFire = nextSlotAfter(c.recur, new Date(now));
|
|
988
|
-
}
|
|
989
|
-
catch {
|
|
990
|
-
continue; // paused + escalated by the run path when it next fires
|
|
991
|
-
}
|
|
992
|
-
advanceCron(c.cron_id, nextFire);
|
|
993
|
-
// Record only the FIRST skip of a streak. A run that overruns its cadence
|
|
994
|
-
// produces one skipped occurrence per period, and recorded
|
|
995
|
-
// unconditionally a 25-minute run on a 1-minute cadence writes 25 skip
|
|
996
|
-
// markers — evicting every real run record from the 20-entry ring, which
|
|
997
|
-
// is the exact loss the ring exists to prevent. The next settled run ends
|
|
998
|
-
// the streak, so the next overlap after it records again.
|
|
999
|
-
if (!latestRunIsSkip(c.cron_id)) {
|
|
1000
|
-
recordCronRun({
|
|
1001
|
-
run_id: randomUUID(),
|
|
1002
|
-
cron_id: c.cron_id,
|
|
1003
|
-
started: nowIso,
|
|
1004
|
-
finished: nowIso,
|
|
1005
|
-
duration_ms: 0,
|
|
1006
|
-
exit_code: null,
|
|
1007
|
-
stdout_head: null,
|
|
1008
|
-
stderr_head: null,
|
|
1009
|
-
delivered: `${SKIP_MARKER}: fire landed while the previous run was in flight (consecutive skips are not re-recorded until a run settles)`,
|
|
1010
|
-
});
|
|
1011
|
-
}
|
|
1012
|
-
continue;
|
|
1013
|
-
}
|
|
1014
|
-
// replace
|
|
1015
|
-
const live = liveRuns.get(c.cron_id);
|
|
1016
|
-
if (c.run_lease_owner === cronDaemonInstanceId && live !== undefined && !live.replaced) {
|
|
1017
|
-
live.replaced = true;
|
|
1018
|
-
live.kill();
|
|
1019
|
-
}
|
|
1020
|
-
}
|
|
1021
|
-
}
|
|
1022
348
|
/** The daemon's cron pass: expire clock-bounded rows, apply overlap policies,
|
|
1023
349
|
* then kick off every due active idle cron. Each child starts synchronously
|
|
1024
350
|
* but is NEVER awaited in the tick; returns one settlement promise per
|
|
@@ -1052,18 +378,3 @@ export function runDueCrons(now, options = {}) {
|
|
|
1052
378
|
emitCapacityDeferral(now, deferred);
|
|
1053
379
|
return settlements;
|
|
1054
380
|
}
|
|
1055
|
-
/** Throttled record of capacity-deferred runs. Unthrottled this would fire on
|
|
1056
|
-
* every 2s poll for as long as the fleet stays at its cap — the same flood the
|
|
1057
|
-
* daemon's own capacity event guards against. */
|
|
1058
|
-
const CAPACITY_DEFER_LOG_THROTTLE_MS = 30_000;
|
|
1059
|
-
let lastCapacityDeferLogAt = Number.NEGATIVE_INFINITY;
|
|
1060
|
-
function emitCapacityDeferral(now, deferred) {
|
|
1061
|
-
if (now - lastCapacityDeferLogAt < CAPACITY_DEFER_LOG_THROTTLE_MS)
|
|
1062
|
-
return;
|
|
1063
|
-
lastCapacityDeferLogAt = now;
|
|
1064
|
-
emitEvent({
|
|
1065
|
-
level: 'warn',
|
|
1066
|
-
event: 'cron.capacity.run_deferred',
|
|
1067
|
-
fields: { deferred_runs: deferred },
|
|
1068
|
-
});
|
|
1069
|
-
}
|