@ouro.bot/cli 0.1.0-alpha.83 → 0.1.0-alpha.831
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +154 -23
- package/RepairGuide.ouro/agent.json +5 -0
- package/RepairGuide.ouro/psyche/IDENTITY.md +19 -0
- package/RepairGuide.ouro/psyche/SOUL.md +55 -0
- package/RepairGuide.ouro/skills/diagnose-broken-remote.md +63 -0
- package/RepairGuide.ouro/skills/diagnose-stacked-typed-issues.md +35 -0
- package/RepairGuide.ouro/skills/diagnose-sync-blocked.md +54 -0
- package/RepairGuide.ouro/skills/diagnose-vault-expired.md +60 -0
- package/{AdoptionSpecialist.ouro → SerpentGuide.ouro}/agent.json +4 -2
- package/{AdoptionSpecialist.ouro → SerpentGuide.ouro}/psyche/SOUL.md +2 -2
- package/{AdoptionSpecialist.ouro → SerpentGuide.ouro}/psyche/identities/the-serpent.md +1 -1
- package/assets/bluebubbles-host +264 -0
- package/assets/sanctuary-host-launcher.sh +16 -0
- package/changelog.json +5308 -16
- package/deploy/unraid/Dockerfile +54 -0
- package/deploy/unraid/README.txt +2983 -0
- package/deploy/unraid/audit-container-spec.sh +4 -0
- package/deploy/unraid/container-runtime.json +1 -0
- package/deploy/unraid/docker-man-template-transaction.mjs +711 -0
- package/deploy/unraid/docker-man-template-xml.cjs +105 -0
- package/deploy/unraid/migrate-sanctuary-bundle.mjs +60 -0
- package/deploy/unraid/ouro-events/bootstrap-spool.sh +92 -0
- package/deploy/unraid/ouro-events/emit-event.mjs +310 -0
- package/deploy/unraid/ouro-events/emit-usenet-event.sh +54 -0
- package/deploy/unraid/ouro-events/install-usenet-guard.sh +435 -0
- package/deploy/unraid/ouro-events/usenet-health.sh +176 -0
- package/deploy/unraid/sanctuary-acceptance-adapter.sh +43 -0
- package/deploy/unraid/sanctuary-acceptance-contract.json +198 -0
- package/deploy/unraid/sanctuary-acceptance-harness.sh +23 -0
- package/deploy/unraid/sanctuary-authority-installation.json +36 -0
- package/deploy/unraid/sanctuary-authority-service.sh +30 -0
- package/deploy/unraid/sanctuary-deployment-target.mjs +710 -0
- package/deploy/unraid/sanctuary-safe-rename.py +24 -0
- package/deploy/unraid/sanctuary-unit16-host-broker.mjs +2084 -0
- package/deploy/unraid/sanctuary-unit16-run.sh +560 -0
- package/deploy/unraid/sanctuary-unit18-target-audit.sh +27 -0
- package/deploy/unraid/sanctuary.ouro/agent.json +66 -0
- package/deploy/unraid/sanctuary.ouro/arc/README.md +3 -0
- package/deploy/unraid/sanctuary.ouro/bundle-meta.json +5 -0
- package/deploy/unraid/sanctuary.ouro/habits/sanctuary-health.md +8 -0
- package/deploy/unraid/sanctuary.ouro/mcp/media-mcp.mjs +795 -0
- package/deploy/unraid/sanctuary.ouro/provider-readiness.json +11 -0
- package/deploy/unraid/sanctuary.ouro/psyche/ASPIRATIONS.md +5 -0
- package/deploy/unraid/sanctuary.ouro/psyche/IDENTITY.md +5 -0
- package/deploy/unraid/sanctuary.ouro/psyche/LORE.md +15 -0
- package/deploy/unraid/sanctuary.ouro/psyche/SOUL.md +25 -0
- package/deploy/unraid/sanctuary.ouro/psyche/TACIT.md +15 -0
- package/deploy/unraid/sanctuary.ouro/state/policy/steward.json +7 -0
- package/deploy/unraid/sanctuary.ouro/tool-profiles.json +135 -0
- package/deploy/unraid/sanctuary.xml +28 -0
- package/dist/a2a/card.js +57 -0
- package/dist/a2a/client.js +180 -0
- package/dist/a2a/config.js +50 -0
- package/dist/a2a/delegation-stores.js +56 -0
- package/dist/a2a/did-resolution.js +93 -0
- package/dist/a2a/identity.js +170 -0
- package/dist/a2a/inbound-share.js +107 -0
- package/dist/a2a/mission-result-wire.js +196 -0
- package/dist/a2a/onboarding.js +115 -0
- package/dist/a2a/pin-store.js +132 -0
- package/dist/a2a/seen-ledger.js +120 -0
- package/dist/a2a/server.js +604 -0
- package/dist/a2a/task-store.js +69 -0
- package/dist/a2a/transport.js +45 -0
- package/dist/a2a/types.js +3 -0
- package/dist/arc/attention-types.js +8 -0
- package/dist/arc/cares.js +360 -0
- package/dist/arc/episodes.js +118 -0
- package/dist/arc/evolution.js +490 -0
- package/dist/arc/flight-recorder.js +1034 -0
- package/dist/arc/intentions.js +134 -0
- package/dist/arc/json-store.js +117 -0
- package/dist/arc/obligations.js +386 -0
- package/dist/arc/packets.js +336 -0
- package/dist/arc/presence.js +185 -0
- package/dist/arc/task-lifecycle.js +57 -0
- package/dist/commerce/store.js +755 -0
- package/dist/commerce/types.js +3 -0
- package/dist/heart/active-work.js +798 -40
- package/dist/heart/agent-entry.js +252 -3
- package/dist/heart/approval-files.js +113 -0
- package/dist/heart/approval-store.js +960 -0
- package/dist/heart/attachments/image-normalize.js +194 -0
- package/dist/heart/attachments/materialize.js +97 -0
- package/dist/heart/attachments/originals.js +88 -0
- package/dist/heart/attachments/render.js +29 -0
- package/dist/heart/attachments/sources/bluebubbles.js +156 -0
- package/dist/heart/attachments/sources/cli-local-file.js +78 -0
- package/dist/heart/attachments/sources/index.js +18 -0
- package/dist/heart/attachments/sources/telegram.js +71 -0
- package/dist/heart/attachments/store.js +132 -0
- package/dist/heart/attachments/types.js +93 -0
- package/dist/heart/auth/auth-flow.js +495 -0
- package/dist/heart/autonomy-budget.js +478 -0
- package/dist/heart/awaiting/await-alert.js +148 -0
- package/dist/heart/awaiting/await-expiry.js +143 -0
- package/dist/heart/awaiting/await-loader.js +91 -0
- package/dist/heart/awaiting/await-parser.js +149 -0
- package/dist/heart/awaiting/await-runtime-state.js +111 -0
- package/dist/heart/awaiting/await-scheduler.js +382 -0
- package/dist/heart/background-operations.js +281 -0
- package/dist/heart/bridges/manager.js +137 -17
- package/dist/heart/bridges/store.js +14 -2
- package/dist/heart/bundle-state.js +168 -0
- package/dist/heart/commitments.js +44 -18
- package/dist/heart/config-registry.js +331 -0
- package/dist/heart/config.js +174 -131
- package/dist/heart/context-loss-gauntlet.js +387 -0
- package/dist/heart/context-loss-sentinel.js +1157 -0
- package/dist/heart/core.js +2169 -581
- package/dist/heart/cross-chat-delivery.js +9 -22
- package/dist/heart/daemon/agent-config-check.js +451 -0
- package/dist/heart/daemon/agent-discovery.js +185 -3
- package/dist/heart/daemon/agent-service.js +542 -0
- package/dist/heart/daemon/agentic-repair.js +547 -0
- package/dist/heart/daemon/await-private-wake.js +59 -0
- package/dist/heart/daemon/bluebubbles-cli-integration.js +71 -0
- package/dist/heart/daemon/bluebubbles-health-diagnostics.js +122 -0
- package/dist/heart/daemon/bluebubbles-host-protocol.js +433 -0
- package/dist/heart/daemon/bluebubbles-host.js +273 -0
- package/dist/heart/daemon/boot-sync-probe.js +197 -0
- package/dist/heart/daemon/cadence.js +180 -0
- package/dist/heart/daemon/cli-defaults.js +854 -0
- package/dist/heart/daemon/cli-desk.js +322 -0
- package/dist/heart/daemon/cli-exec.js +8822 -0
- package/dist/heart/daemon/cli-help.js +716 -0
- package/dist/heart/daemon/cli-parse.js +2771 -0
- package/dist/heart/daemon/cli-render-doctor.js +57 -0
- package/dist/heart/daemon/cli-render.js +777 -0
- package/dist/heart/daemon/cli-types.js +8 -0
- package/dist/heart/daemon/connect-bay.js +323 -0
- package/dist/heart/daemon/container-credential-bootstrap.js +273 -0
- package/dist/heart/daemon/container-healthcheck.js +112 -0
- package/dist/heart/daemon/container-runtime.js +97 -0
- package/dist/heart/daemon/container-spec-auditor-main.js +171 -0
- package/dist/heart/daemon/container-spec-auditor.js +229 -0
- package/dist/heart/daemon/daemon-bootstrap-startup.js +115 -0
- package/dist/heart/daemon/daemon-cli.js +29 -2349
- package/dist/heart/daemon/daemon-entry.js +930 -11
- package/dist/heart/daemon/daemon-health.js +186 -0
- package/dist/heart/daemon/daemon-rollup.js +57 -0
- package/dist/heart/daemon/daemon-runtime-sync.js +88 -13
- package/dist/heart/daemon/daemon-tombstone.js +236 -0
- package/dist/heart/daemon/daemon.js +2084 -105
- package/dist/heart/daemon/dns-workflow.js +394 -0
- package/dist/heart/daemon/doctor-types.js +8 -0
- package/dist/heart/daemon/doctor.js +1755 -0
- package/dist/heart/daemon/freshness.js +345 -0
- package/dist/heart/daemon/habit-private-wake.js +61 -0
- package/dist/heart/daemon/health-monitor.js +142 -11
- package/dist/heart/daemon/hooks/agent-config-v2.js +33 -0
- package/dist/heart/daemon/hooks/bundle-meta.js +135 -1
- package/dist/heart/daemon/http-health-probe.js +80 -0
- package/dist/heart/daemon/human-command-screens.js +234 -0
- package/dist/heart/daemon/human-readiness.js +114 -0
- package/dist/heart/daemon/inner-status.js +78 -0
- package/dist/heart/daemon/interactive-repair.js +394 -0
- package/dist/heart/daemon/launchd.js +37 -8
- package/dist/heart/daemon/log-tailer.js +79 -10
- package/dist/heart/daemon/logs-prune.js +132 -0
- package/dist/heart/daemon/mcp-canary.js +524 -0
- package/dist/heart/daemon/message-router.js +65 -5
- package/dist/heart/daemon/migrate-to-desk.js +848 -0
- package/dist/heart/daemon/os-cron-deps.js +135 -0
- package/dist/heart/daemon/os-cron.js +22 -14
- package/dist/heart/daemon/ouro-bot-entry.js +4 -2
- package/dist/heart/daemon/ouro-entry.js +3 -1
- package/dist/heart/daemon/plugin-cli.js +432 -0
- package/dist/heart/daemon/process-manager.js +733 -61
- package/dist/heart/daemon/provider-discovery.js +137 -0
- package/dist/heart/daemon/provider-ping-progress.js +83 -0
- package/dist/heart/daemon/prunable-bundle.js +144 -0
- package/dist/heart/daemon/pulse.js +511 -0
- package/dist/heart/daemon/readiness-repair.js +365 -0
- package/dist/heart/daemon/run-hooks.js +2 -0
- package/dist/heart/daemon/runtime-logging.js +47 -14
- package/dist/heart/daemon/runtime-metadata.js +2 -30
- package/dist/heart/daemon/runtime-mode.js +13 -2
- package/dist/heart/daemon/safe-mode.js +161 -0
- package/dist/heart/daemon/sanctuary-acceptance-adapter.js +2654 -0
- package/dist/heart/daemon/sanctuary-acceptance-harness.js +1830 -0
- package/dist/heart/daemon/sanctuary-acceptance-marker.js +267 -0
- package/dist/heart/daemon/sanctuary-acceptance-scenarios.js +955 -0
- package/dist/heart/daemon/sanctuary-authority-codec.js +86 -0
- package/dist/heart/daemon/sanctuary-authority-epoch.js +256 -0
- package/dist/heart/daemon/sanctuary-authority-installation.js +140 -0
- package/dist/heart/daemon/sanctuary-authority-ledger.js +241 -0
- package/dist/heart/daemon/sanctuary-authority-root-lifecycle.js +819 -0
- package/dist/heart/daemon/sanctuary-authority-vault-migration.js +79 -0
- package/dist/heart/daemon/sanctuary-bundle-migration.js +729 -0
- package/dist/heart/daemon/sanctuary-host-authority.js +1008 -0
- package/dist/heart/daemon/sanctuary-host-detached-supervisor.js +494 -0
- package/dist/heart/daemon/sanctuary-host-executor.js +670 -0
- package/dist/heart/daemon/sanctuary-host-linux-kernel.js +334 -0
- package/dist/heart/daemon/sanctuary-host-supervisor-entry.js +207 -0
- package/dist/heart/daemon/sanctuary-host-supervisor.js +95 -0
- package/dist/heart/daemon/sanctuary-package-management.js +103 -0
- package/dist/heart/daemon/sanctuary-scheduler-liveness.js +227 -0
- package/dist/heart/daemon/sanctuary-scheduler-origin.js +132 -0
- package/dist/heart/daemon/sanctuary-telegram-authority-entry.js +445 -0
- package/dist/heart/daemon/sanctuary-telegram-authority-gateway.js +585 -0
- package/dist/heart/daemon/sanctuary-telegram-authority-service.js +630 -0
- package/dist/heart/daemon/sense-manager.js +699 -48
- package/dist/heart/daemon/session-id-resolver.js +131 -0
- package/dist/heart/daemon/skill-management-installer.js +1 -1
- package/dist/heart/daemon/socket-client.js +180 -12
- package/dist/heart/daemon/stale-bundle-prune.js +113 -0
- package/dist/heart/daemon/startup-tui.js +339 -0
- package/dist/heart/daemon/supercronic-supervisor.js +282 -0
- package/dist/heart/daemon/task-scheduler.js +117 -39
- package/dist/heart/daemon/terminal-ui.js +499 -0
- package/dist/heart/daemon/thoughts.js +138 -54
- package/dist/heart/daemon/up-progress.js +366 -0
- package/dist/heart/daemon/vault-items.js +56 -0
- package/dist/heart/delegation.js +1 -4
- package/dist/heart/external-events/router.js +1430 -0
- package/dist/heart/frontend-approval-runtime.js +506 -0
- package/dist/heart/frontend-journal.js +215 -0
- package/dist/heart/frontend-session-service.js +237 -0
- package/dist/heart/frontend-socket-client.js +164 -0
- package/dist/heart/frontend-socket.js +400 -0
- package/dist/heart/habits/habit-cancel.js +810 -0
- package/dist/heart/habits/habit-lifecycle.js +1327 -0
- package/dist/heart/habits/habit-migration.js +194 -0
- package/dist/heart/habits/habit-parser.js +278 -0
- package/dist/heart/habits/habit-runtime-state.js +93 -0
- package/dist/heart/habits/habit-scheduler.js +653 -0
- package/dist/heart/habits/habit-session-summary.js +318 -0
- package/dist/heart/habits/habit-session.js +636 -0
- package/dist/heart/{daemon → hatch}/hatch-flow.js +55 -62
- package/dist/heart/{daemon → hatch}/hatch-specialist.js +6 -8
- package/dist/heart/{daemon → hatch}/specialist-prompt.js +12 -9
- package/dist/heart/{daemon → hatch}/specialist-tools.js +53 -18
- package/dist/heart/identity.js +196 -59
- package/dist/heart/kept-notes.js +289 -0
- package/dist/heart/kicks.js +1 -1
- package/dist/heart/machine-identity.js +161 -0
- package/dist/heart/mail-import-discovery.js +356 -0
- package/dist/heart/mailbox/mailbox-http-hooks.js +98 -0
- package/dist/heart/mailbox/mailbox-http-response.js +7 -0
- package/dist/heart/mailbox/mailbox-http-routes.js +398 -0
- package/dist/heart/mailbox/mailbox-http-static.js +103 -0
- package/dist/heart/mailbox/mailbox-http-transport.js +129 -0
- package/dist/heart/mailbox/mailbox-http.js +99 -0
- package/dist/heart/mailbox/mailbox-read.js +39 -0
- package/dist/heart/mailbox/mailbox-types.js +27 -0
- package/dist/heart/mailbox/mailbox-view.js +197 -0
- package/dist/heart/mailbox/readers/agent-machine.js +428 -0
- package/dist/heart/mailbox/readers/continuity-readers.js +329 -0
- package/dist/heart/mailbox/readers/mail.js +375 -0
- package/dist/heart/mailbox/readers/runtime-readers.js +823 -0
- package/dist/heart/mailbox/readers/sessions.js +231 -0
- package/dist/heart/mailbox/readers/shared.js +111 -0
- package/dist/heart/mcp/mcp-server.js +696 -0
- package/dist/heart/migrate-config.js +100 -0
- package/dist/heart/model-capabilities.js +11 -0
- package/dist/heart/orientation-frame.js +345 -0
- package/dist/heart/platform.js +81 -0
- package/dist/heart/primitives.js +1 -1
- package/dist/heart/private-runtime/decision-renderer.js +158 -0
- package/dist/heart/private-runtime/index.js +18 -0
- package/dist/heart/private-runtime/ledger.js +221 -0
- package/dist/heart/private-runtime/policy.js +252 -0
- package/dist/heart/private-runtime/types.js +2 -0
- package/dist/heart/progress-story.js +1 -1
- package/dist/heart/provider-attempt.js +137 -0
- package/dist/heart/provider-binding-resolver.js +273 -0
- package/dist/heart/provider-credentials.js +449 -0
- package/dist/heart/provider-failover.js +311 -0
- package/dist/heart/provider-models.js +96 -0
- package/dist/heart/provider-ping.js +265 -0
- package/dist/heart/provider-readiness-cache.js +119 -0
- package/dist/heart/provider-visibility.js +188 -0
- package/dist/heart/providers/anthropic-token.js +131 -0
- package/dist/heart/providers/anthropic.js +173 -60
- package/dist/heart/providers/azure.js +23 -11
- package/dist/heart/providers/error-classification.js +127 -0
- package/dist/heart/providers/github-copilot.js +34 -38
- package/dist/heart/providers/minimax-vlm.js +189 -0
- package/dist/heart/providers/minimax.js +26 -8
- package/dist/heart/providers/openai-codex-token.js +349 -0
- package/dist/heart/providers/openai-codex.js +55 -40
- package/dist/heart/providers/openai-compatible.js +177 -0
- package/dist/heart/run-ledger.js +247 -0
- package/dist/heart/runtime-capability-check.js +170 -0
- package/dist/heart/runtime-credentials.js +607 -0
- package/dist/heart/runtime-cwd.js +87 -0
- package/dist/heart/sense-truth.js +22 -4
- package/dist/heart/session-activity.js +92 -24
- package/dist/heart/session-events.js +1504 -0
- package/dist/heart/session-playback-cli-main.js +5 -0
- package/dist/heart/session-playback-cli.js +36 -0
- package/dist/heart/session-playback.js +231 -0
- package/dist/heart/session-redaction-repair-cli-main.js +31 -0
- package/dist/heart/session-redaction-repair.js +552 -0
- package/dist/heart/session-stats-cli-main.js +5 -0
- package/dist/heart/session-stats.js +183 -0
- package/dist/heart/session-transcript.js +133 -0
- package/dist/heart/start-of-turn-packet.js +372 -0
- package/dist/heart/steward-policy.js +618 -0
- package/dist/heart/streaming.js +604 -207
- package/dist/heart/structured-output.js +196 -0
- package/dist/heart/sync-classification.js +176 -0
- package/dist/heart/sync.js +449 -0
- package/dist/heart/target-resolution.js +11 -7
- package/dist/heart/tempo.js +93 -0
- package/dist/heart/temporal-view.js +41 -0
- package/dist/heart/timeouts.js +101 -0
- package/dist/heart/tool-activity-callbacks.js +59 -0
- package/dist/heart/tool-approval.js +382 -0
- package/dist/heart/tool-description.js +155 -0
- package/dist/heart/tool-friction.js +55 -0
- package/dist/heart/tool-loop.js +200 -0
- package/dist/heart/turn-context.js +482 -0
- package/dist/heart/turn-execution-lease.js +30 -0
- package/dist/heart/{daemon → versioning}/ouro-bot-global-installer.js +6 -5
- package/dist/heart/{daemon → versioning}/ouro-bot-wrapper.js +1 -1
- package/dist/heart/versioning/ouro-path-installer.js +432 -0
- package/dist/heart/versioning/ouro-recovery-launcher.js +76 -0
- package/dist/heart/versioning/ouro-version-manager.js +409 -0
- package/dist/heart/{daemon → versioning}/staged-restart.js +40 -8
- package/dist/heart/{daemon → versioning}/update-checker.js +6 -1
- package/dist/heart/versioning/update-hooks.js +154 -0
- package/dist/heart/versioning/version-intent.js +114 -0
- package/dist/heart/work-card.js +380 -0
- package/dist/mailbox-ui/assets/index-Cyv4mw3Y.js +1 -0
- package/dist/mailbox-ui/assets/index-Du_9G9WO.css +1 -0
- package/dist/mailbox-ui/assets/vendor-CcN1XpQ9.js +61 -0
- package/dist/mailbox-ui/index.html +16 -0
- package/dist/mailroom/attention.js +167 -0
- package/dist/mailroom/autonomy.js +209 -0
- package/dist/mailroom/blob-store.js +889 -0
- package/dist/mailroom/body-cache.js +61 -0
- package/dist/mailroom/cache-sync-cli.js +58 -0
- package/dist/mailroom/core.js +803 -0
- package/dist/mailroom/entry.js +160 -0
- package/dist/mailroom/file-store.js +568 -0
- package/dist/mailroom/hosted-cache-sync.js +384 -0
- package/dist/mailroom/mbox-import.js +393 -0
- package/dist/mailroom/migration.js +164 -0
- package/dist/mailroom/outbound.js +380 -0
- package/dist/mailroom/policy.js +263 -0
- package/dist/mailroom/reader.js +292 -0
- package/dist/mailroom/search-cache.js +513 -0
- package/dist/mailroom/search-relevance.js +319 -0
- package/dist/mailroom/smtp-ingress.js +176 -0
- package/dist/mailroom/source-state.js +176 -0
- package/dist/mailroom/thread.js +109 -0
- package/dist/mailroom/travel-extract.js +89 -0
- package/dist/mind/bundle-manifest.js +24 -2
- package/dist/mind/context.js +309 -111
- package/dist/mind/desk-section.js +362 -0
- package/dist/mind/diary-integrity.js +60 -0
- package/dist/mind/{memory.js → diary.js} +68 -77
- package/dist/mind/embedding-provider.js +60 -0
- package/dist/mind/file-state.js +179 -0
- package/dist/mind/{associative-recall.js → note-search.js} +47 -58
- package/dist/mind/obligation-steering.js +199 -3
- package/dist/mind/pending.js +36 -10
- package/dist/mind/prompt-budget.js +479 -0
- package/dist/mind/prompt-refresh.js +3 -2
- package/dist/mind/prompt.js +1050 -189
- package/dist/mind/provenance-trust.js +26 -0
- package/dist/mind/record-paths.js +312 -0
- package/dist/mind/scrutiny.js +173 -0
- package/dist/mind/session-transaction.js +564 -0
- package/dist/nerves/cli-logging.js +7 -1
- package/dist/nerves/coverage/audit-rules.js +15 -6
- package/dist/nerves/coverage/audit.js +28 -2
- package/dist/nerves/coverage/cli.js +1 -1
- package/dist/nerves/coverage/contract.js +5 -5
- package/dist/nerves/coverage/file-completeness.js +139 -5
- package/dist/nerves/coverage/run-artifacts.js +55 -35
- package/dist/nerves/coverage/source-scanner.js +4 -4
- package/dist/nerves/event-buffer.js +111 -0
- package/dist/nerves/index.js +293 -9
- package/dist/nerves/observation.js +20 -0
- package/dist/nerves/redact.js +79 -0
- package/dist/nerves/review/cli-main.js +5 -0
- package/dist/nerves/review/cli.js +156 -0
- package/dist/nerves/review/core.js +159 -0
- package/dist/nerves/runtime.js +17 -1
- package/dist/repertoire/ado-client.js +15 -56
- package/dist/repertoire/ado-semantic.js +16 -10
- package/dist/repertoire/api-client.js +97 -0
- package/dist/repertoire/bitwarden-store.js +1045 -0
- package/dist/repertoire/bundle-templates.js +71 -0
- package/dist/repertoire/bw-installer.js +180 -0
- package/dist/repertoire/coding/codex-jsonl.js +64 -0
- package/dist/repertoire/coding/context-pack.js +331 -0
- package/dist/repertoire/coding/feedback.js +174 -32
- package/dist/repertoire/coding/index.js +15 -3
- package/dist/repertoire/coding/manager.js +159 -10
- package/dist/repertoire/coding/spawner.js +58 -11
- package/dist/repertoire/coding/tools.js +221 -15
- package/dist/repertoire/coding/workbench-client.js +272 -0
- package/dist/repertoire/coding/workbench-manager.js +346 -0
- package/dist/repertoire/commerce-errors.js +109 -0
- package/dist/repertoire/commerce-self-test.js +156 -0
- package/dist/repertoire/credential-access.js +178 -0
- package/dist/repertoire/desk/classifier.js +362 -0
- package/dist/repertoire/duffel-client.js +185 -0
- package/dist/repertoire/github-client.js +14 -55
- package/dist/repertoire/graph-client.js +11 -52
- package/dist/repertoire/guardrails.js +811 -18
- package/dist/repertoire/mcp-client.js +44 -3
- package/dist/repertoire/mcp-manager.js +427 -139
- package/dist/repertoire/mcp-tools.js +124 -0
- package/dist/repertoire/plugin-mcp.js +175 -0
- package/dist/repertoire/plugins.js +253 -0
- package/dist/repertoire/relationship-authorization.js +322 -0
- package/dist/repertoire/shell-sessions.js +134 -0
- package/dist/repertoire/skills.js +48 -4
- package/dist/repertoire/stripe-client.js +131 -0
- package/dist/repertoire/tool-arguments.js +170 -0
- package/dist/repertoire/tool-results.js +29 -0
- package/dist/repertoire/tools-a2a.js +699 -0
- package/dist/repertoire/tools-attachments.js +322 -0
- package/dist/repertoire/tools-awaiting.js +575 -0
- package/dist/repertoire/tools-base.js +77 -1138
- package/dist/repertoire/tools-bluebubbles.js +2 -0
- package/dist/repertoire/tools-bridge.js +144 -0
- package/dist/repertoire/tools-bundle.js +993 -0
- package/dist/repertoire/tools-commerce.js +253 -0
- package/dist/repertoire/tools-config.js +186 -0
- package/dist/repertoire/tools-continuity.js +575 -0
- package/dist/repertoire/tools-credential.js +383 -0
- package/dist/repertoire/tools-evolution.js +527 -0
- package/dist/repertoire/tools-files.js +344 -0
- package/dist/repertoire/tools-flight.js +290 -0
- package/dist/repertoire/tools-flow.js +119 -0
- package/dist/repertoire/tools-github.js +3 -8
- package/dist/repertoire/tools-habits.js +103 -0
- package/dist/repertoire/tools-mail.js +1828 -0
- package/dist/repertoire/tools-notes.js +485 -0
- package/dist/repertoire/tools-obligations.js +182 -0
- package/dist/repertoire/tools-orientation.js +31 -0
- package/dist/repertoire/tools-record.js +482 -0
- package/dist/repertoire/tools-rsvp.js +139 -0
- package/dist/repertoire/tools-runtime.js +150 -0
- package/dist/repertoire/tools-sanctuary-host.js +202 -0
- package/dist/repertoire/tools-session.js +1133 -0
- package/dist/repertoire/tools-shell.js +149 -0
- package/dist/repertoire/tools-steward-policy.js +119 -0
- package/dist/repertoire/tools-stripe.js +224 -0
- package/dist/repertoire/tools-surface.js +369 -0
- package/dist/repertoire/tools-teams.js +12 -39
- package/dist/repertoire/tools-telegram-contacts.js +37 -0
- package/dist/repertoire/tools-travel.js +125 -0
- package/dist/repertoire/tools-trip.js +982 -0
- package/dist/repertoire/tools-unraid.js +531 -0
- package/dist/repertoire/tools-user-profile.js +146 -0
- package/dist/repertoire/tools-vault.js +40 -0
- package/dist/repertoire/tools-voice.js +145 -0
- package/dist/repertoire/tools.js +494 -148
- package/dist/repertoire/travel-api-client.js +360 -0
- package/dist/repertoire/unraid-client.js +155 -0
- package/dist/repertoire/unraid-restart.js +339 -0
- package/dist/repertoire/user-profile.js +131 -0
- package/dist/repertoire/vault-setup.js +246 -0
- package/dist/repertoire/vault-unlock.js +628 -0
- package/dist/rsvp/aisleplanner-client.js +354 -0
- package/dist/rsvp/cli.js +926 -0
- package/dist/rsvp/config.js +327 -0
- package/dist/rsvp/cutover.js +433 -0
- package/dist/rsvp/diagnostics.js +232 -0
- package/dist/rsvp/diff-renderer.js +176 -0
- package/dist/rsvp/habit-policy.js +110 -0
- package/dist/rsvp/habit-stage.js +123 -0
- package/dist/rsvp/incident-bundle.js +97 -0
- package/dist/rsvp/migration.js +204 -0
- package/dist/rsvp/native-habit-runner.js +630 -0
- package/dist/rsvp/outbound-state.js +628 -0
- package/dist/rsvp/query.js +118 -0
- package/dist/rsvp/replay.js +233 -0
- package/dist/rsvp/snapshot.js +232 -0
- package/dist/rsvp/spend-ledger.js +175 -0
- package/dist/scripts/claude-code-hook.js +41 -0
- package/dist/scripts/claude-code-stop-hook.js +47 -0
- package/dist/senses/a2a-entry.js +115 -0
- package/dist/senses/acp-server.js +386 -0
- package/dist/senses/attention-queue.js +186 -0
- package/dist/senses/await-turn-message.js +58 -0
- package/dist/senses/bluebubbles/active-turns.js +216 -0
- package/dist/senses/bluebubbles/attachment-cache.js +53 -0
- package/dist/senses/bluebubbles/attachment-download.js +137 -0
- package/dist/senses/{bluebubbles-client.js → bluebubbles/client.js} +355 -40
- package/dist/senses/bluebubbles/context-packet.js +207 -0
- package/dist/senses/bluebubbles/context-smoke.js +144 -0
- package/dist/senses/bluebubbles/entry.js +90 -0
- package/dist/senses/{bluebubbles-inbound-log.js → bluebubbles/inbound-log.js} +21 -3
- package/dist/senses/bluebubbles/index.js +3841 -0
- package/dist/senses/bluebubbles/latest-turn.js +311 -0
- package/dist/senses/{bluebubbles-media.js → bluebubbles/media.js} +121 -71
- package/dist/senses/{bluebubbles-model.js → bluebubbles/model.js} +159 -28
- package/dist/senses/{bluebubbles-mutation-log.js → bluebubbles/mutation-log.js} +24 -13
- package/dist/senses/bluebubbles/outbound-state.js +308 -0
- package/dist/senses/bluebubbles/processed-log.js +133 -0
- package/dist/senses/bluebubbles/reaction-policy.js +47 -0
- package/dist/senses/bluebubbles/replay.js +284 -0
- package/dist/senses/{bluebubbles-runtime-state.js → bluebubbles/runtime-state.js} +30 -2
- package/dist/senses/bluebubbles/semantic-receipts.js +1986 -0
- package/dist/senses/{bluebubbles-session-cleanup.js → bluebubbles/session-cleanup.js} +1 -1
- package/dist/senses/bluebubbles/webhook-registration.js +228 -0
- package/dist/senses/bluebubbles-meta-guard.js +39 -0
- package/dist/senses/cli/bracketed-paste.js +82 -0
- package/dist/senses/cli/image-paste.js +287 -0
- package/dist/senses/cli/image-ref-navigation.js +75 -0
- package/dist/senses/cli/ink-app.js +156 -0
- package/dist/senses/cli/inline-diff.js +64 -0
- package/dist/senses/cli/input-keys.js +174 -0
- package/dist/senses/cli/kill-ring.js +86 -0
- package/dist/senses/cli/message-list.js +51 -0
- package/dist/senses/cli/ouro-tui.js +607 -0
- package/dist/senses/cli/spinner-imperative.js +135 -0
- package/dist/senses/cli/spinner.js +101 -0
- package/dist/senses/cli/status-line.js +60 -0
- package/dist/senses/cli/streaming-markdown.js +526 -0
- package/dist/senses/cli/tool-display.js +85 -0
- package/dist/senses/cli/tool-render.js +85 -0
- package/dist/senses/cli/tui-store.js +241 -0
- package/dist/senses/cli/virtual-list.js +35 -0
- package/dist/senses/cli-entry.js +60 -8
- package/dist/senses/cli.js +604 -320
- package/dist/senses/commands.js +66 -3
- package/dist/senses/context-packet-ledger.js +371 -0
- package/dist/senses/context-packets.js +154 -0
- package/dist/senses/habit-lifecycle-guard.js +24 -0
- package/dist/senses/habit-turn-message.js +160 -0
- package/dist/senses/ingress-evidence.js +2 -0
- package/dist/senses/mail-entry.js +66 -0
- package/dist/senses/mail.js +524 -0
- package/dist/senses/pipeline.js +1115 -204
- package/dist/senses/private-runtime-worker.js +859 -0
- package/dist/senses/private-runtime.js +1517 -0
- package/dist/senses/proactive-content-guard.js +51 -0
- package/dist/senses/root-host-approval-port.js +345 -0
- package/dist/senses/root-host-approval-runtime.js +393 -0
- package/dist/senses/sanctuary-authority-resident.js +102 -0
- package/dist/senses/sanctuary-download-credit-presentation.js +31 -0
- package/dist/senses/sanctuary-full-visibility-contract.js +302 -0
- package/dist/senses/sanctuary-grounding.js +118 -0
- package/dist/senses/sanctuary-health-acceptance-probe-entry.js +12 -0
- package/dist/senses/sanctuary-health-acceptance-probe.js +774 -0
- package/dist/senses/sanctuary-health-runner.js +100 -0
- package/dist/senses/sanctuary-health.js +408 -0
- package/dist/senses/sanctuary-install-state-contract.js +126 -0
- package/dist/senses/sanctuary-interactive-control.js +410 -0
- package/dist/senses/sanctuary-media-catalog-contract.js +297 -0
- package/dist/senses/sanctuary-media-optimization.js +514 -0
- package/dist/senses/sanctuary-runtime.js +340 -0
- package/dist/senses/sanctuary-sab.js +85 -0
- package/dist/senses/sanctuary-storage-optimization-contract.js +32 -0
- package/dist/senses/shared-turn.js +772 -0
- package/dist/senses/surface-tool.js +109 -0
- package/dist/senses/teams-entry.js +60 -8
- package/dist/senses/teams.js +519 -218
- package/dist/senses/telegram-admission.js +718 -0
- package/dist/senses/telegram-approval-runtime.js +793 -0
- package/dist/senses/telegram-attachments.js +186 -0
- package/dist/senses/telegram-audit-ledger.js +220 -0
- package/dist/senses/telegram-authority-transport.js +231 -0
- package/dist/senses/telegram-client.js +1749 -0
- package/dist/senses/telegram-effect-adapter.js +896 -0
- package/dist/senses/telegram-entry.js +83 -0
- package/dist/senses/telegram.js +2324 -0
- package/dist/senses/trust-gate.js +104 -9
- package/dist/senses/voice/audio-playback.js +237 -0
- package/dist/senses/voice/audio-routing.js +119 -0
- package/dist/senses/voice/elevenlabs.js +202 -0
- package/dist/senses/voice/floor-control.js +431 -0
- package/dist/senses/voice/floor-controller.js +115 -0
- package/dist/senses/voice/golden-path.js +116 -0
- package/dist/senses/voice/index.js +29 -0
- package/dist/senses/voice/meeting.js +113 -0
- package/dist/senses/voice/outbound.js +190 -0
- package/dist/senses/voice/phone.js +33 -0
- package/dist/senses/voice/playback.js +139 -0
- package/dist/senses/voice/realtime-eval.js +496 -0
- package/dist/senses/voice/realtime-trace.js +531 -0
- package/dist/senses/voice/transcript.js +70 -0
- package/dist/senses/voice/turn.js +191 -0
- package/dist/senses/voice/twilio-phone-runtime.js +807 -0
- package/dist/senses/voice/twilio-phone.js +5031 -0
- package/dist/senses/voice/types.js +2 -0
- package/dist/senses/voice/whisper.js +161 -0
- package/dist/senses/voice-entry.js +81 -0
- package/dist/senses/voice-realtime-eval-command.js +99 -0
- package/dist/senses/voice-realtime-eval-entry.js +21 -0
- package/dist/senses/voice-twilio-entry.js +87 -0
- package/dist/trips/core.js +138 -0
- package/dist/trips/store.js +265 -0
- package/dist/util/frontmatter.js +69 -0
- package/npm-shrinkwrap.json +8138 -0
- package/package.json +72 -13
- package/skills/agent-commerce.md +113 -0
- package/skills/browser-navigation.md +117 -0
- package/skills/commerce-setup-guide.md +116 -0
- package/skills/commerce-setup.md +84 -0
- package/skills/configure-dev-tools.md +99 -0
- package/skills/travel-planning.md +138 -0
- package/dist/heart/daemon/auth-flow.js +0 -430
- package/dist/heart/daemon/ouro-path-installer.js +0 -260
- package/dist/heart/daemon/ouro-version-manager.js +0 -171
- package/dist/heart/daemon/update-hooks.js +0 -138
- package/dist/heart/obligations.js +0 -191
- package/dist/heart/safe-workspace.js +0 -228
- package/dist/heart/session-recall.js +0 -116
- package/dist/mind/friends/channel.js +0 -84
- package/dist/mind/friends/group-context.js +0 -144
- package/dist/mind/friends/resolver.js +0 -84
- package/dist/mind/friends/store-file.js +0 -190
- package/dist/mind/friends/store.js +0 -4
- package/dist/mind/friends/tokens.js +0 -26
- package/dist/mind/friends/trust-explanation.js +0 -74
- package/dist/mind/friends/types.js +0 -29
- package/dist/repertoire/tasks/board.js +0 -134
- package/dist/repertoire/tasks/index.js +0 -224
- package/dist/repertoire/tasks/lifecycle.js +0 -80
- package/dist/repertoire/tasks/middleware.js +0 -65
- package/dist/repertoire/tasks/parser.js +0 -173
- package/dist/repertoire/tasks/scanner.js +0 -132
- package/dist/repertoire/tasks/transitions.js +0 -144
- package/dist/senses/bluebubbles-entry.js +0 -13
- package/dist/senses/bluebubbles.js +0 -1181
- package/dist/senses/debug-activity.js +0 -154
- package/dist/senses/inner-dialog-worker.js +0 -90
- package/dist/senses/inner-dialog.js +0 -525
- package/subagents/README.md +0 -7
- /package/{AdoptionSpecialist.ouro → SerpentGuide.ouro}/psyche/identities/basilisk.md +0 -0
- /package/{AdoptionSpecialist.ouro → SerpentGuide.ouro}/psyche/identities/jafar.md +0 -0
- /package/{AdoptionSpecialist.ouro → SerpentGuide.ouro}/psyche/identities/jormungandr.md +0 -0
- /package/{AdoptionSpecialist.ouro → SerpentGuide.ouro}/psyche/identities/kaa.md +0 -0
- /package/{AdoptionSpecialist.ouro → SerpentGuide.ouro}/psyche/identities/medusa.md +0 -0
- /package/{AdoptionSpecialist.ouro → SerpentGuide.ouro}/psyche/identities/monty.md +0 -0
- /package/{AdoptionSpecialist.ouro → SerpentGuide.ouro}/psyche/identities/nagini.md +0 -0
- /package/{AdoptionSpecialist.ouro → SerpentGuide.ouro}/psyche/identities/ouroboros.md +0 -0
- /package/{AdoptionSpecialist.ouro → SerpentGuide.ouro}/psyche/identities/python.md +0 -0
- /package/{AdoptionSpecialist.ouro → SerpentGuide.ouro}/psyche/identities/quetzalcoatl.md +0 -0
- /package/{AdoptionSpecialist.ouro → SerpentGuide.ouro}/psyche/identities/sir-hiss.md +0 -0
- /package/{AdoptionSpecialist.ouro → SerpentGuide.ouro}/psyche/identities/the-snake.md +0 -0
- /package/dist/{repertoire/tasks/types.js → heart/attachments/sources/adapter.js} +0 -0
- /package/dist/heart/{daemon → hatch}/hatch-animation.js +0 -0
- /package/dist/heart/{daemon → hatch}/specialist-orchestrator.js +0 -0
- /package/dist/heart/{daemon → versioning}/ouro-uti.js +0 -0
- /package/dist/heart/{daemon → versioning}/wrapper-publish-guard.js +0 -0
|
@@ -0,0 +1,2084 @@
|
|
|
1
|
+
#!/usr/local/bin/node
|
|
2
|
+
|
|
3
|
+
import { spawn, spawnSync } from "node:child_process"
|
|
4
|
+
import { createHash, createHmac, timingSafeEqual } from "node:crypto"
|
|
5
|
+
import { chmodSync, chownSync, closeSync, constants, fstatSync, fsyncSync, mkdirSync, openSync, opendirSync, readFileSync, readSync, readdirSync, realpathSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync } from "node:fs"
|
|
6
|
+
import { createServer } from "node:net"
|
|
7
|
+
import { targetProfile } from "./sanctuary-deployment-target.mjs"
|
|
8
|
+
|
|
9
|
+
const KEY_ROOT = "/boot/config/plugins/dynamix.my.servers/keys"
|
|
10
|
+
const RECOVERY_ROOT = "/mnt/user/appdata/ouro-butler/acceptance/revoked-key-proof"
|
|
11
|
+
const UNRAID_API = "/usr/local/sbin/unraid-api"
|
|
12
|
+
const DOCKER = "/usr/bin/docker"
|
|
13
|
+
let activeContainer = "ouro-butler"
|
|
14
|
+
let activeContainerId = "0".repeat(64)
|
|
15
|
+
let activeProfile = targetProfile("final")
|
|
16
|
+
const AUTOSTART_FILE = "/var/lib/docker/unraid-autostart"
|
|
17
|
+
const RUNTIME_POLICY_FILE = "/opt/ouro/container-runtime.json"
|
|
18
|
+
const PRODUCTION_RUNTIME_SOURCE = "/mnt/user/appdata/ouro-butler/runtime/.ouro-cli"
|
|
19
|
+
const PRODUCTION_BUNDLE_SOURCE = "/mnt/user/appdata/ouro-butler/agent/sanctuary.ouro"
|
|
20
|
+
const PRODUCTION_EVENT_SPOOL_SOURCE = "/boot/config/custom/ouro-events/spool"
|
|
21
|
+
const AUTHORITY_ROOT = "/mnt/user/appdata/ouro-authority"
|
|
22
|
+
const GRAPHQL_ENDPOINT = "http://127.0.0.1/graphql"
|
|
23
|
+
const BOOT_ID = "/proc/sys/kernel/random/boot_id"
|
|
24
|
+
const MDCMD = "/usr/local/sbin/mdcmd"
|
|
25
|
+
const TAILSCALE = "/usr/local/sbin/tailscale"
|
|
26
|
+
const PGREP = "/usr/bin/pgrep"
|
|
27
|
+
const REBOOT = "/sbin/reboot"
|
|
28
|
+
const TARGET_SERVER = "sanctuary-unraid"
|
|
29
|
+
const TARGET_HOST = "sanctuary"
|
|
30
|
+
const KEY_ID = /^[A-Za-z0-9._:-]+$/u
|
|
31
|
+
const KEY_NAME = /^Butler (?:RO|RW)(?: Rotation [0-9a-f]{16})?$/u
|
|
32
|
+
const PERMISSION = /^[A-Z_]+:(?:CREATE|READ|UPDATE|DELETE)_(?:ANY|OWN)$/u
|
|
33
|
+
const MAX_REQUEST = 256 * 1024
|
|
34
|
+
const IMAGE_ID = /^sha256:[0-9a-f]{64}$/u
|
|
35
|
+
const SHA256 = /^[0-9a-f]{64}$/u
|
|
36
|
+
const HEALTH_PROBE_ENTRY = "/opt/ouro/dist/senses/sanctuary-health-acceptance-probe-entry.js"
|
|
37
|
+
const ACCEPTANCE_ADAPTER = "/opt/ouro/deploy/unraid/sanctuary-acceptance-adapter.sh"
|
|
38
|
+
const HEALTH_PROBE_TERM_GRACE_MS = 5_000
|
|
39
|
+
const HEALTH_PROBE_KILL_GRACE_MS = 5_000
|
|
40
|
+
const HEALTH_PROBE_LABELS = new Set(["unit-16f-cron-fingerprint", "unit-16g-health-transition", "unit-16h-acceptance-delivery-probe"])
|
|
41
|
+
const ASYNC_RESTART_SCENARIO = Symbol("asyncRestartScenario")
|
|
42
|
+
const HEALTH_PROBE_RECEIPT_KEYS = [
|
|
43
|
+
"schemaVersion", "label", "scenarioHandleDigest", "ownerImageDigestBefore", "ownerImageDigestAfter",
|
|
44
|
+
"ownerContainerDigestBefore", "ownerContainerDigestAfter", "beforeStateDigest", "restoredStateDigest",
|
|
45
|
+
"cronFingerprintBefore", "cronFingerprintAfter", "cronRegisteredBefore", "cronRegisteredAfter",
|
|
46
|
+
"cronDegradedBefore", "cronDegradedAfter", "fixtureSequenceDigest", "clockMode", "effectiveNow", "phases",
|
|
47
|
+
"providerInvocationCount", "privateTurnCount", "deliveryCount", "workspaceAbsent", "socketAbsent",
|
|
48
|
+
"snapshotAbsent", "realCheckEquivalent", "productionRestored", "schedulerReceipt",
|
|
49
|
+
]
|
|
50
|
+
const RO_PERMISSIONS = ["ARRAY", "DASHBOARD", "DISK", "DOCKER", "INFO", "LOGS", "NOTIFICATIONS", "SHARE", "VARS"]
|
|
51
|
+
.map((resource) => `${resource}:READ_ANY`).sort()
|
|
52
|
+
let expectedImageId = ""
|
|
53
|
+
const activeHealthProbes = new Map()
|
|
54
|
+
const ownerMutationCoordinator = createOwnerMutationCoordinator()
|
|
55
|
+
const interactiveRestartDriver = createInteractiveRestartDriver()
|
|
56
|
+
const healthProbeCoordinator = {
|
|
57
|
+
start: (scenario, operation) => ownerMutationCoordinator.healthStart(scenario, operation),
|
|
58
|
+
recover: (scenario, operation) => ownerMutationCoordinator.healthRecover(scenario, operation),
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function object(value, label) {
|
|
62
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(`${label} must be an object`)
|
|
63
|
+
return value
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function text(value, label, pattern) {
|
|
67
|
+
if (typeof value !== "string" || !value || (pattern && !pattern.test(value))) throw new Error(`${label} is invalid`)
|
|
68
|
+
return value
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function exactKeys(value, expected, label) {
|
|
72
|
+
if (JSON.stringify(Object.keys(value).sort()) !== JSON.stringify([...expected].sort())) throw new Error(`${label} shape is invalid`)
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function observeRebootPreflight(dependencies = {
|
|
76
|
+
readArrayStatus: () => {
|
|
77
|
+
const result = spawnSync(MDCMD, ["status"], { cwd: "/", encoding: "utf8", timeout: 10_000, maxBuffer: 1024 * 1024, env: { PATH: "/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" }, stdio: ["ignore", "pipe", "ignore"] })
|
|
78
|
+
if (result.error || result.status !== 0) throw new Error("reboot preflight array state is unknown")
|
|
79
|
+
return result.stdout
|
|
80
|
+
},
|
|
81
|
+
readMoverStatus: () => spawnSync(PGREP, ["-x", "mover"], { cwd: "/", encoding: "utf8", timeout: 5_000, env: { PATH: "/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" }, stdio: ["ignore", "ignore", "ignore"] }),
|
|
82
|
+
mutationActive: () => healthOwnerMutationActive() || ownerMutationCoordinator.active(),
|
|
83
|
+
}) {
|
|
84
|
+
const values = Object.fromEntries(String(dependencies.readArrayStatus()).split(/\r?\n/u).filter(Boolean).map((line) => {
|
|
85
|
+
const separator = line.indexOf("=")
|
|
86
|
+
if (separator < 1) throw new Error("reboot preflight array state is unknown")
|
|
87
|
+
return [line.slice(0, separator), line.slice(separator + 1)]
|
|
88
|
+
}))
|
|
89
|
+
if (values.mdState !== "STARTED" || !/^[0-9]+$/u.test(values.mdResync ?? "")) throw new Error("reboot preflight array state is unknown")
|
|
90
|
+
const mover = dependencies.readMoverStatus()
|
|
91
|
+
if (mover.error || (mover.status !== 0 && mover.status !== 1)) throw new Error("reboot preflight mover state is unknown")
|
|
92
|
+
const state = { arrayReady: true, parityActive: Number(values.mdResync) !== 0, moverActive: mover.status === 0, mutationActive: dependencies.mutationActive() === true }
|
|
93
|
+
if (state.parityActive || state.moverActive || state.mutationActive) throw new Error("reboot preflight found an active host operation")
|
|
94
|
+
const digest = createHash("sha256").update(JSON.stringify(state)).digest("hex")
|
|
95
|
+
return { ...state, safe: true, digest }
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function readPrivateJson(file) {
|
|
99
|
+
const fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW)
|
|
100
|
+
try {
|
|
101
|
+
const metadata = fstatSync(fd)
|
|
102
|
+
if (!metadata.isFile() || metadata.uid !== 0 || (metadata.mode & 0o777) !== 0o600) throw new Error("Unraid key file metadata is invalid")
|
|
103
|
+
return object(JSON.parse(readFileSync(fd, "utf8")), "Unraid key record")
|
|
104
|
+
} finally { closeSync(fd) }
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function normalizeRecord(raw) {
|
|
108
|
+
const id = text(raw.id, "key id", KEY_ID)
|
|
109
|
+
const name = text(raw.name, "key name")
|
|
110
|
+
if (!Array.isArray(raw.permissions) || !Array.isArray(raw.roles)) throw new Error("Unraid key record is invalid")
|
|
111
|
+
const permissions = raw.permissions.map((entry) => {
|
|
112
|
+
const value = object(entry, "permission")
|
|
113
|
+
exactKeys(value, ["resource", "actions"], "permission")
|
|
114
|
+
const resource = text(value.resource, "permission resource", /^[A-Z_]+$/u)
|
|
115
|
+
if (!Array.isArray(value.actions) || value.actions.length === 0) throw new Error("permission actions are invalid")
|
|
116
|
+
const actions = value.actions.map((action) => text(action, "permission action", /^(?:CREATE|READ|UPDATE|DELETE)_(?:ANY|OWN)$/u)).sort()
|
|
117
|
+
if (new Set(actions).size !== actions.length) throw new Error("permission actions are ambiguous")
|
|
118
|
+
return { resource, actions }
|
|
119
|
+
})
|
|
120
|
+
if (!raw.roles.every((role) => typeof role === "string")) throw new Error("Unraid key roles are invalid")
|
|
121
|
+
return { id, name, permissions, roles: [...raw.roles], key: text(raw.key, "key descriptor") }
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function inventoryRecords() {
|
|
125
|
+
const root = statSync(KEY_ROOT)
|
|
126
|
+
if (!root.isDirectory() || root.uid !== 0 || (root.mode & 0o077) !== 0) throw new Error("Unraid key directory metadata is invalid")
|
|
127
|
+
const records = readdirSync(KEY_ROOT, { withFileTypes: true }).map((entry) => {
|
|
128
|
+
if (!entry.isFile() || !entry.name.endsWith(".json")) throw new Error("unexpected Unraid key directory entry")
|
|
129
|
+
return normalizeRecord(readPrivateJson(`${KEY_ROOT}/${entry.name}`))
|
|
130
|
+
})
|
|
131
|
+
if (new Set(records.map(({ id }) => id)).size !== records.length || new Set(records.map(({ name }) => name)).size !== records.length) {
|
|
132
|
+
throw new Error("Unraid key inventory is ambiguous")
|
|
133
|
+
}
|
|
134
|
+
return records.sort((left, right) => left.id.localeCompare(right.id))
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function publicRecord({ key: _key, ...record }) { return record }
|
|
138
|
+
|
|
139
|
+
function flattened(record) {
|
|
140
|
+
return record.permissions.flatMap(({ resource, actions }) => actions.map((action) => `${resource}:${action}`)).sort()
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function runUnraid(args) {
|
|
144
|
+
const result = spawnSync(UNRAID_API, args, {
|
|
145
|
+
cwd: "/", encoding: "utf8", timeout: 20_000, maxBuffer: 1024 * 1024,
|
|
146
|
+
env: { PATH: "/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" },
|
|
147
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
148
|
+
})
|
|
149
|
+
if (result.error || result.status !== 0) throw new Error("bounded Unraid API operation failed")
|
|
150
|
+
return object(JSON.parse(result.stdout), "Unraid API response")
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
function permissionObjects(values) {
|
|
154
|
+
const grouped = new Map()
|
|
155
|
+
for (const value of values) {
|
|
156
|
+
const [resource, action] = value.split(":")
|
|
157
|
+
const actions = grouped.get(resource) ?? []
|
|
158
|
+
actions.push(action)
|
|
159
|
+
grouped.set(resource, actions)
|
|
160
|
+
}
|
|
161
|
+
return [...grouped.entries()].sort(([left], [right]) => left.localeCompare(right))
|
|
162
|
+
.map(([resource, actions]) => ({ resource, actions: actions.sort() }))
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
async function createUnraidKey(name, requested, dependencies = {
|
|
166
|
+
runUnraid, inventoryRecords, persistRecovery, readRecovery: optionalRecoveryRecord,
|
|
167
|
+
persistIntent: persistCreateIntent, readIntent: optionalCreateIntent, removeIntent: removeCreateIntent,
|
|
168
|
+
fetchImpl: fetch,
|
|
169
|
+
}) {
|
|
170
|
+
const before = dependencies.inventoryRecords()
|
|
171
|
+
const beforeIds = new Set(before.map((record) => record.id))
|
|
172
|
+
const provisionalPermissions = [...requested, "API_KEY:UPDATE_ANY"].sort()
|
|
173
|
+
let recoveryId = null
|
|
174
|
+
let cliIdentity = null
|
|
175
|
+
try {
|
|
176
|
+
const occupied = before.filter((record) => record.name === name)
|
|
177
|
+
const existingIntent = dependencies.readIntent?.(name) ?? null
|
|
178
|
+
if (existingIntent === null) {
|
|
179
|
+
if (occupied.length !== 0) throw new Error("existing provisional key has no durable creation intent")
|
|
180
|
+
dependencies.persistIntent(name, provisionalPermissions)
|
|
181
|
+
} else if (existingIntent.name !== name || JSON.stringify(existingIntent.permissions) !== JSON.stringify(provisionalPermissions)) {
|
|
182
|
+
throw new Error("Unraid key creation intent is invalid")
|
|
183
|
+
}
|
|
184
|
+
let created
|
|
185
|
+
let provisional
|
|
186
|
+
let resumed = false
|
|
187
|
+
let resumedFinal = false
|
|
188
|
+
if (occupied.length === 1) {
|
|
189
|
+
provisional = occupied[0]
|
|
190
|
+
let recovery = dependencies.readRecovery?.(provisional.id) ?? null
|
|
191
|
+
const provisionalExact = JSON.stringify(flattened(provisional)) === JSON.stringify(provisionalPermissions)
|
|
192
|
+
&& JSON.stringify(provisional.roles) === JSON.stringify(["GUEST"])
|
|
193
|
+
if (recovery === null && provisionalExact) {
|
|
194
|
+
dependencies.persistRecovery(provisional)
|
|
195
|
+
recovery = provisional
|
|
196
|
+
}
|
|
197
|
+
if (!recovery || recovery.id !== provisional.id || recovery.name !== provisional.name || recovery.key !== provisional.key
|
|
198
|
+
|| JSON.stringify(flattened(recovery)) !== JSON.stringify(provisionalPermissions)
|
|
199
|
+
|| JSON.stringify(recovery.roles) !== JSON.stringify(["GUEST"])
|
|
200
|
+
|| !(provisionalExact
|
|
201
|
+
|| (JSON.stringify(flattened(provisional)) === JSON.stringify(requested) && provisional.roles.length === 0))) {
|
|
202
|
+
throw new Error("existing provisional key is not exact owned recovery")
|
|
203
|
+
}
|
|
204
|
+
resumedFinal = provisional.roles.length === 0
|
|
205
|
+
created = { id: provisional.id, key: provisional.key, name: provisional.name }
|
|
206
|
+
resumed = true
|
|
207
|
+
} else if (occupied.length === 0) {
|
|
208
|
+
created = object(dependencies.runUnraid(["apikey", "--name", name, "--create", "--roles", "GUEST", "--permissions", provisionalPermissions.join(","), "--json"]), "created Unraid key")
|
|
209
|
+
exactKeys(created, ["id", "key", "name"], "created Unraid key")
|
|
210
|
+
cliIdentity = { id: text(created.id, "created key id", KEY_ID), key: text(created.key, "created key descriptor"), name: created.name }
|
|
211
|
+
} else {
|
|
212
|
+
throw new Error("created provisional key name is ambiguous")
|
|
213
|
+
}
|
|
214
|
+
const id = text(created.id, "created key id", KEY_ID)
|
|
215
|
+
const key = text(created.key, "created key descriptor")
|
|
216
|
+
if (created.name !== name || (!resumed && beforeIds.has(id))) throw new Error("created key identity is invalid")
|
|
217
|
+
const matches = resumed ? [provisional] : dependencies.inventoryRecords().filter((record) => record.id === id)
|
|
218
|
+
if (matches.length !== 1 || matches[0].name !== name || matches[0].key !== key
|
|
219
|
+
|| (!resumedFinal && (JSON.stringify(flattened(matches[0])) !== JSON.stringify(provisionalPermissions)
|
|
220
|
+
|| JSON.stringify(matches[0].roles) !== JSON.stringify(["GUEST"])))) {
|
|
221
|
+
throw new Error("created provisional key record does not match the exact CLI result")
|
|
222
|
+
}
|
|
223
|
+
if (!resumed) dependencies.persistRecovery(matches[0])
|
|
224
|
+
recoveryId = id
|
|
225
|
+
const headers = { "content-type": "application/json", "x-api-key": key }
|
|
226
|
+
if (!resumedFinal) try {
|
|
227
|
+
const updateResponse = await dependencies.fetchImpl(GRAPHQL_ENDPOINT, {
|
|
228
|
+
method: "POST", headers,
|
|
229
|
+
body: JSON.stringify({
|
|
230
|
+
query: "mutation AcceptanceApiKeyDowngrade($input: UpdateApiKeyInput!) { apiKey { update(input: $input) { id name roles permissions { resource actions } } } }",
|
|
231
|
+
variables: { input: { id, roles: [], permissions: permissionObjects(requested) } },
|
|
232
|
+
}),
|
|
233
|
+
signal: AbortSignal.timeout(10_000),
|
|
234
|
+
})
|
|
235
|
+
const updateEnvelope = object(await updateResponse.json(), "Unraid key downgrade response")
|
|
236
|
+
if (!updateResponse.ok || updateEnvelope.errors || !updateEnvelope.data) throw new Error("Unraid key downgrade was rejected")
|
|
237
|
+
} catch {
|
|
238
|
+
// A lost response is accepted only when disk and live-auth attestation below prove convergence.
|
|
239
|
+
}
|
|
240
|
+
const finalized = dependencies.inventoryRecords().filter((record) => record.id === id)
|
|
241
|
+
if (finalized.length !== 1 || finalized[0].name !== name || finalized[0].key !== key
|
|
242
|
+
|| JSON.stringify(flattened(finalized[0])) !== JSON.stringify(requested) || finalized[0].roles.length !== 0) {
|
|
243
|
+
throw new Error("created key did not converge to its exact final scope")
|
|
244
|
+
}
|
|
245
|
+
const probeResponse = await dependencies.fetchImpl(GRAPHQL_ENDPOINT, {
|
|
246
|
+
method: "POST", headers,
|
|
247
|
+
body: JSON.stringify({ query: "query AcceptanceAuthProbe { info { os { hostname } } }", variables: {} }),
|
|
248
|
+
signal: AbortSignal.timeout(10_000),
|
|
249
|
+
})
|
|
250
|
+
const probeEnvelope = object(await probeResponse.json(), "Unraid key readiness response")
|
|
251
|
+
if (!probeResponse.ok || probeEnvelope.errors || !probeEnvelope.data) throw new Error("created key readiness probe failed")
|
|
252
|
+
return finalized[0]
|
|
253
|
+
} catch (error) {
|
|
254
|
+
const candidates = dependencies.inventoryRecords().filter((record) => record.name === name
|
|
255
|
+
&& !beforeIds.has(record.id)
|
|
256
|
+
&& JSON.stringify(flattened(record)) === JSON.stringify(provisionalPermissions)
|
|
257
|
+
&& JSON.stringify(record.roles) === JSON.stringify(["GUEST"])
|
|
258
|
+
&& (cliIdentity === null || (record.id === cliIdentity.id && record.key === cliIdentity.key && record.name === cliIdentity.name)))
|
|
259
|
+
if (candidates.length === 1 && candidates[0].id !== recoveryId) dependencies.persistRecovery(candidates[0])
|
|
260
|
+
throw error
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
function autostartFileExact() {
|
|
265
|
+
const fd = openSync(AUTOSTART_FILE, constants.O_RDONLY | constants.O_NOFOLLOW)
|
|
266
|
+
let content
|
|
267
|
+
try {
|
|
268
|
+
if (!fstatSync(fd).isFile()) throw new Error("Unraid autostart state is invalid")
|
|
269
|
+
content = readFileSync(fd, "utf8")
|
|
270
|
+
} finally { closeSync(fd) }
|
|
271
|
+
const counts = { production: 0, staging: 0, rollback: 0, legacy: 0 }
|
|
272
|
+
for (const line of content.split(/\r?\n/u)) {
|
|
273
|
+
const name = line.trim().split(/\s+/u)[0]
|
|
274
|
+
if (name === "ouro-butler") counts.production += 1
|
|
275
|
+
else if (name === "ouro-butler-staging") counts.staging += 1
|
|
276
|
+
else if (name === "ouro-butler-rollback") counts.rollback += 1
|
|
277
|
+
else if (name === "ouro-butler-legacy-evidence") counts.legacy += 1
|
|
278
|
+
}
|
|
279
|
+
return activeProfile.name === "staging"
|
|
280
|
+
? counts.production === 0 && counts.staging === 1 && counts.rollback === 0 && counts.legacy === 0
|
|
281
|
+
: counts.production === 1 && counts.staging === 0 && counts.rollback === 0 && counts.legacy === 0
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
function optionalStoppedContainerExact(name, containerId, run = spawnSync) {
|
|
285
|
+
if (name !== "ouro-butler-rollback" || !SHA256.test(containerId)) return false
|
|
286
|
+
const template = '{"containerId":{{json .Id}},"name":{{json .Name}},"running":{{json .State.Running}}}'
|
|
287
|
+
const result = run(DOCKER, ["inspect", "--format", template, containerId], {
|
|
288
|
+
cwd: "/", encoding: "utf8", timeout: 20_000, maxBuffer: 64 * 1024,
|
|
289
|
+
env: { PATH: "/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" }, stdio: ["ignore", "pipe", "ignore"],
|
|
290
|
+
})
|
|
291
|
+
if (result.error || result.status !== 0) return false
|
|
292
|
+
try {
|
|
293
|
+
const value = object(JSON.parse(result.stdout ?? ""), "optional rollback inspection")
|
|
294
|
+
return value.containerId === containerId && value.name === `/${name}` && value.running === false
|
|
295
|
+
} catch { return false }
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
async function queryGraphqlAutostart(records = inventoryRecords(), fetchImpl = fetch, expectedContainerId = activeContainerId, profile = activeProfile, inspectOptionalStopped = optionalStoppedContainerExact) {
|
|
299
|
+
text(expectedContainerId, "attested target container id", SHA256)
|
|
300
|
+
const matches = records.filter((record) => record.name === "Butler RO" && record.roles.length === 0
|
|
301
|
+
&& JSON.stringify(flattened(record)) === JSON.stringify(RO_PERMISSIONS))
|
|
302
|
+
if (matches.length !== 1) throw new Error("canonical read-only Unraid key is absent or ambiguous")
|
|
303
|
+
const descriptor = text(matches[0].key, "canonical read-only key descriptor")
|
|
304
|
+
const response = await fetchImpl(GRAPHQL_ENDPOINT, {
|
|
305
|
+
method: "POST",
|
|
306
|
+
headers: { "content-type": "application/json", "x-api-key": descriptor },
|
|
307
|
+
body: JSON.stringify({ query: "query AcceptanceContainerTopology { vars { id } docker { containers(skipCache: true) { id names autoStart } } }", variables: {} }),
|
|
308
|
+
signal: AbortSignal.timeout(10_000),
|
|
309
|
+
})
|
|
310
|
+
if (!response.ok) throw new Error("Unraid container topology query failed")
|
|
311
|
+
const envelope = object(await response.json(), "Unraid container topology response")
|
|
312
|
+
if (envelope.errors || !envelope.data) throw new Error("Unraid container topology query was rejected")
|
|
313
|
+
const data = object(envelope.data, "Unraid topology data")
|
|
314
|
+
const serverIdentity = /^([0-9a-f]{64}):vars$/u.exec(object(data.vars, "Unraid topology vars").id)
|
|
315
|
+
if (!serverIdentity) throw new Error("Unraid server identity is invalid")
|
|
316
|
+
const containers = object(data.docker, "Unraid topology docker").containers
|
|
317
|
+
if (!Array.isArray(containers)) throw new Error("Unraid container topology is invalid")
|
|
318
|
+
const canonicalNames = new Set(["ouro-butler", "ouro-butler-staging", "ouro-butler-rollback"])
|
|
319
|
+
const topology = new Map()
|
|
320
|
+
for (const raw of containers) {
|
|
321
|
+
const container = object(raw, "Unraid topology container")
|
|
322
|
+
if (!Array.isArray(container.names) || container.names.some((name) => typeof name !== "string")) throw new Error("Unraid container topology identity is invalid")
|
|
323
|
+
const canonical = container.names.map((name) => name.replace(/^\//u, "")).filter((name) => canonicalNames.has(name))
|
|
324
|
+
if (canonical.length > 1) return false
|
|
325
|
+
if (canonical.length === 0) continue
|
|
326
|
+
const identity = typeof container.id === "string" ? /^([0-9a-f]{64}):([0-9a-f]{64})$/u.exec(container.id) : null
|
|
327
|
+
if (!identity || identity[1] !== serverIdentity[1] || typeof container.autoStart !== "boolean" || topology.has(canonical[0])) return false
|
|
328
|
+
topology.set(canonical[0], { containerId: identity[2], autoStart: container.autoStart })
|
|
329
|
+
}
|
|
330
|
+
const requiredNames = new Set([profile.containerName, ...profile.requiredStopped])
|
|
331
|
+
const allowedNames = new Set([...requiredNames, ...profile.optionalStopped])
|
|
332
|
+
if ([...requiredNames].some((name) => !topology.has(name)) || [...topology.keys()].some((name) => !allowedNames.has(name))
|
|
333
|
+
|| profile.forbidden.some((name) => topology.has(name))) return false
|
|
334
|
+
const target = topology.get(profile.containerName)
|
|
335
|
+
if (!target || target.containerId !== expectedContainerId || target.autoStart !== true) return false
|
|
336
|
+
return [...profile.requiredStopped, ...profile.optionalStopped].every((name) => {
|
|
337
|
+
const stopped = topology.get(name)
|
|
338
|
+
return !stopped || (stopped.autoStart === false && inspectOptionalStopped(name, stopped.containerId))
|
|
339
|
+
})
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
function updaterDisabled(expectedImage) {
|
|
343
|
+
const result = spawnSync(DOCKER, ["run", "--rm", "--pull=never", "--network", "none", "--entrypoint", "/bin/cat", expectedImage, RUNTIME_POLICY_FILE], {
|
|
344
|
+
cwd: "/", encoding: "utf8", timeout: 20_000, maxBuffer: 64 * 1024,
|
|
345
|
+
env: { PATH: "/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" }, stdio: ["ignore", "pipe", "ignore"],
|
|
346
|
+
})
|
|
347
|
+
if (result.error || result.status !== 0) throw new Error("container runtime policy read failed")
|
|
348
|
+
const policy = object(JSON.parse(result.stdout), "container runtime policy")
|
|
349
|
+
exactKeys(policy, ["scheduler", "updates"], "container runtime policy")
|
|
350
|
+
return policy.scheduler === "supercronic" && policy.updates === "disabled"
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
function parseVaultStatus(output, succeeded) {
|
|
354
|
+
const lines = output.split(/\r?\n/u)
|
|
355
|
+
const runtimeLine = lines.find((line) => line.startsWith("runtime credentials: "))
|
|
356
|
+
const runtimeMatch = /^runtime credentials: (.+) \([^)]+\)$/u.exec(runtimeLine ?? "")
|
|
357
|
+
const runtimeFields = runtimeMatch ? runtimeMatch[1].split(", ") : []
|
|
358
|
+
const providerReady = (provider) => {
|
|
359
|
+
const line = lines.find((candidate) => candidate.startsWith(` ${provider}: `))
|
|
360
|
+
const match = /^ [a-z0-9-]+: credential fields (.+), config fields (.+)$/u.exec(line ?? "")
|
|
361
|
+
if (!match) return false
|
|
362
|
+
return match[1].split(", ").includes("apiKey") && match[2].split(", ").includes("baseUrl")
|
|
363
|
+
}
|
|
364
|
+
const unlocked = succeeded && /^local unlock: available$/mu.test(output)
|
|
365
|
+
&& runtimeMatch !== null && !runtimeFields.includes("telegramBotToken")
|
|
366
|
+
&& providerReady("minimax")
|
|
367
|
+
return { vaultUnlocked: unlocked, manualAuthRequired: !unlocked }
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
function vaultStatus(running, healthy) {
|
|
371
|
+
if (!running || !healthy) return { vaultUnlocked: false, manualAuthRequired: true }
|
|
372
|
+
const result = spawnSync(DOCKER, ["exec", activeContainerId, "node", "/opt/ouro/dist/heart/daemon/ouro-entry.js", "vault", "status", "--agent", "sanctuary", "--store", "plaintext-file"], {
|
|
373
|
+
cwd: "/", encoding: "utf8", timeout: 30_000, maxBuffer: 1024 * 1024,
|
|
374
|
+
env: { PATH: "/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" }, stdio: ["ignore", "pipe", "ignore"],
|
|
375
|
+
})
|
|
376
|
+
return parseVaultStatus(result.stdout ?? "", !result.error && result.status === 0)
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
function recoveryMilestones(running, healthy) {
|
|
380
|
+
const array = spawnSync(MDCMD, ["status"], { cwd: "/", encoding: "utf8", timeout: 10_000, stdio: ["ignore", "pipe", "ignore"] })
|
|
381
|
+
const tailscale = spawnSync(TAILSCALE, ["status", "--json"], { cwd: "/", encoding: "utf8", timeout: 10_000, stdio: ["ignore", "pipe", "ignore"] })
|
|
382
|
+
const ssh = spawnSync(PGREP, ["-x", "sshd"], { cwd: "/", encoding: "utf8", timeout: 10_000, stdio: ["ignore", "pipe", "ignore"] })
|
|
383
|
+
let tailscaleReady = false
|
|
384
|
+
try {
|
|
385
|
+
const status = object(JSON.parse(tailscale.stdout), "Tailscale status")
|
|
386
|
+
tailscaleReady = !tailscale.error && tailscale.status === 0 && status.BackendState === "Running"
|
|
387
|
+
} catch { tailscaleReady = false }
|
|
388
|
+
return {
|
|
389
|
+
hostReady: /^[A-Za-z0-9-]{4,128}$/u.test(readFileSync(BOOT_ID, "utf8").trim()),
|
|
390
|
+
arrayReady: !array.error && array.status === 0 && /^mdState=STARTED$/mu.test(array.stdout),
|
|
391
|
+
dockerReady: true,
|
|
392
|
+
butlerReady: running && healthy,
|
|
393
|
+
tailscaleReady,
|
|
394
|
+
sshReady: !ssh.error && ssh.status === 0 && ssh.stdout.trim().length > 0,
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
function readBoundedProcStatus(statusPath) {
|
|
399
|
+
const fd = openSync(statusPath, constants.O_RDONLY | constants.O_NOFOLLOW)
|
|
400
|
+
try {
|
|
401
|
+
const metadata = fstatSync(fd)
|
|
402
|
+
if (!metadata.isFile()) throw new Error("container PID1 status is invalid")
|
|
403
|
+
const buffer = Buffer.alloc(128 * 1024 + 1)
|
|
404
|
+
const length = readSync(fd, buffer, 0, buffer.length, 0)
|
|
405
|
+
if (length > 128 * 1024) throw new Error("container PID1 status exceeds its bound")
|
|
406
|
+
return buffer.subarray(0, length).toString("utf8")
|
|
407
|
+
} finally { closeSync(fd) }
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
function readBoundedProcIdentityFile(filePath) {
|
|
411
|
+
const fd = openSync(filePath, constants.O_RDONLY | constants.O_NOFOLLOW)
|
|
412
|
+
try {
|
|
413
|
+
const metadata = fstatSync(fd, { bigint: true })
|
|
414
|
+
if (!metadata.isFile()) throw new Error("container PID1 process file is invalid")
|
|
415
|
+
const buffer = Buffer.alloc(128 * 1024 + 1)
|
|
416
|
+
const length = readSync(fd, buffer, 0, buffer.length, 0)
|
|
417
|
+
if (length > 128 * 1024) throw new Error("container PID1 process file exceeds its bound")
|
|
418
|
+
return { content: buffer.subarray(0, length).toString("utf8"), inode: metadata.ino.toString() }
|
|
419
|
+
} finally { closeSync(fd) }
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
function parseProcStartTime(raw) {
|
|
423
|
+
if (typeof raw !== "string" || Buffer.byteLength(raw, "utf8") > 128 * 1024) throw new Error("container PID1 stat is invalid")
|
|
424
|
+
const close = raw.lastIndexOf(")")
|
|
425
|
+
if (close < 3) throw new Error("container PID1 stat is invalid")
|
|
426
|
+
const fields = raw.slice(close + 1).trim().split(/\s+/u)
|
|
427
|
+
const startTime = fields[19]
|
|
428
|
+
if (!startTime || !/^[0-9]+$/u.test(startTime)) throw new Error("container PID1 stat start time is invalid")
|
|
429
|
+
return startTime
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
function liveContainerProcessIdentity(pid, dependencies = { readFile: readBoundedProcIdentityFile }) {
|
|
433
|
+
if (!Number.isSafeInteger(pid) || pid <= 0 || pid > 4_194_304) throw new Error("container PID1 is invalid")
|
|
434
|
+
const status = dependencies.readFile(`/proc/${pid}/status`)
|
|
435
|
+
const stat = dependencies.readFile(`/proc/${pid}/stat`)
|
|
436
|
+
const user = liveContainerProcessUser(pid, { readFile: () => status.content })
|
|
437
|
+
return { user, processStartTime: parseProcStartTime(stat.content), processInode: `${status.inode}:${stat.inode}` }
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
function liveContainerProcessUser(pid, dependencies = { readFile: readBoundedProcStatus }) {
|
|
441
|
+
if (!Number.isSafeInteger(pid) || pid <= 0 || pid > 4_194_304) throw new Error("container PID1 is invalid")
|
|
442
|
+
const statusPath = `/proc/${pid}/status`
|
|
443
|
+
const status = dependencies.readFile(statusPath, "utf8")
|
|
444
|
+
if (typeof status !== "string" || Buffer.byteLength(status, "utf8") > 128 * 1024) throw new Error("container PID1 status is invalid")
|
|
445
|
+
const lines = status.split(/\r?\n/u)
|
|
446
|
+
const parseEffective = (key) => {
|
|
447
|
+
const matches = lines.filter((line) => line.startsWith(`${key}:`))
|
|
448
|
+
if (matches.length !== 1) throw new Error(`container PID1 ${key} status is invalid`)
|
|
449
|
+
const match = new RegExp(`^${key}:\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)$`, "u").exec(matches[0])
|
|
450
|
+
if (!match) throw new Error(`container PID1 ${key} status is invalid`)
|
|
451
|
+
return Number(match[2])
|
|
452
|
+
}
|
|
453
|
+
const uid = parseEffective("Uid")
|
|
454
|
+
const gid = parseEffective("Gid")
|
|
455
|
+
if (uid !== 10001 || gid !== 10001) throw new Error("container PID1 effective identity is invalid")
|
|
456
|
+
return `${uid}:${gid}`
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
function assertStableContainerProcess(before, after) {
|
|
460
|
+
const stableFields = ["containerId", "imageId", "pid", "restartCount", "startedAt", "health", "processStartTime", "processInode"]
|
|
461
|
+
if (before.running !== true || after.running !== true || stableFields.some((field) => before[field] !== after[field])) throw new Error("production container PID1 changed during attestation")
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
function productionProcessBindingDigest(value) {
|
|
465
|
+
const generation = {
|
|
466
|
+
containerId: text(value.containerId, "process binding container id", SHA256),
|
|
467
|
+
imageId: text(value.imageId, "process binding image id", IMAGE_ID),
|
|
468
|
+
pid: value.pid,
|
|
469
|
+
restartCount: value.restartCount,
|
|
470
|
+
startedAt: text(value.startedAt, "process binding started at"),
|
|
471
|
+
processStartTime: text(value.processStartTime, "process binding start time", /^[0-9]+$/u),
|
|
472
|
+
processInode: text(value.processInode, "process binding inode", /^[0-9]+:[0-9]+$/u),
|
|
473
|
+
}
|
|
474
|
+
if (!Number.isSafeInteger(generation.pid) || generation.pid <= 0 || !Number.isSafeInteger(generation.restartCount) || generation.restartCount < 0) {
|
|
475
|
+
throw new Error("production process binding generation is invalid")
|
|
476
|
+
}
|
|
477
|
+
return createHash("sha256").update(JSON.stringify(generation)).digest("hex")
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
async function containerSnapshot(expectedImage) {
|
|
481
|
+
text(expectedImage, "expected image id", IMAGE_ID)
|
|
482
|
+
const template = '{"containerId":{{json .Id}},"imageId":{{json .Image}},"running":{{json .State.Running}},"pid":{{json .State.Pid}},"startedAt":{{json .State.StartedAt}},"health":{{json .State.Health.Status}},"user":{{json .Config.User}},"readOnlyRoot":{{json .HostConfig.ReadonlyRootfs}},"mounts":{{json .Mounts}},"ports":{{json .NetworkSettings.Ports}},"networkMode":{{json .HostConfig.NetworkMode}},"restartPolicy":{{json .HostConfig.RestartPolicy.Name}},"restartCount":{{json .RestartCount}},"privileged":{{json .HostConfig.Privileged}},"capAdd":{{json .HostConfig.CapAdd}},"capDrop":{{json .HostConfig.CapDrop}},"securityOpt":{{json .HostConfig.SecurityOpt}}}'
|
|
483
|
+
const result = spawnSync(DOCKER, ["inspect", "--format", template, activeContainerId], {
|
|
484
|
+
cwd: "/", encoding: "utf8", timeout: 20_000, maxBuffer: 1024 * 1024,
|
|
485
|
+
env: { PATH: "/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" },
|
|
486
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
487
|
+
})
|
|
488
|
+
if (result.error || result.status !== 0) throw new Error("bounded production container inspection failed")
|
|
489
|
+
const value = object(JSON.parse(result.stdout), "production container inspection")
|
|
490
|
+
if (value.containerId !== activeContainerId || value.imageId !== expectedImage
|
|
491
|
+
|| !Number.isSafeInteger(value.pid) || value.pid <= 0 || value.pid > 4_194_304
|
|
492
|
+
|| !Number.isSafeInteger(value.restartCount) || value.restartCount < 0
|
|
493
|
+
|| typeof value.startedAt !== "string" || !/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[^\s]{1,64}Z$/u.test(value.startedAt)) {
|
|
494
|
+
throw new Error("production container identity is invalid")
|
|
495
|
+
}
|
|
496
|
+
if (typeof value.readOnlyRoot !== "boolean") throw new Error("production container root mode is invalid")
|
|
497
|
+
const ports = object(value.ports ?? {}, "published ports")
|
|
498
|
+
const publishedPortCount = Object.values(ports).reduce((count, bindings) => count + (Array.isArray(bindings) ? bindings.length : 0), 0)
|
|
499
|
+
const mounts = Array.isArray(value.mounts) ? value.mounts.map((raw) => {
|
|
500
|
+
const mount = object(raw, "container mount")
|
|
501
|
+
return { destination: mount.Destination, source: mount.Source, mode: mount.Mode, propagation: mount.Propagation, rw: mount.RW, type: mount.Type }
|
|
502
|
+
}).sort((left, right) => String(left.destination).localeCompare(String(right.destination))) : []
|
|
503
|
+
const expectedMounts = [
|
|
504
|
+
{ destination: "/home/ouro/.ouro-cli", source: PRODUCTION_RUNTIME_SOURCE, propagation: "rprivate", rw: true, type: "bind" },
|
|
505
|
+
{ destination: "/home/ouro/AgentBundles/sanctuary.ouro", source: PRODUCTION_BUNDLE_SOURCE, propagation: "rprivate", rw: true, type: "bind" },
|
|
506
|
+
{ destination: "/run/ouro-events", source: PRODUCTION_EVENT_SPOOL_SOURCE, propagation: "rprivate", rw: false, type: "bind" },
|
|
507
|
+
{ destination: "/run/ouro-authority", source: "/run/ouro-authority", propagation: "rprivate", rw: false, type: "bind" },
|
|
508
|
+
]
|
|
509
|
+
const mountsExact = mounts.length === expectedMounts.length && expectedMounts.every((expected) => mounts.some((mount) => mount.destination === expected.destination && mount.source === expected.source && mount.propagation === expected.propagation && mount.rw === expected.rw && mount.type === expected.type))
|
|
510
|
+
const securityExact = value.privileged === false && (value.capAdd === null || (Array.isArray(value.capAdd) && value.capAdd.length === 0))
|
|
511
|
+
&& (value.capDrop === null || (Array.isArray(value.capDrop) && value.capDrop.length === 0))
|
|
512
|
+
&& (value.securityOpt === null || (Array.isArray(value.securityOpt) && value.securityOpt.length === 0))
|
|
513
|
+
const running = value.running === true
|
|
514
|
+
if (!running) throw new Error("production container is not running")
|
|
515
|
+
const configuredUser = text(value.user, "container user")
|
|
516
|
+
if (configuredUser !== "10001:10001") throw new Error("production container configured identity is invalid")
|
|
517
|
+
const processBefore = liveContainerProcessIdentity(value.pid)
|
|
518
|
+
const reboundResult = spawnSync(DOCKER, ["inspect", "--format", template, activeContainerId], {
|
|
519
|
+
cwd: "/", encoding: "utf8", timeout: 20_000, maxBuffer: 1024 * 1024,
|
|
520
|
+
env: { PATH: "/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" }, stdio: ["ignore", "pipe", "ignore"],
|
|
521
|
+
})
|
|
522
|
+
if (reboundResult.error || reboundResult.status !== 0) throw new Error("bounded production container rebound inspection failed")
|
|
523
|
+
const rebound = object(JSON.parse(reboundResult.stdout ?? ""), "production container rebound inspection")
|
|
524
|
+
const processAfter = liveContainerProcessIdentity(rebound.pid)
|
|
525
|
+
assertStableContainerProcess({ ...value, ...processBefore }, { ...rebound, ...processAfter })
|
|
526
|
+
const liveProcessUser = processAfter.user
|
|
527
|
+
const processBindingDigest = productionProcessBindingDigest({ ...rebound, ...processAfter })
|
|
528
|
+
const healthy = value.health === "healthy"
|
|
529
|
+
const vault = vaultStatus(running, healthy)
|
|
530
|
+
const milestones = recoveryMilestones(running, healthy)
|
|
531
|
+
return {
|
|
532
|
+
schemaVersion: 1,
|
|
533
|
+
containerId: text(value.containerId, "container id", /^[0-9a-f]{64}$/u),
|
|
534
|
+
imageId: expectedImage,
|
|
535
|
+
running,
|
|
536
|
+
health: text(value.health, "container health", /^(?:healthy|starting|unhealthy|missing)$/u),
|
|
537
|
+
user: configuredUser,
|
|
538
|
+
liveProcessUser,
|
|
539
|
+
processBindingDigest,
|
|
540
|
+
readOnlyRoot: value.readOnlyRoot,
|
|
541
|
+
mountCount: mounts.length,
|
|
542
|
+
mountsDigest: createHash("sha256").update(JSON.stringify(mounts)).digest("hex"),
|
|
543
|
+
mountsExact,
|
|
544
|
+
publishedPortCount,
|
|
545
|
+
networkMode: text(value.networkMode, "container network mode"),
|
|
546
|
+
securityExact,
|
|
547
|
+
writableKeyExposure: mounts.some((mount) => String(mount.destination).endsWith("docker.sock") || mount.destination === KEY_ROOT),
|
|
548
|
+
restartPolicy: text(value.restartPolicy, "container restart policy"),
|
|
549
|
+
restartCount: value.restartCount,
|
|
550
|
+
autostartExact: autostartFileExact() && await queryGraphqlAutostart(),
|
|
551
|
+
updaterDisabled: updaterDisabled(expectedImage),
|
|
552
|
+
...vault,
|
|
553
|
+
recoveryMilestones: milestones,
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
function inspectRebootOwner(containerId = activeContainerId) {
|
|
558
|
+
const template = '{"containerId":{{json .Id}},"name":{{json .Name}},"imageId":{{json .Image}},"running":{{json .State.Running}},"pid":{{json .State.Pid}},"startedAt":{{json .State.StartedAt}},"health":{{json .State.Health.Status}},"restartCount":{{json .RestartCount}}}'
|
|
559
|
+
const result = spawnSync(DOCKER, ["inspect", "--format", template, containerId], {
|
|
560
|
+
cwd: "/", encoding: "utf8", timeout: 20_000, maxBuffer: 64 * 1024,
|
|
561
|
+
env: { PATH: "/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" }, stdio: ["ignore", "pipe", "ignore"],
|
|
562
|
+
})
|
|
563
|
+
if (result.error || result.status !== 0) throw new Error("bounded reboot owner inspection failed")
|
|
564
|
+
return object(JSON.parse(result.stdout ?? ""), "reboot owner inspection")
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
function readAuthorityText(file) {
|
|
568
|
+
if (!file.startsWith(`${AUTHORITY_ROOT}/`) || realpathSync(file) !== file) throw new Error("root authority path is unsafe")
|
|
569
|
+
const fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW)
|
|
570
|
+
try {
|
|
571
|
+
const stat = fstatSync(fd)
|
|
572
|
+
if (!stat.isFile() || stat.uid !== 0 || stat.gid !== 0 || stat.nlink !== 1 || (stat.mode & 0o7777) !== 0o600 || stat.size < 1 || stat.size > 65536) throw new Error("root authority metadata is unsafe")
|
|
573
|
+
return readFileSync(fd, "utf8")
|
|
574
|
+
} finally { closeSync(fd) }
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
function telegramGatewayQuiescence() {
|
|
578
|
+
const observe = () => {
|
|
579
|
+
const config = object(JSON.parse(readAuthorityText(`${AUTHORITY_ROOT}/active.json`)), "root authority configuration")
|
|
580
|
+
const keyId = text(config.keyId, "root issuer", /^[A-Za-z0-9_-]{1,128}$/u)
|
|
581
|
+
const botId = text(config.botId, "root bot identity", /^[1-9][0-9]*$/u)
|
|
582
|
+
const publicKeyDigest = text(config.publicKeyDigest, "root issuer digest", /^sha256:[a-f0-9]{64}$/u)
|
|
583
|
+
const epochRoot = `${AUTHORITY_ROOT}/epochs/${keyId}`
|
|
584
|
+
if (config.lockPath !== `${epochRoot}/authority.lock`) throw new Error("root authority lock path changed")
|
|
585
|
+
const epoch = object(JSON.parse(readAuthorityText(`${epochRoot}/epoch.json`)), "root authority epoch")
|
|
586
|
+
if (epoch.schemaVersion !== 1 || epoch.state !== "prepared" || epoch.epochId !== keyId || epoch.botId !== botId || epoch.publicKeyDigest !== publicKeyDigest
|
|
587
|
+
|| epoch.revokedTokenStatus !== 401 || !/^sha256:[a-f0-9]{64}$/u.test(epoch.tokenDigest) || !/^sha256:[a-f0-9]{64}$/u.test(epoch.previousTokenDigest)
|
|
588
|
+
|| epoch.tokenDigest === epoch.previousTokenDigest) throw new Error("root authority token epoch is invalid")
|
|
589
|
+
const pid = Number(readAuthorityText(config.lockPath))
|
|
590
|
+
if (!Number.isSafeInteger(pid) || pid <= 0 || pid > 4_194_304) throw new Error("root authority lock PID is invalid")
|
|
591
|
+
const processes = spawnSync(PGREP, ["-f", "sanctuary-telegram-authority-entry[.]js"], {
|
|
592
|
+
cwd: "/", encoding: "utf8", timeout: 5_000, maxBuffer: 64 * 1024, stdio: ["ignore", "pipe", "ignore"],
|
|
593
|
+
})
|
|
594
|
+
if (processes.error || processes.status !== 0 || processes.stdout.trim() !== String(pid)) throw new Error("root authority does not have exactly one live poller")
|
|
595
|
+
const status = readBoundedProcStatus(`/proc/${pid}/status`)
|
|
596
|
+
if (!status.split("\n").includes("Uid:\t0\t0\t0\t0") || !status.split("\n").includes("Gid:\t0\t0\t0\t0")) throw new Error("root authority process identity is invalid")
|
|
597
|
+
const command = readBoundedProcStatus(`/proc/${pid}/cmdline`)
|
|
598
|
+
if (command !== ["/usr/local/bin/node", `${AUTHORITY_ROOT}/package/dist/heart/daemon/sanctuary-telegram-authority-entry.js`, "--config", `${AUTHORITY_ROOT}/active.json`, ""].join("\0")) throw new Error("root authority process command changed")
|
|
599
|
+
const processStartTime = parseProcStartTime(readBoundedProcStatus(`/proc/${pid}/stat`))
|
|
600
|
+
const resident = inspectRebootOwner(activeContainerId)
|
|
601
|
+
if (resident.containerId !== activeContainerId || resident.name !== `/${activeContainer}` || resident.imageId !== expectedImageId || resident.running !== false || resident.pid !== 0) throw new Error("resident Telegram owner is not stopped")
|
|
602
|
+
return { keyId, botId, publicKeyDigest, pid, processStartTime, bootId: readFileSync(BOOT_ID, "utf8"), resident }
|
|
603
|
+
}
|
|
604
|
+
const before = observe()
|
|
605
|
+
const after = observe()
|
|
606
|
+
if (JSON.stringify(before) !== JSON.stringify(after)) throw new Error("root authority process generation changed")
|
|
607
|
+
return {
|
|
608
|
+
schemaVersion: 1, activePollers: 1, residentStopped: true, keyId: after.keyId, botId: after.botId, publicKeyDigest: after.publicKeyDigest,
|
|
609
|
+
processBindingDigest: createHash("sha256").update(JSON.stringify(after)).digest("hex"), observedAt: new Date().toISOString(),
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
function runningRebootOwnerGeneration() {
|
|
614
|
+
const before = inspectRebootOwner()
|
|
615
|
+
if (before.name !== `/${activeContainer}` || before.imageId !== expectedImageId || before.running !== true || before.health !== "healthy"
|
|
616
|
+
|| !Number.isSafeInteger(before.pid) || before.pid <= 0 || !Number.isSafeInteger(before.restartCount) || before.restartCount < 0) {
|
|
617
|
+
throw new Error("reboot owner generation is invalid")
|
|
618
|
+
}
|
|
619
|
+
const processBefore = liveContainerProcessIdentity(before.pid)
|
|
620
|
+
const after = inspectRebootOwner(before.containerId)
|
|
621
|
+
const processAfter = liveContainerProcessIdentity(after.pid)
|
|
622
|
+
assertStableContainerProcess({ ...before, ...processBefore }, { ...after, ...processAfter })
|
|
623
|
+
return { ...after, ...processAfter, processBindingDigest: productionProcessBindingDigest({ ...after, ...processAfter }) }
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
function stopExactRebootOwner(expectedBinding) {
|
|
627
|
+
text(expectedBinding, "reboot process binding", SHA256)
|
|
628
|
+
const generation = runningRebootOwnerGeneration()
|
|
629
|
+
if (generation.processBindingDigest !== expectedBinding) throw new Error("production process generation changed before exact stop")
|
|
630
|
+
const result = spawnSync(DOCKER, ["stop", "--time", "30", generation.containerId], {
|
|
631
|
+
cwd: "/", encoding: "utf8", timeout: 45_000, maxBuffer: 64 * 1024,
|
|
632
|
+
env: { PATH: "/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" }, stdio: ["ignore", "ignore", "ignore"],
|
|
633
|
+
})
|
|
634
|
+
if (result.error || result.status !== 0) throw new Error("exact production owner stop failed")
|
|
635
|
+
const proof = { containerId: generation.containerId, imageId: generation.imageId, restartCount: generation.restartCount, startedAt: generation.startedAt, processBindingDigest: expectedBinding }
|
|
636
|
+
verifyStoppedRebootOwner(proof)
|
|
637
|
+
return proof
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
function verifyStoppedRebootOwner(proof) {
|
|
641
|
+
const value = inspectRebootOwner(text(proof.containerId, "stopped owner container id", SHA256))
|
|
642
|
+
if (value.containerId !== proof.containerId || value.name !== `/${activeContainer}` || value.imageId !== proof.imageId
|
|
643
|
+
|| value.restartCount !== proof.restartCount || value.startedAt !== proof.startedAt || value.running !== false || value.pid !== 0) {
|
|
644
|
+
throw new Error("exact stopped production owner generation changed")
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
function denialTargetSnapshot(dependencies = { run: spawnSync }) {
|
|
649
|
+
const template = '{"containerId":{{json .Id}},"imageId":{{json .Image}},"running":{{json .State.Running}},"status":{{json .State.Status}},"restartCount":{{json .RestartCount}},"startedAt":{{json .State.StartedAt}}}'
|
|
650
|
+
const result = dependencies.run(DOCKER, ["inspect", "--format", template, "calibre-web"], {
|
|
651
|
+
cwd: "/", encoding: "utf8", timeout: 10_000, maxBuffer: 64 * 1024,
|
|
652
|
+
env: { PATH: "/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" }, stdio: ["ignore", "pipe", "ignore"],
|
|
653
|
+
})
|
|
654
|
+
if (result.error || result.status !== 0) throw new Error("bounded denial target inspection failed")
|
|
655
|
+
const value = object(JSON.parse(result.stdout ?? ""), "denial target inspection")
|
|
656
|
+
exactKeys(value, ["containerId", "imageId", "running", "status", "restartCount", "startedAt"], "denial target inspection")
|
|
657
|
+
const containerId = text(value.containerId, "denial target container id", SHA256)
|
|
658
|
+
const imageId = text(value.imageId, "denial target image id", IMAGE_ID)
|
|
659
|
+
const status = text(value.status, "denial target status", /^(?:created|running|paused|restarting|removing|exited|dead)$/u)
|
|
660
|
+
const startedAt = text(value.startedAt, "denial target started at", /^[0-9]{4}-[0-9]{2}-[0-9]{2}T[^\s]{1,64}Z$/u)
|
|
661
|
+
if (typeof value.running !== "boolean" || !Number.isSafeInteger(value.restartCount) || value.restartCount < 0) {
|
|
662
|
+
throw new Error("denial target lifecycle is invalid")
|
|
663
|
+
}
|
|
664
|
+
return {
|
|
665
|
+
containerIdDigest: createHash("sha256").update(containerId).digest("hex"),
|
|
666
|
+
imageDigest: createHash("sha256").update(imageId).digest("hex"),
|
|
667
|
+
running: value.running,
|
|
668
|
+
status,
|
|
669
|
+
restartCount: value.restartCount,
|
|
670
|
+
startedAtDigest: createHash("sha256").update(startedAt).digest("hex"),
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
function containerOwnerSnapshot(expectedImage) {
|
|
675
|
+
text(expectedImage, "expected image id", IMAGE_ID)
|
|
676
|
+
const template = '{"containerId":{{json .Id}},"imageId":{{json .Image}}}'
|
|
677
|
+
const result = spawnSync(DOCKER, ["inspect", "--format", template, activeContainerId], {
|
|
678
|
+
cwd: "/", encoding: "utf8", timeout: 10_000, maxBuffer: 64 * 1024,
|
|
679
|
+
env: { PATH: "/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" }, stdio: ["ignore", "pipe", "ignore"],
|
|
680
|
+
})
|
|
681
|
+
if (result.error || result.status !== 0) throw new Error("bounded production owner inspection failed")
|
|
682
|
+
const value = object(JSON.parse(result.stdout), "production owner inspection")
|
|
683
|
+
if (value.containerId !== activeContainerId || value.imageId !== expectedImage) throw new Error("production owner identity is invalid")
|
|
684
|
+
return {
|
|
685
|
+
imageId: expectedImage,
|
|
686
|
+
containerId: text(value.containerId, "container id", /^[0-9a-f]{64}$/u),
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
function containerRestartSnapshot(expectedImage) {
|
|
691
|
+
text(expectedImage, "expected image id", IMAGE_ID)
|
|
692
|
+
const template = '{"containerId":{{json .Id}},"imageId":{{json .Image}},"running":{{json .State.Running}},"health":{{json .State.Health.Status}},"restartCount":{{json .RestartCount}}}'
|
|
693
|
+
const result = spawnSync(DOCKER, ["inspect", "--format", template, activeContainerId], {
|
|
694
|
+
cwd: "/", encoding: "utf8", timeout: 10_000, maxBuffer: 64 * 1024,
|
|
695
|
+
env: { PATH: "/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" }, stdio: ["ignore", "pipe", "ignore"],
|
|
696
|
+
})
|
|
697
|
+
if (result.error || result.status !== 0) throw new Error("bounded production restart inspection failed")
|
|
698
|
+
const value = object(JSON.parse(result.stdout), "production restart inspection")
|
|
699
|
+
if (value.containerId !== activeContainerId || value.imageId !== expectedImage || !Number.isSafeInteger(value.restartCount) || value.restartCount < 0) throw new Error("production restart identity is invalid")
|
|
700
|
+
return {
|
|
701
|
+
imageId: expectedImage, containerId: text(value.containerId, "container id", SHA256),
|
|
702
|
+
running: value.running === true, health: text(value.health, "container health", /^(?:healthy|starting|unhealthy|missing)$/u),
|
|
703
|
+
restartCount: value.restartCount,
|
|
704
|
+
}
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
function recoveryPath(id) { return `${RECOVERY_ROOT}/${id}.json` }
|
|
708
|
+
function createIntentPath(name) { return `${RECOVERY_ROOT}/create-${createHash("sha256").update(name).digest("hex")}.json` }
|
|
709
|
+
function rejectionReceiptPath(id) { return `${RECOVERY_ROOT}/${id}.rejected.json` }
|
|
710
|
+
|
|
711
|
+
function persistPrivateJson(target, value) {
|
|
712
|
+
mkdirSync(RECOVERY_ROOT, { recursive: true, mode: 0o700 })
|
|
713
|
+
chmodSync(RECOVERY_ROOT, 0o700)
|
|
714
|
+
chownSync(RECOVERY_ROOT, 0, 0)
|
|
715
|
+
const temporary = `${target}.tmp-${process.pid}`
|
|
716
|
+
const fd = openSync(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL, 0o600)
|
|
717
|
+
try { chownSync(temporary, 0, 0); writeFileSync(fd, `${JSON.stringify(value)}\n`); fsyncSync(fd) } finally { closeSync(fd) }
|
|
718
|
+
renameSync(temporary, target)
|
|
719
|
+
const directory = openSync(RECOVERY_ROOT, constants.O_RDONLY)
|
|
720
|
+
try { fsyncSync(directory) } finally { closeSync(directory) }
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
function optionalPrivateJson(target) {
|
|
724
|
+
try { return object(readPrivateJson(target), "private acceptance record") }
|
|
725
|
+
catch (error) { if (error?.code === "ENOENT") return null; throw error }
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
function persistCreateIntent(name, requested) {
|
|
729
|
+
mkdirSync(RECOVERY_ROOT, { recursive: true, mode: 0o700 })
|
|
730
|
+
chmodSync(RECOVERY_ROOT, 0o700)
|
|
731
|
+
chownSync(RECOVERY_ROOT, 0, 0)
|
|
732
|
+
const target = createIntentPath(name)
|
|
733
|
+
const fd = openSync(target, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL, 0o600)
|
|
734
|
+
try {
|
|
735
|
+
chownSync(target, 0, 0)
|
|
736
|
+
writeFileSync(fd, `${JSON.stringify({ schemaVersion: 1, name, permissions: requested })}\n`)
|
|
737
|
+
fsyncSync(fd)
|
|
738
|
+
} finally { closeSync(fd) }
|
|
739
|
+
const directory = openSync(RECOVERY_ROOT, constants.O_RDONLY)
|
|
740
|
+
try { fsyncSync(directory) } finally { closeSync(directory) }
|
|
741
|
+
}
|
|
742
|
+
|
|
743
|
+
function optionalCreateIntent(name) {
|
|
744
|
+
const value = optionalPrivateJson(createIntentPath(name))
|
|
745
|
+
if (value === null) return null
|
|
746
|
+
exactKeys(value, ["schemaVersion", "name", "permissions"], "Unraid key creation intent")
|
|
747
|
+
if (value.schemaVersion !== 1 || value.name !== name) throw new Error("Unraid key creation intent is invalid")
|
|
748
|
+
return { name, permissions: permissions(value.permissions) }
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
function removePrivateFile(target) {
|
|
752
|
+
try { unlinkSync(target) } catch (error) { if (error?.code !== "ENOENT") throw error }
|
|
753
|
+
const directory = openSync(RECOVERY_ROOT, constants.O_RDONLY)
|
|
754
|
+
try { fsyncSync(directory) } finally { closeSync(directory) }
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
function removeCreateIntent(name) { removePrivateFile(createIntentPath(name)) }
|
|
758
|
+
|
|
759
|
+
function optionalRejectionReceipt(id) {
|
|
760
|
+
const value = optionalPrivateJson(rejectionReceiptPath(id))
|
|
761
|
+
if (value === null) return null
|
|
762
|
+
exactKeys(value, ["schemaVersion", "id", "status", "proofDigest"], "revoked key rejection receipt")
|
|
763
|
+
if (value.schemaVersion !== 1 || value.id !== id || (value.status !== 401 && value.status !== 403)
|
|
764
|
+
|| value.proofDigest !== createHash("sha256").update(`revoked-key-rejected\0${id}\0${value.status}`).digest("hex")) {
|
|
765
|
+
throw new Error("revoked key rejection receipt is invalid")
|
|
766
|
+
}
|
|
767
|
+
return { valid: false, status: value.status, id }
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
function persistRejectionReceipt(id, status) {
|
|
771
|
+
persistPrivateJson(rejectionReceiptPath(id), {
|
|
772
|
+
schemaVersion: 1, id, status,
|
|
773
|
+
proofDigest: createHash("sha256").update(`revoked-key-rejected\0${id}\0${status}`).digest("hex"),
|
|
774
|
+
})
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
async function probeRevokedUnraidKey(id, dependencies = {
|
|
778
|
+
readReceipt: optionalRejectionReceipt,
|
|
779
|
+
readRecovery: optionalRecoveryRecord,
|
|
780
|
+
persistReceipt: persistRejectionReceipt,
|
|
781
|
+
removeRecovery,
|
|
782
|
+
fetchImpl: fetch,
|
|
783
|
+
}) {
|
|
784
|
+
text(id, "key id", KEY_ID)
|
|
785
|
+
const receipt = dependencies.readReceipt(id)
|
|
786
|
+
if (receipt !== null) {
|
|
787
|
+
const staleRecovery = dependencies.readRecovery(id)
|
|
788
|
+
if (staleRecovery !== null) {
|
|
789
|
+
if (staleRecovery.id !== id) throw new Error("recovery key id is invalid")
|
|
790
|
+
dependencies.removeRecovery(id)
|
|
791
|
+
}
|
|
792
|
+
return receipt
|
|
793
|
+
}
|
|
794
|
+
const record = dependencies.readRecovery(id)
|
|
795
|
+
if (record === null || record.id !== id) throw new Error("recovery key id is invalid")
|
|
796
|
+
const response = await dependencies.fetchImpl(GRAPHQL_ENDPOINT, {
|
|
797
|
+
method: "POST", headers: { "content-type": "application/json", "x-api-key": record.key },
|
|
798
|
+
body: JSON.stringify({ query: "query AcceptanceAuthProbe { info { os { hostname } } }", variables: {} }),
|
|
799
|
+
signal: AbortSignal.timeout(10_000),
|
|
800
|
+
})
|
|
801
|
+
if (response.status !== 401 && response.status !== 403) throw new Error("revoked key still authenticates")
|
|
802
|
+
dependencies.persistReceipt(id, response.status)
|
|
803
|
+
dependencies.removeRecovery(id)
|
|
804
|
+
return { valid: false, status: response.status, id }
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
function optionalRecoveryRecord(id) {
|
|
808
|
+
try { return normalizeRecord(readPrivateJson(recoveryPath(id))) }
|
|
809
|
+
catch (error) { if (error?.code === "ENOENT") return null; throw error }
|
|
810
|
+
}
|
|
811
|
+
|
|
812
|
+
function persistRecovery(record) {
|
|
813
|
+
persistPrivateJson(recoveryPath(record.id), record)
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
function removeRecovery(id) {
|
|
817
|
+
unlinkSync(recoveryPath(id))
|
|
818
|
+
const directory = openSync(RECOVERY_ROOT, constants.O_RDONLY)
|
|
819
|
+
try { fsyncSync(directory) } finally { closeSync(directory) }
|
|
820
|
+
}
|
|
821
|
+
|
|
822
|
+
function acknowledgeStoredUnraidKey(id, dependencies = {
|
|
823
|
+
readRecovery: optionalRecoveryRecord,
|
|
824
|
+
inventoryRecords,
|
|
825
|
+
removeRecovery,
|
|
826
|
+
removeIntent: removeCreateIntent,
|
|
827
|
+
}) {
|
|
828
|
+
text(id, "key id", KEY_ID)
|
|
829
|
+
const matches = dependencies.inventoryRecords().filter((record) => record.id === id)
|
|
830
|
+
if (matches.length !== 1 || !KEY_NAME.test(matches[0].name)) throw new Error("Unraid key final storage acknowledgement is invalid")
|
|
831
|
+
const expected = matches[0].name.startsWith("Butler RO") ? RO_PERMISSIONS : [...RO_PERMISSIONS, "DOCKER:UPDATE_ANY"].sort()
|
|
832
|
+
if (matches[0].roles.length !== 0 || JSON.stringify(flattened(matches[0])) !== JSON.stringify(expected)) {
|
|
833
|
+
throw new Error("Unraid key final storage acknowledgement is invalid")
|
|
834
|
+
}
|
|
835
|
+
const recovery = dependencies.readRecovery(id)
|
|
836
|
+
if (recovery === null) {
|
|
837
|
+
dependencies.removeIntent?.(matches[0].name)
|
|
838
|
+
return { acknowledged: true, id }
|
|
839
|
+
}
|
|
840
|
+
const provisional = [...expected, "API_KEY:UPDATE_ANY"].sort()
|
|
841
|
+
if (recovery.id !== id || recovery.name !== matches[0].name || recovery.key !== matches[0].key
|
|
842
|
+
|| JSON.stringify(recovery.roles) !== JSON.stringify(["GUEST"])
|
|
843
|
+
|| JSON.stringify(flattened(recovery)) !== JSON.stringify(provisional)) {
|
|
844
|
+
throw new Error("Unraid key recovery ownership is invalid")
|
|
845
|
+
}
|
|
846
|
+
dependencies.removeRecovery(id)
|
|
847
|
+
dependencies.removeIntent?.(matches[0].name)
|
|
848
|
+
return { acknowledged: true, id }
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
function permissions(value) {
|
|
852
|
+
if (!Array.isArray(value) || value.length === 0) throw new Error("permissions are invalid")
|
|
853
|
+
const result = value.map((item) => text(item, "permission", PERMISSION)).sort()
|
|
854
|
+
if (new Set(result).size !== result.length) throw new Error("permissions are ambiguous")
|
|
855
|
+
return result
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
function canonicalHealthProbeInput(input) {
|
|
859
|
+
const value = object(input, "health probe input")
|
|
860
|
+
exactKeys(value, ["label", "scenarioHandleDigest", "ownerImageDigest", "ownerContainerDigest"], "health probe input")
|
|
861
|
+
if (!HEALTH_PROBE_LABELS.has(value.label)) throw new Error("health probe label is invalid")
|
|
862
|
+
for (const key of ["scenarioHandleDigest", "ownerImageDigest", "ownerContainerDigest"]) text(value[key], `health probe ${key}`, SHA256)
|
|
863
|
+
return value
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
function canonicalHealthProbeRequest(input) {
|
|
867
|
+
const value = object(input, "health probe request")
|
|
868
|
+
exactKeys(value, ["label", "scenarioHandleDigest"], "health probe request")
|
|
869
|
+
if (!HEALTH_PROBE_LABELS.has(value.label)) throw new Error("health probe label is invalid")
|
|
870
|
+
text(value.scenarioHandleDigest, "health probe scenario digest", SHA256)
|
|
871
|
+
return value
|
|
872
|
+
}
|
|
873
|
+
|
|
874
|
+
function healthProbeDockerArgs(mode, input) {
|
|
875
|
+
if (mode !== "run" && mode !== "stop" && mode !== "recover") throw new Error("health probe mode is invalid")
|
|
876
|
+
const value = canonicalHealthProbeInput(input)
|
|
877
|
+
return [
|
|
878
|
+
"exec", activeContainerId, "/usr/local/bin/node", HEALTH_PROBE_ENTRY, mode,
|
|
879
|
+
"--label", value.label,
|
|
880
|
+
"--scenario", value.scenarioHandleDigest,
|
|
881
|
+
"--owner-image", value.ownerImageDigest,
|
|
882
|
+
"--owner-container", value.ownerContainerDigest,
|
|
883
|
+
]
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
function healthProbeFinalizeDockerArgs(before, after) {
|
|
887
|
+
const initial = canonicalHealthProbeInput(before)
|
|
888
|
+
const observed = canonicalHealthProbeInput(after)
|
|
889
|
+
return [
|
|
890
|
+
"exec", activeContainerId, "/usr/local/bin/node", HEALTH_PROBE_ENTRY, "finalize",
|
|
891
|
+
"--label", initial.label,
|
|
892
|
+
"--scenario", initial.scenarioHandleDigest,
|
|
893
|
+
"--owner-image", initial.ownerImageDigest,
|
|
894
|
+
"--owner-container", initial.ownerContainerDigest,
|
|
895
|
+
"--owner-image-after", observed.ownerImageDigest,
|
|
896
|
+
"--owner-container-after", observed.ownerContainerDigest,
|
|
897
|
+
]
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
function requireStableHealthProbeOwner(before, after) {
|
|
901
|
+
const initial = canonicalHealthProbeInput(before)
|
|
902
|
+
const observed = canonicalHealthProbeInput(after)
|
|
903
|
+
if (initial.label !== observed.label || initial.scenarioHandleDigest !== observed.scenarioHandleDigest) throw new Error("health probe scenario binding drifted")
|
|
904
|
+
if (initial.ownerImageDigest !== observed.ownerImageDigest || initial.ownerContainerDigest !== observed.ownerContainerDigest) throw new Error("health probe owner binding drifted")
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
function healthProbeReceiptPath(scenarioHandleDigest) {
|
|
908
|
+
return `${PRODUCTION_BUNDLE_SOURCE}/state/acceptance/health-probe-receipts/${scenarioHandleDigest}.json`
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
function healthProbeWorkspacePath(scenarioHandleDigest) {
|
|
912
|
+
return `${PRODUCTION_BUNDLE_SOURCE}/state/acceptance/health-probe-workspaces/${scenarioHandleDigest}`
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
function healthProbePendingPath(scenarioHandleDigest) {
|
|
916
|
+
return `${PRODUCTION_BUNDLE_SOURCE}/state/acceptance/health-probe-pending/${scenarioHandleDigest}.json`
|
|
917
|
+
}
|
|
918
|
+
|
|
919
|
+
function healthProbeProcessPath(scenarioHandleDigest) {
|
|
920
|
+
return `${PRODUCTION_BUNDLE_SOURCE}/state/acceptance/health-probe-processes/${scenarioHandleDigest}.json`
|
|
921
|
+
}
|
|
922
|
+
|
|
923
|
+
function healthOwnerMutationActive() {
|
|
924
|
+
if (activeHealthProbes.size > 0) return true
|
|
925
|
+
for (const root of ["health-probe-workspaces", "health-probe-pending", "health-probe-processes"]
|
|
926
|
+
.map((name) => `${PRODUCTION_BUNDLE_SOURCE}/state/acceptance/${name}`)) {
|
|
927
|
+
let directory
|
|
928
|
+
try { directory = opendirSync(root) } catch (error) { if (error.code === "ENOENT") continue; throw error }
|
|
929
|
+
try { if (directory.readSync() !== null) return true } finally { directory.closeSync() }
|
|
930
|
+
}
|
|
931
|
+
return false
|
|
932
|
+
}
|
|
933
|
+
|
|
934
|
+
function healthProbeArtifactDisposition({ receipt, workspace, pending }) {
|
|
935
|
+
if (receipt) return "complete"
|
|
936
|
+
if (workspace || pending) return "recovery_required"
|
|
937
|
+
return "absent"
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
function healthProbeOperationBudgets() {
|
|
941
|
+
return { startMaxMs: 115_000, completeStatusMaxMs: 130_000, recoveryMaxMs: 85_000, composedCaptureMaxMs: 165_000 }
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u
|
|
945
|
+
const SCHEDULER_COMMAND = "/usr/local/bin/node /opt/ouro/dist/heart/daemon/ouro-entry.js poke sanctuary --habit sanctuary-health --trigger cron"
|
|
946
|
+
const SCHEDULER_CRONTAB = "/home/ouro/.ouro-cli/scheduler/sanctuary.crontab"
|
|
947
|
+
|
|
948
|
+
function canonicalIso(value) {
|
|
949
|
+
if (typeof value !== "string") return false
|
|
950
|
+
const date = new Date(value)
|
|
951
|
+
return Number.isFinite(date.getTime()) && date.toISOString() === value
|
|
952
|
+
}
|
|
953
|
+
|
|
954
|
+
function safeMacEqual(observed, expected) {
|
|
955
|
+
return typeof observed === "string" && SHA256.test(observed) && timingSafeEqual(Buffer.from(observed, "hex"), Buffer.from(expected, "hex"))
|
|
956
|
+
}
|
|
957
|
+
|
|
958
|
+
function requireExactSchedulerReceipt(scheduler, request, identityKey, phases) {
|
|
959
|
+
exactKeys(scheduler, ["after", "before", "deliveryDelta", "label", "nonReplay", "occurrenceId", "privateTurnCount", "providerInvocationCount", "receiptMac", "recordedAt", "runnerId", "scenarioHandleDigest", "schedulerOrigin", "schemaVersion", "supervisor", "sweep", "sweepDelta", "trigger"], "scheduler receipt")
|
|
960
|
+
const before = object(scheduler.before, "scheduler before cursor")
|
|
961
|
+
const after = object(scheduler.after, "scheduler after cursor")
|
|
962
|
+
const sweep = object(scheduler.sweep, "scheduler sweep")
|
|
963
|
+
const supervisor = object(scheduler.supervisor, "scheduler supervisor")
|
|
964
|
+
const origin = object(scheduler.schedulerOrigin, "scheduler origin")
|
|
965
|
+
exactKeys(before, ["deliveryCount", "sweepCount"], "scheduler before cursor")
|
|
966
|
+
exactKeys(after, ["deliveryCount", "sweepCount"], "scheduler after cursor")
|
|
967
|
+
exactKeys(sweep, ["deliveryId", "digestDue", "opened", "recordDigest", "recovered"], "scheduler sweep")
|
|
968
|
+
exactKeys(supervisor, ["args", "binaryPath", "childCount", "childPid", "crontabPath", "daemonPid", "healthy", "manifest", "namespace", "renderedCrontab", "schemaVersion"], "scheduler supervisor")
|
|
969
|
+
exactKeys(origin, ["invocationPid", "invocationStartTime", "occurrenceId", "parentPid", "parentStartTime", "proofMac", "scenarioHandleDigest", "schedulerRunId", "slot"], "scheduler origin")
|
|
970
|
+
const manifest = Array.isArray(supervisor.manifest) ? supervisor.manifest : []
|
|
971
|
+
if (manifest.length !== 1) throw new Error("scheduler manifest is invalid")
|
|
972
|
+
const job = object(manifest[0], "scheduler manifest job")
|
|
973
|
+
exactKeys(job, ["agent", "command", "id", "lastRun", "schedule", "taskId", "taskPath"], "scheduler manifest job")
|
|
974
|
+
const unsignedScheduler = Object.fromEntries(Object.entries(scheduler).filter(([key]) => key !== "receiptMac"))
|
|
975
|
+
const expectedReceiptMac = createHmac("sha256", identityKey).update(`sanctuary-scheduler-liveness-receipt-v2\0${JSON.stringify(unsignedScheduler)}`).digest("hex")
|
|
976
|
+
const proofCommand = {
|
|
977
|
+
kind: "habit.scheduler-fire", agent: "sanctuary", habitName: "sanctuary-health", trigger: "cron",
|
|
978
|
+
slot: origin.slot, occurrenceId: origin.occurrenceId, schedulerRunId: origin.schedulerRunId,
|
|
979
|
+
invocationPid: origin.invocationPid, parentPid: origin.parentPid, parentStartTime: origin.parentStartTime,
|
|
980
|
+
invocationStartTime: origin.invocationStartTime, scenarioHandleDigest: origin.scenarioHandleDigest,
|
|
981
|
+
}
|
|
982
|
+
const expectedProofMac = createHmac("sha256", identityKey).update(JSON.stringify(proofCommand)).digest("hex")
|
|
983
|
+
if (scheduler.schemaVersion !== "sanctuary-scheduler-liveness-receipt-v1" || scheduler.label !== request.label
|
|
984
|
+
|| scheduler.scenarioHandleDigest !== request.scenarioHandleDigest || scheduler.trigger !== "cron"
|
|
985
|
+
|| !UUID.test(scheduler.runnerId) || !canonicalIso(scheduler.recordedAt)
|
|
986
|
+
|| !Number.isSafeInteger(before.sweepCount) || before.sweepCount < 0 || !Number.isSafeInteger(before.deliveryCount) || before.deliveryCount < 0
|
|
987
|
+
|| after.sweepCount !== before.sweepCount + 1 || after.deliveryCount !== before.deliveryCount
|
|
988
|
+
|| scheduler.sweepDelta !== 1 || scheduler.deliveryDelta !== 0 || scheduler.providerInvocationCount !== 0 || scheduler.privateTurnCount !== 0 || scheduler.nonReplay !== true
|
|
989
|
+
|| !SHA256.test(sweep.recordDigest) || sweep.opened !== 0 || sweep.recovered !== 0 || sweep.digestDue !== false || sweep.deliveryId !== null
|
|
990
|
+
|| phases.length !== 1 || sweep.recordDigest !== phases[0]?.sweepReceiptDigest
|
|
991
|
+
|| supervisor.schemaVersion !== "supercronic-supervisor-snapshot-v1" || supervisor.daemonPid !== 1 || supervisor.childCount !== 1
|
|
992
|
+
|| !Number.isSafeInteger(supervisor.childPid) || supervisor.childPid <= 1 || supervisor.healthy !== true
|
|
993
|
+
|| supervisor.binaryPath !== "/usr/local/bin/supercronic" || supervisor.crontabPath !== SCHEDULER_CRONTAB
|
|
994
|
+
|| JSON.stringify(supervisor.args) !== JSON.stringify(["-split-logs", "-inotify", SCHEDULER_CRONTAB]) || supervisor.namespace !== "habit:sanctuary"
|
|
995
|
+
|| job.id !== "sanctuary:sanctuary-health" || job.agent !== "sanctuary" || job.taskId !== "sanctuary-health" || job.schedule !== "*/15 * * * *"
|
|
996
|
+
|| (job.lastRun !== null && !canonicalIso(job.lastRun)) || job.command !== SCHEDULER_COMMAND || job.taskPath !== "/home/ouro/AgentBundles/sanctuary.ouro/habits/sanctuary-health.md"
|
|
997
|
+
|| typeof supervisor.renderedCrontab !== "string" || !supervisor.renderedCrontab.includes(`# ouro:habit:sanctuary:sanctuary:sanctuary-health\n*/15 * * * * ${SCHEDULER_COMMAND}\n`)
|
|
998
|
+
|| typeof origin.slot !== "string" || !canonicalIso(origin.slot) || scheduler.occurrenceId !== `cron:${origin.slot}` || origin.occurrenceId !== scheduler.occurrenceId
|
|
999
|
+
|| !UUID.test(origin.schedulerRunId) || origin.scenarioHandleDigest !== request.scenarioHandleDigest
|
|
1000
|
+
|| !Number.isSafeInteger(origin.invocationPid) || origin.invocationPid <= 1 || !Number.isSafeInteger(origin.parentPid) || origin.parentPid !== supervisor.childPid
|
|
1001
|
+
|| typeof origin.parentStartTime !== "string" || !/^[0-9]+$/u.test(origin.parentStartTime)
|
|
1002
|
+
|| typeof origin.invocationStartTime !== "string" || !/^[0-9]+$/u.test(origin.invocationStartTime)
|
|
1003
|
+
|| !safeMacEqual(origin.proofMac, expectedProofMac) || !safeMacEqual(scheduler.receiptMac, expectedReceiptMac)) {
|
|
1004
|
+
throw new Error("scheduler receipt semantics are invalid")
|
|
1005
|
+
}
|
|
1006
|
+
}
|
|
1007
|
+
|
|
1008
|
+
function requireHealthProbeCompleteAttestationUnchecked(receipt, snapshot, input, readSchedulerIdentityKey = () => readFileSync(`${PRODUCTION_BUNDLE_SOURCE}/state/senses/telegram/identity.key`, "utf8").trim()) {
|
|
1009
|
+
const request = canonicalHealthProbeRequest(input)
|
|
1010
|
+
const value = object(receipt, "health probe receipt")
|
|
1011
|
+
exactKeys(value, HEALTH_PROBE_RECEIPT_KEYS, "health probe receipt")
|
|
1012
|
+
const observed = object(snapshot, "health probe complete owner")
|
|
1013
|
+
const phases = Array.isArray(value.phases) ? value.phases.map((phase) => object(phase, "health probe phase")) : []
|
|
1014
|
+
const digestFields = ["ownerImageDigestBefore", "ownerImageDigestAfter", "ownerContainerDigestBefore", "ownerContainerDigestAfter", "beforeStateDigest", "restoredStateDigest", "cronFingerprintBefore", "cronFingerprintAfter", "fixtureSequenceDigest"]
|
|
1015
|
+
const booleanFields = ["cronRegisteredBefore", "cronRegisteredAfter", "cronDegradedBefore", "cronDegradedAfter", "workspaceAbsent", "socketAbsent", "snapshotAbsent", "realCheckEquivalent", "productionRestored"]
|
|
1016
|
+
for (const phase of phases) exactKeys(phase, ["deliveryKind", "deliveryReceiptDigest", "digestDue", "fixtureStatus", "name", "opened", "ordinal", "recovered", "sweepReceiptDigest", "trigger"], "health probe phase")
|
|
1017
|
+
const scheduler = request.label === "unit-16f-cron-fingerprint" ? object(value.schedulerReceipt, "scheduler receipt") : null
|
|
1018
|
+
if (scheduler) requireExactSchedulerReceipt(scheduler, request, readSchedulerIdentityKey(), phases)
|
|
1019
|
+
const unit16fPhase = phases.length === 1 && phases[0].ordinal === 1 && phases[0].name === "cron-unchanged" && phases[0].trigger === "cron"
|
|
1020
|
+
&& phases[0].fixtureStatus === null && phases[0].opened === 0 && phases[0].recovered === 0 && phases[0].digestDue === false
|
|
1021
|
+
&& phases[0].deliveryKind === null && phases[0].deliveryReceiptDigest === null && SHA256.test(phases[0].sweepReceiptDigest)
|
|
1022
|
+
const unit16fEvidence = request.label !== "unit-16f-cron-fingerprint" || (
|
|
1023
|
+
digestFields.every((field) => typeof value[field] === "string" && SHA256.test(value[field]))
|
|
1024
|
+
&& booleanFields.every((field) => typeof value[field] === "boolean") && canonicalIso(value.effectiveNow)
|
|
1025
|
+
&& value.clockMode === "ambient" && value.providerInvocationCount === 0 && value.privateTurnCount === 0 && value.deliveryCount === 0
|
|
1026
|
+
&& value.fixtureSequenceDigest === createHash("sha256").update(JSON.stringify([])).digest("hex")
|
|
1027
|
+
&& value.cronFingerprintBefore === value.cronFingerprintAfter && value.cronRegisteredBefore === true && value.cronRegisteredAfter === true
|
|
1028
|
+
&& value.cronDegradedBefore === false && value.cronDegradedAfter === false && value.workspaceAbsent === true && value.socketAbsent === true
|
|
1029
|
+
&& value.snapshotAbsent === true && value.realCheckEquivalent === true && value.productionRestored === true && unit16fPhase
|
|
1030
|
+
)
|
|
1031
|
+
if (value.schemaVersion !== "sanctuary-health-probe-receipt-v1" || value.label !== request.label
|
|
1032
|
+
|| value.scenarioHandleDigest !== request.scenarioHandleDigest
|
|
1033
|
+
|| !SHA256.test(value.ownerImageDigestBefore) || !SHA256.test(value.ownerContainerDigestBefore)
|
|
1034
|
+
|| !SHA256.test(value.ownerImageDigestAfter) || !SHA256.test(value.ownerContainerDigestAfter)
|
|
1035
|
+
|| value.ownerImageDigestBefore !== value.ownerImageDigestAfter || value.ownerContainerDigestBefore !== value.ownerContainerDigestAfter
|
|
1036
|
+
|| observed.imageId !== `sha256:${value.ownerImageDigestAfter}` || observed.containerId !== value.ownerContainerDigestAfter
|
|
1037
|
+
|| observed.running !== true || observed.health !== "healthy" || !unit16fEvidence
|
|
1038
|
+
|| (request.label !== "unit-16f-cron-fingerprint" && value.schedulerReceipt !== null)) {
|
|
1039
|
+
throw new Error("health probe complete attestation is invalid")
|
|
1040
|
+
}
|
|
1041
|
+
}
|
|
1042
|
+
|
|
1043
|
+
function requireHealthProbeCompleteAttestation(receipt, snapshot, input, readSchedulerIdentityKey) {
|
|
1044
|
+
try { requireHealthProbeCompleteAttestationUnchecked(receipt, snapshot, input, readSchedulerIdentityKey) }
|
|
1045
|
+
catch { throw new Error("health probe complete attestation is invalid") }
|
|
1046
|
+
}
|
|
1047
|
+
|
|
1048
|
+
function finalizeHealthProbeAfterAttestation(receipt, snapshot, request, finalize, readSchedulerIdentityKey) {
|
|
1049
|
+
requireHealthProbeCompleteAttestation(receipt, snapshot, request, readSchedulerIdentityKey)
|
|
1050
|
+
return finalize()
|
|
1051
|
+
}
|
|
1052
|
+
|
|
1053
|
+
function completeHealthProbeFromReceipt(request, snapshot, readReceipt) {
|
|
1054
|
+
const value = object(snapshot, "health probe complete owner")
|
|
1055
|
+
requireHealthProbeCompleteAttestation(readReceipt(), value, request)
|
|
1056
|
+
return { state: "complete", containerSnapshot: value }
|
|
1057
|
+
}
|
|
1058
|
+
|
|
1059
|
+
function readHealthProbeReceipt(file) {
|
|
1060
|
+
const fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW)
|
|
1061
|
+
try {
|
|
1062
|
+
const metadata = fstatSync(fd)
|
|
1063
|
+
if (!metadata.isFile() || metadata.uid !== 10001 || (metadata.mode & 0o777) !== 0o600 || metadata.size > MAX_REQUEST) {
|
|
1064
|
+
throw new Error("health probe receipt metadata is invalid")
|
|
1065
|
+
}
|
|
1066
|
+
return object(JSON.parse(readFileSync(fd, "utf8")), "health probe receipt")
|
|
1067
|
+
} finally { closeSync(fd) }
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
function readHealthProbePendingReceipt(file) {
|
|
1071
|
+
const envelope = readHealthProbeReceipt(file)
|
|
1072
|
+
exactKeys(envelope, ["receipt", "schemaVersion"], "health probe pending envelope")
|
|
1073
|
+
if (envelope.schemaVersion !== "sanctuary-health-probe-pending-v1") throw new Error("health probe pending envelope is invalid")
|
|
1074
|
+
return object(envelope.receipt, "health probe pending receipt")
|
|
1075
|
+
}
|
|
1076
|
+
|
|
1077
|
+
function attestHealthProbeProcessAbsent(input, dependencies = {
|
|
1078
|
+
run: spawnSync,
|
|
1079
|
+
markerPresent: () => Boolean(statIfPresent(healthProbeProcessPath(input.scenarioHandleDigest))),
|
|
1080
|
+
}) {
|
|
1081
|
+
const value = canonicalHealthProbeInput(input)
|
|
1082
|
+
const stopped = dependencies.run(DOCKER, healthProbeDockerArgs("stop", value), {
|
|
1083
|
+
cwd: "/", encoding: "utf8", timeout: 15_000, maxBuffer: 64 * 1024,
|
|
1084
|
+
env: { PATH: "/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" }, stdio: ["ignore", "ignore", "ignore"],
|
|
1085
|
+
})
|
|
1086
|
+
if (stopped.error || stopped.status !== 0 || dependencies.markerPresent()) throw new Error("health probe in-container process termination failed")
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
async function waitForHealthProbeProcessMarker(record, timeoutMs = 5_000) {
|
|
1090
|
+
const marker = healthProbeProcessPath(record.input.scenarioHandleDigest)
|
|
1091
|
+
const deadline = Date.now() + timeoutMs
|
|
1092
|
+
while (Date.now() < deadline) {
|
|
1093
|
+
const metadata = statIfPresent(marker)
|
|
1094
|
+
if (metadata) {
|
|
1095
|
+
if (!metadata.isFile() || metadata.uid !== 10001 || (metadata.mode & 0o777) !== 0o600) throw new Error("health probe process marker metadata is invalid")
|
|
1096
|
+
return
|
|
1097
|
+
}
|
|
1098
|
+
if (record.state !== "running") throw new Error("health probe exited before process registration")
|
|
1099
|
+
await new Promise((resolve) => setTimeout(resolve, 50))
|
|
1100
|
+
}
|
|
1101
|
+
throw new Error("health probe process registration timed out")
|
|
1102
|
+
}
|
|
1103
|
+
|
|
1104
|
+
async function startHealthProbe(input) {
|
|
1105
|
+
const value = canonicalHealthProbeInput(input)
|
|
1106
|
+
const existing = activeHealthProbes.get(value.scenarioHandleDigest)
|
|
1107
|
+
const artifacts = {
|
|
1108
|
+
receipt: statIfPresent(healthProbeReceiptPath(value.scenarioHandleDigest)),
|
|
1109
|
+
workspace: statIfPresent(healthProbeWorkspacePath(value.scenarioHandleDigest)),
|
|
1110
|
+
pending: statIfPresent(healthProbePendingPath(value.scenarioHandleDigest)),
|
|
1111
|
+
}
|
|
1112
|
+
if (existing || healthProbeArtifactDisposition(artifacts) !== "absent" || statIfPresent(healthProbeProcessPath(value.scenarioHandleDigest))) {
|
|
1113
|
+
throw new Error("health probe requires inspect-before-retry")
|
|
1114
|
+
}
|
|
1115
|
+
const child = spawn(DOCKER, healthProbeDockerArgs("run", value), {
|
|
1116
|
+
cwd: "/", env: { PATH: "/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" }, stdio: "ignore",
|
|
1117
|
+
})
|
|
1118
|
+
const record = { input: value, child, state: "running", exitCode: null }
|
|
1119
|
+
activeHealthProbes.set(value.scenarioHandleDigest, record)
|
|
1120
|
+
child.once("error", () => {
|
|
1121
|
+
if (record.state !== "terminating") record.state = "failed"
|
|
1122
|
+
record.exitCode = -1
|
|
1123
|
+
})
|
|
1124
|
+
child.once("exit", (code) => {
|
|
1125
|
+
record.state = record.state === "terminating" ? "terminated" : code === 0 ? "exited" : "failed"
|
|
1126
|
+
record.exitCode = code
|
|
1127
|
+
})
|
|
1128
|
+
child.unref()
|
|
1129
|
+
record.readyPromise = waitForHealthProbeProcessMarker(record)
|
|
1130
|
+
await record.readyPromise
|
|
1131
|
+
return {
|
|
1132
|
+
state: "started",
|
|
1133
|
+
operationDigest: createHash("sha256").update(JSON.stringify({ operation: "start_health_probe", ...value })).digest("hex"),
|
|
1134
|
+
}
|
|
1135
|
+
}
|
|
1136
|
+
|
|
1137
|
+
function statIfPresent(file) {
|
|
1138
|
+
try { return statSync(file) } catch (error) { if (error.code === "ENOENT") return null; throw error }
|
|
1139
|
+
}
|
|
1140
|
+
|
|
1141
|
+
async function healthProbeStatus(input, fullSnapshot = () => containerSnapshot(expectedImageId)) {
|
|
1142
|
+
const request = canonicalHealthProbeRequest(input)
|
|
1143
|
+
const receipt = statIfPresent(healthProbeReceiptPath(request.scenarioHandleDigest))
|
|
1144
|
+
if (receipt) {
|
|
1145
|
+
if (!receipt.isFile() || receipt.uid !== 10001 || (receipt.mode & 0o777) !== 0o600) throw new Error("health probe receipt metadata is invalid")
|
|
1146
|
+
const snapshot = object(await fullSnapshot(), "health probe complete owner")
|
|
1147
|
+
return completeHealthProbeFromReceipt(request, snapshot, () => readHealthProbeReceipt(healthProbeReceiptPath(request.scenarioHandleDigest)))
|
|
1148
|
+
}
|
|
1149
|
+
const record = activeHealthProbes.get(request.scenarioHandleDigest)
|
|
1150
|
+
if (!record) return { state: healthProbeArtifactDisposition({
|
|
1151
|
+
receipt: null,
|
|
1152
|
+
workspace: statIfPresent(healthProbeWorkspacePath(request.scenarioHandleDigest)),
|
|
1153
|
+
pending: statIfPresent(healthProbePendingPath(request.scenarioHandleDigest)),
|
|
1154
|
+
}) }
|
|
1155
|
+
if (record.input.label !== request.label) throw new Error("health probe scenario binding drifted")
|
|
1156
|
+
if (record.state === "failed") return { state: "failed" }
|
|
1157
|
+
if (record.state === "running") return { state: "running" }
|
|
1158
|
+
if (record.state === "terminating" || record.state === "terminated" || record.state === "recovering") return { state: "recovery_required" }
|
|
1159
|
+
const snapshot = object(await fullSnapshot(), "health probe production owner")
|
|
1160
|
+
const value = healthProbeCoordinates({ targetId: TARGET_HOST, ...request }, snapshot)
|
|
1161
|
+
requireStableHealthProbeOwner(record.input, value)
|
|
1162
|
+
const finalized = finalizeHealthProbeAfterAttestation(
|
|
1163
|
+
readHealthProbePendingReceipt(healthProbePendingPath(value.scenarioHandleDigest)), snapshot, request,
|
|
1164
|
+
() => spawnSync(DOCKER, healthProbeFinalizeDockerArgs(record.input, value), {
|
|
1165
|
+
cwd: "/", encoding: "utf8", timeout: 20_000, maxBuffer: 64 * 1024,
|
|
1166
|
+
env: { PATH: "/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" }, stdio: ["ignore", "ignore", "ignore"],
|
|
1167
|
+
}),
|
|
1168
|
+
)
|
|
1169
|
+
if (finalized.error || finalized.status !== 0) throw new Error("health probe final attestation failed")
|
|
1170
|
+
const finalReceipt = statIfPresent(healthProbeReceiptPath(value.scenarioHandleDigest))
|
|
1171
|
+
if (!finalReceipt || !finalReceipt.isFile() || finalReceipt.uid !== 10001 || (finalReceipt.mode & 0o777) !== 0o600) {
|
|
1172
|
+
throw new Error("health probe final receipt is absent or invalid")
|
|
1173
|
+
}
|
|
1174
|
+
requireHealthProbeCompleteAttestation(readHealthProbeReceipt(healthProbeReceiptPath(value.scenarioHandleDigest)), snapshot, request)
|
|
1175
|
+
activeHealthProbes.delete(value.scenarioHandleDigest)
|
|
1176
|
+
return { state: "complete", containerSnapshot: snapshot }
|
|
1177
|
+
}
|
|
1178
|
+
|
|
1179
|
+
function healthProbeChildExited(child) {
|
|
1180
|
+
return child.exitCode !== null || child.signalCode !== null
|
|
1181
|
+
}
|
|
1182
|
+
|
|
1183
|
+
function waitForHealthProbeChildExit(child, timeoutMs) {
|
|
1184
|
+
if (healthProbeChildExited(child)) return Promise.resolve(true)
|
|
1185
|
+
return new Promise((resolve) => {
|
|
1186
|
+
let settled = false
|
|
1187
|
+
let timer
|
|
1188
|
+
const finish = (exited) => {
|
|
1189
|
+
if (settled) return
|
|
1190
|
+
settled = true
|
|
1191
|
+
if (timer) clearTimeout(timer)
|
|
1192
|
+
child.removeListener("exit", onExit)
|
|
1193
|
+
resolve(exited)
|
|
1194
|
+
}
|
|
1195
|
+
const onExit = () => finish(true)
|
|
1196
|
+
child.once("exit", onExit)
|
|
1197
|
+
timer = setTimeout(() => finish(healthProbeChildExited(child)), timeoutMs)
|
|
1198
|
+
if (healthProbeChildExited(child)) finish(true)
|
|
1199
|
+
})
|
|
1200
|
+
}
|
|
1201
|
+
|
|
1202
|
+
function terminateHealthProbeChild(record, options = {}) {
|
|
1203
|
+
if (record.terminationPromise) return record.terminationPromise
|
|
1204
|
+
if (record.state !== "running" && record.state !== "terminating") return Promise.resolve()
|
|
1205
|
+
const termGraceMs = options.termGraceMs ?? HEALTH_PROBE_TERM_GRACE_MS
|
|
1206
|
+
const killGraceMs = options.killGraceMs ?? HEALTH_PROBE_KILL_GRACE_MS
|
|
1207
|
+
record.state = "terminating"
|
|
1208
|
+
const termination = (async () => {
|
|
1209
|
+
try {
|
|
1210
|
+
if (!healthProbeChildExited(record.child)) record.child.kill("SIGTERM")
|
|
1211
|
+
if (!await waitForHealthProbeChildExit(record.child, termGraceMs)) {
|
|
1212
|
+
record.child.kill("SIGKILL")
|
|
1213
|
+
if (!await waitForHealthProbeChildExit(record.child, killGraceMs)) throw new Error("health probe child did not exit")
|
|
1214
|
+
}
|
|
1215
|
+
record.state = "terminated"
|
|
1216
|
+
} catch (error) {
|
|
1217
|
+
record.state = "failed"
|
|
1218
|
+
throw error
|
|
1219
|
+
}
|
|
1220
|
+
})()
|
|
1221
|
+
record.terminationPromise = termination
|
|
1222
|
+
return termination
|
|
1223
|
+
}
|
|
1224
|
+
|
|
1225
|
+
async function recoverAfterHealthProbeTermination(record, recovery, options = {}) {
|
|
1226
|
+
if (record) await terminateHealthProbeChild(record, options)
|
|
1227
|
+
return await recovery()
|
|
1228
|
+
}
|
|
1229
|
+
|
|
1230
|
+
async function recoverHealthProbe(input) {
|
|
1231
|
+
const value = canonicalHealthProbeInput(input)
|
|
1232
|
+
const record = activeHealthProbes.get(value.scenarioHandleDigest)
|
|
1233
|
+
if (record && JSON.stringify(record.input) !== JSON.stringify(value)) throw new Error("health probe owner binding drifted")
|
|
1234
|
+
if (record?.recoveryPromise) return await record.recoveryPromise
|
|
1235
|
+
const recovery = (async () => {
|
|
1236
|
+
if (record?.readyPromise) {
|
|
1237
|
+
try { await record.readyPromise } catch { /* recovery still terminates an unready launch */ }
|
|
1238
|
+
}
|
|
1239
|
+
const processMarker = statIfPresent(healthProbeProcessPath(value.scenarioHandleDigest))
|
|
1240
|
+
if (processMarker) {
|
|
1241
|
+
if (!processMarker.isFile() || processMarker.uid !== 10001 || (processMarker.mode & 0o777) !== 0o600) throw new Error("health probe process marker metadata is invalid")
|
|
1242
|
+
}
|
|
1243
|
+
attestHealthProbeProcessAbsent(value)
|
|
1244
|
+
return await recoverAfterHealthProbeTermination(record, () => {
|
|
1245
|
+
if (record) record.state = "recovering"
|
|
1246
|
+
const workspace = statIfPresent(healthProbeWorkspacePath(value.scenarioHandleDigest))
|
|
1247
|
+
const pending = statIfPresent(healthProbePendingPath(value.scenarioHandleDigest))
|
|
1248
|
+
if (!workspace && !pending) {
|
|
1249
|
+
activeHealthProbes.delete(value.scenarioHandleDigest)
|
|
1250
|
+
return { recovered: true }
|
|
1251
|
+
}
|
|
1252
|
+
const result = spawnSync(DOCKER, healthProbeDockerArgs("recover", value), {
|
|
1253
|
+
cwd: "/", encoding: "utf8", timeout: 45_000, maxBuffer: 64 * 1024,
|
|
1254
|
+
env: { PATH: "/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" }, stdio: ["ignore", "ignore", "ignore"],
|
|
1255
|
+
})
|
|
1256
|
+
if (result.error || result.status !== 0 || statIfPresent(healthProbeWorkspacePath(value.scenarioHandleDigest)) || statIfPresent(healthProbePendingPath(value.scenarioHandleDigest))) {
|
|
1257
|
+
throw new Error("health probe recovery failed")
|
|
1258
|
+
}
|
|
1259
|
+
activeHealthProbes.delete(value.scenarioHandleDigest)
|
|
1260
|
+
return { recovered: true }
|
|
1261
|
+
})
|
|
1262
|
+
})()
|
|
1263
|
+
if (record) record.recoveryPromise = recovery
|
|
1264
|
+
try {
|
|
1265
|
+
return await recovery
|
|
1266
|
+
} catch (error) {
|
|
1267
|
+
if (record) record.recoveryPromise = undefined
|
|
1268
|
+
throw error
|
|
1269
|
+
}
|
|
1270
|
+
}
|
|
1271
|
+
|
|
1272
|
+
function healthProbeCoordinates(payload, snapshot) {
|
|
1273
|
+
if (payload.targetId !== TARGET_HOST) throw new Error("target host is invalid")
|
|
1274
|
+
const label = text(payload.label, "health probe label")
|
|
1275
|
+
if (!HEALTH_PROBE_LABELS.has(label)) throw new Error("health probe label is invalid")
|
|
1276
|
+
const scenarioHandleDigest = text(payload.scenarioHandleDigest, "health probe scenario digest", SHA256)
|
|
1277
|
+
const ownerImage = text(snapshot.imageId, "health probe owner image", IMAGE_ID)
|
|
1278
|
+
const ownerContainerDigest = text(snapshot.containerId, "health probe owner container", SHA256)
|
|
1279
|
+
if (expectedImageId && ownerImage !== expectedImageId) throw new Error("health probe production owner drifted")
|
|
1280
|
+
return { label, scenarioHandleDigest, ownerImageDigest: ownerImage.slice("sha256:".length), ownerContainerDigest }
|
|
1281
|
+
}
|
|
1282
|
+
|
|
1283
|
+
function createDispatchDrain() {
|
|
1284
|
+
let accepting = true
|
|
1285
|
+
const inFlight = new Set()
|
|
1286
|
+
return {
|
|
1287
|
+
run(operation) {
|
|
1288
|
+
if (!accepting) return Promise.reject(new Error("host broker is shutting down"))
|
|
1289
|
+
const task = Promise.resolve().then(operation)
|
|
1290
|
+
inFlight.add(task)
|
|
1291
|
+
void task.finally(() => inFlight.delete(task)).catch(() => {})
|
|
1292
|
+
return task
|
|
1293
|
+
},
|
|
1294
|
+
async stopAndDrain() {
|
|
1295
|
+
accepting = false
|
|
1296
|
+
await Promise.allSettled([...inFlight])
|
|
1297
|
+
},
|
|
1298
|
+
}
|
|
1299
|
+
}
|
|
1300
|
+
|
|
1301
|
+
function createHealthProbeOperationCoordinator() {
|
|
1302
|
+
const tails = new Map()
|
|
1303
|
+
const recovered = new Set()
|
|
1304
|
+
const enqueue = (scenario, operation) => {
|
|
1305
|
+
const prior = tails.get(scenario) ?? Promise.resolve()
|
|
1306
|
+
const task = prior.catch(() => {}).then(operation)
|
|
1307
|
+
tails.set(scenario, task)
|
|
1308
|
+
void task.finally(() => { if (tails.get(scenario) === task) tails.delete(scenario) }).catch(() => {})
|
|
1309
|
+
return task
|
|
1310
|
+
}
|
|
1311
|
+
return {
|
|
1312
|
+
start(scenario, operation) {
|
|
1313
|
+
if (recovered.has(scenario)) return Promise.reject(new Error("health probe recovered scenario cannot restart"))
|
|
1314
|
+
return enqueue(scenario, operation)
|
|
1315
|
+
},
|
|
1316
|
+
recover(scenario, operation) {
|
|
1317
|
+
recovered.add(scenario)
|
|
1318
|
+
return enqueue(scenario, operation)
|
|
1319
|
+
},
|
|
1320
|
+
}
|
|
1321
|
+
}
|
|
1322
|
+
|
|
1323
|
+
function createOwnerMutationCoordinator() {
|
|
1324
|
+
let ownerTail = Promise.resolve()
|
|
1325
|
+
let pendingOperations = 0
|
|
1326
|
+
let rebootReservation = null
|
|
1327
|
+
const activeHealth = new Set()
|
|
1328
|
+
const enqueue = (operation) => {
|
|
1329
|
+
if (rebootReservation !== null) return Promise.reject(new Error("owner mutation refused by reboot reservation"))
|
|
1330
|
+
pendingOperations += 1
|
|
1331
|
+
const task = ownerTail.then(async () => {
|
|
1332
|
+
try { return await operation() } finally { pendingOperations -= 1 }
|
|
1333
|
+
})
|
|
1334
|
+
ownerTail = task.then(() => {}, () => {})
|
|
1335
|
+
return task
|
|
1336
|
+
}
|
|
1337
|
+
return {
|
|
1338
|
+
active() { return pendingOperations > 0 },
|
|
1339
|
+
async reserveReboot(reservationId, processBindingDigest, operation) {
|
|
1340
|
+
text(reservationId, "reboot reservation", SHA256)
|
|
1341
|
+
text(processBindingDigest, "reboot process binding", SHA256)
|
|
1342
|
+
if (rebootReservation !== null) throw new Error("reboot reservation already exists")
|
|
1343
|
+
rebootReservation = { id: reservationId, processBindingDigest, stoppedProof: null, attempted: false }
|
|
1344
|
+
try {
|
|
1345
|
+
await ownerTail
|
|
1346
|
+
if (pendingOperations !== 0 || activeHealth.size !== 0) throw new Error("reboot reservation could not drain owner mutations")
|
|
1347
|
+
return await operation()
|
|
1348
|
+
} catch (error) {
|
|
1349
|
+
rebootReservation = null
|
|
1350
|
+
throw error
|
|
1351
|
+
}
|
|
1352
|
+
},
|
|
1353
|
+
async stopRebootOwner(reservationId, processBindingDigest, operation) {
|
|
1354
|
+
text(reservationId, "reboot reservation", SHA256)
|
|
1355
|
+
text(processBindingDigest, "reboot process binding", SHA256)
|
|
1356
|
+
if (rebootReservation?.id !== reservationId || rebootReservation.processBindingDigest !== processBindingDigest) throw new Error("reboot reservation process binding is absent or mismatched")
|
|
1357
|
+
if (rebootReservation.stoppedProof !== null) throw new Error("reboot owner was already stopped")
|
|
1358
|
+
const proof = object(await operation(), "stopped reboot owner proof")
|
|
1359
|
+
if (proof.processBindingDigest !== processBindingDigest) throw new Error("stopped reboot owner proof binding is invalid")
|
|
1360
|
+
rebootReservation.stoppedProof = proof
|
|
1361
|
+
return proof
|
|
1362
|
+
},
|
|
1363
|
+
async commitReboot(reservationId, processBindingDigest, operation) {
|
|
1364
|
+
text(reservationId, "reboot reservation", SHA256)
|
|
1365
|
+
text(processBindingDigest, "reboot process binding", SHA256)
|
|
1366
|
+
if (rebootReservation?.id !== reservationId || rebootReservation.processBindingDigest !== processBindingDigest) throw new Error("reboot reservation is absent or mismatched")
|
|
1367
|
+
if (rebootReservation.stoppedProof === null) throw new Error("reboot owner is not stopped")
|
|
1368
|
+
if (rebootReservation.attempted) throw new Error("reboot commit was already attempted")
|
|
1369
|
+
const markAttempted = () => {
|
|
1370
|
+
if (rebootReservation.attempted) throw new Error("reboot commit was already attempted")
|
|
1371
|
+
rebootReservation.attempted = true
|
|
1372
|
+
}
|
|
1373
|
+
const result = await operation(rebootReservation.stoppedProof, markAttempted)
|
|
1374
|
+
if (!rebootReservation.attempted) throw new Error("reboot commit was not attempted")
|
|
1375
|
+
return result
|
|
1376
|
+
},
|
|
1377
|
+
healthStart(scenario, operation) {
|
|
1378
|
+
text(scenario, "health owner scenario", SHA256)
|
|
1379
|
+
return enqueue(async () => {
|
|
1380
|
+
if (activeHealth.has(scenario)) throw new Error("health probe scenario is already active")
|
|
1381
|
+
activeHealth.add(scenario)
|
|
1382
|
+
try { return await operation() } catch (error) { activeHealth.delete(scenario); throw error }
|
|
1383
|
+
})
|
|
1384
|
+
},
|
|
1385
|
+
healthRecover(scenario, operation) {
|
|
1386
|
+
text(scenario, "health owner scenario", SHA256)
|
|
1387
|
+
return enqueue(async () => {
|
|
1388
|
+
try { const result = await operation(); activeHealth.delete(scenario); return result } catch (error) { throw error }
|
|
1389
|
+
})
|
|
1390
|
+
},
|
|
1391
|
+
healthOperation(scenario, operation) {
|
|
1392
|
+
text(scenario, "health owner scenario", SHA256)
|
|
1393
|
+
return enqueue(operation)
|
|
1394
|
+
},
|
|
1395
|
+
interactive(operation) {
|
|
1396
|
+
return enqueue(async () => {
|
|
1397
|
+
if (activeHealth.size > 0) throw new Error("owner mutation refused while health probe is active")
|
|
1398
|
+
return await operation()
|
|
1399
|
+
})
|
|
1400
|
+
},
|
|
1401
|
+
}
|
|
1402
|
+
}
|
|
1403
|
+
|
|
1404
|
+
function createInteractiveRestartDriver() {
|
|
1405
|
+
const records = new Map()
|
|
1406
|
+
const tasks = new Set()
|
|
1407
|
+
return {
|
|
1408
|
+
poll(input, operation) {
|
|
1409
|
+
const key = input.scenarioHandleDigest
|
|
1410
|
+
const existing = records.get(key)
|
|
1411
|
+
if (existing?.state === "complete") return { state: "complete", receipt: existing.receipt }
|
|
1412
|
+
if (existing?.state === "failed") return { state: "failed", errorDigest: existing.errorDigest }
|
|
1413
|
+
if (existing) return { state: "waiting" }
|
|
1414
|
+
records.set(key, { state: "waiting", operation })
|
|
1415
|
+
return { state: "waiting" }
|
|
1416
|
+
},
|
|
1417
|
+
arm(key) {
|
|
1418
|
+
const record = records.get(key)
|
|
1419
|
+
if (!record || record.state !== "waiting" || !record.operation) return
|
|
1420
|
+
const operation = record.operation
|
|
1421
|
+
records.set(key, { state: "waiting" })
|
|
1422
|
+
const task = Promise.resolve().then(operation).then(
|
|
1423
|
+
(receipt) => { records.set(key, { state: "complete", receipt }) },
|
|
1424
|
+
(error) => { records.set(key, { state: "failed", errorDigest: createHash("sha256").update(interactiveFailureCategory(error)).digest("hex") }) },
|
|
1425
|
+
)
|
|
1426
|
+
tasks.add(task)
|
|
1427
|
+
void task.finally(() => tasks.delete(task))
|
|
1428
|
+
},
|
|
1429
|
+
async stopAndDrain() { await Promise.allSettled([...tasks]) },
|
|
1430
|
+
}
|
|
1431
|
+
}
|
|
1432
|
+
|
|
1433
|
+
function armRestartAfterResponseClosed(connection, scenarioHandleDigest, driver = interactiveRestartDriver) {
|
|
1434
|
+
let written = false
|
|
1435
|
+
let closed = false
|
|
1436
|
+
let armed = false
|
|
1437
|
+
const maybeArm = () => {
|
|
1438
|
+
if (!armed && written && closed) { armed = true; driver.arm(scenarioHandleDigest) }
|
|
1439
|
+
}
|
|
1440
|
+
connection.once("close", () => { closed = true; maybeArm() })
|
|
1441
|
+
return () => { written = true; maybeArm() }
|
|
1442
|
+
}
|
|
1443
|
+
|
|
1444
|
+
function interactiveFailureCategory(error) {
|
|
1445
|
+
const message = error instanceof Error ? error.message : ""
|
|
1446
|
+
if (/inspect-before-retry/u.test(message)) return "inspect-before-retry"
|
|
1447
|
+
if (/production runtime/u.test(message)) return "production-runtime"
|
|
1448
|
+
if (/owner restart|did not return|restart failed/u.test(message)) return "owner-restart"
|
|
1449
|
+
if (/prepared receipt/u.test(message)) return "prepared-receipt"
|
|
1450
|
+
if (/reconciliation/u.test(message)) return "reconciliation"
|
|
1451
|
+
if (/receipt/u.test(message)) return "receipt-validation"
|
|
1452
|
+
return "unknown"
|
|
1453
|
+
}
|
|
1454
|
+
|
|
1455
|
+
async function waitForInteractiveRuntimeReady(input, dependencies) {
|
|
1456
|
+
for (let attempt = 0; attempt < 60; attempt += 1) {
|
|
1457
|
+
const readiness = object(await dependencies.runtime("interactive_runtime_ready", input), "interactive runtime readiness")
|
|
1458
|
+
exactKeys(readiness, ["ready"], "interactive runtime readiness")
|
|
1459
|
+
if (readiness.ready === true) return
|
|
1460
|
+
if (readiness.ready !== false) throw new Error("interactive runtime readiness is invalid")
|
|
1461
|
+
await dependencies.sleep(1_000)
|
|
1462
|
+
}
|
|
1463
|
+
throw new Error("interactive production runtime readiness timed out")
|
|
1464
|
+
}
|
|
1465
|
+
|
|
1466
|
+
function canonicalInteractiveRequest(input, expectedLabel) {
|
|
1467
|
+
const value = object(input, "interactive driver input")
|
|
1468
|
+
exactKeys(value, ["label", "scenarioHandleDigest"], "interactive driver input")
|
|
1469
|
+
if (value.label !== expectedLabel) throw new Error("interactive driver label is invalid")
|
|
1470
|
+
text(value.scenarioHandleDigest, "interactive driver scenario digest", SHA256)
|
|
1471
|
+
return value
|
|
1472
|
+
}
|
|
1473
|
+
|
|
1474
|
+
const DUPLICATE_RECEIPT_KEYS = [
|
|
1475
|
+
"schemaVersion", "phase", "label", "scenarioHandleDigest", "approvalIdDigest", "checkpointDigest",
|
|
1476
|
+
"suspendedSessionRevisionDigest", "approvalEpochBefore", "callbackAttempts", "distinctQueryCount",
|
|
1477
|
+
"callbackDataDigest", "barrierObserved", "settledCount", "claimCount", "mutationCount",
|
|
1478
|
+
"staleReplayAttempts", "staleReplaySettled", "staleReplayMutationCount", "promptTerminal",
|
|
1479
|
+
"writeCredentialObserved",
|
|
1480
|
+
]
|
|
1481
|
+
const TIMEOUT_RECEIPT_KEYS = [
|
|
1482
|
+
"schemaVersion", "phase", "label", "scenarioHandleDigest", "approvalIdDigest", "checkpointDigest",
|
|
1483
|
+
"suspendedSessionRevisionDigest", "approvalEpochBefore", "callbackAttempts", "distinctQueryCount",
|
|
1484
|
+
"callbackDataDigest", "settledCount", "claimCount", "mutationCount", "staleAcknowledged", "promptTerminal",
|
|
1485
|
+
]
|
|
1486
|
+
|
|
1487
|
+
const RESTART_RECEIPT_KEYS = [
|
|
1488
|
+
"schemaVersion", "phase", "label", "scenarioHandleDigest", "approvalIdDigest", "checkpointDigest",
|
|
1489
|
+
"suspendedSessionRevisionDigest", "approvalEpochBefore", "approvalEpochAfterRestart", "continuationEpochAfter",
|
|
1490
|
+
"ownerImageDigest", "ownerContainerDigest", "restartCountBefore", "restartCountAfter", "pendingDigestBefore",
|
|
1491
|
+
"pendingDigestAfter", "pendingRestored", "callbackAttempts", "mutationCount", "indeterminateRecoveryObserved",
|
|
1492
|
+
"attemptedRecoveryReopened", "attemptedRecordDigest", "recoveredRecordDigest", "indeterminateRetryCount",
|
|
1493
|
+
]
|
|
1494
|
+
const PREPARED_RESTART_RECEIPT_KEYS = [
|
|
1495
|
+
"schemaVersion", "phase", "label", "scenarioHandleDigest", "approvalIdDigest", "checkpointDigest",
|
|
1496
|
+
"suspendedSessionRevisionDigest", "approvalEpochBefore", "pendingDigestBefore", "indeterminateRecoveryObserved",
|
|
1497
|
+
"attemptedRecoveryReopened", "attemptedRecordDigest", "recoveredRecordDigest",
|
|
1498
|
+
"ownerImageDigest", "ownerContainerDigest", "restartCountBefore",
|
|
1499
|
+
]
|
|
1500
|
+
|
|
1501
|
+
function requireInteractiveReceipt(receipt, input) {
|
|
1502
|
+
const value = object(receipt, "interactive driver receipt")
|
|
1503
|
+
const duplicate = input.label === "unit-16l-duplicate-callback"
|
|
1504
|
+
exactKeys(value, duplicate ? DUPLICATE_RECEIPT_KEYS : RESTART_RECEIPT_KEYS, "interactive driver receipt")
|
|
1505
|
+
if (value.schemaVersion !== "sanctuary-interactive-driver-receipt-v2" || value.phase !== "complete"
|
|
1506
|
+
|| value.label !== input.label || value.scenarioHandleDigest !== input.scenarioHandleDigest
|
|
1507
|
+
|| ![value.approvalIdDigest, value.checkpointDigest, value.suspendedSessionRevisionDigest].every((item) => typeof item === "string" && SHA256.test(item))
|
|
1508
|
+
|| !Number.isSafeInteger(value.approvalEpochBefore) || value.approvalEpochBefore < 0) {
|
|
1509
|
+
throw new Error("interactive driver receipt is invalid")
|
|
1510
|
+
}
|
|
1511
|
+
if (duplicate) {
|
|
1512
|
+
if (!SHA256.test(value.callbackDataDigest) || value.callbackAttempts !== 2 || value.distinctQueryCount !== 2
|
|
1513
|
+
|| value.barrierObserved !== true || value.settledCount !== 2 || value.claimCount !== 1 || value.mutationCount !== 1
|
|
1514
|
+
|| value.staleReplayAttempts !== 1 || value.staleReplaySettled !== true || value.staleReplayMutationCount !== 0
|
|
1515
|
+
|| value.promptTerminal !== true || value.writeCredentialObserved !== false) throw new Error("interactive driver receipt is invalid")
|
|
1516
|
+
} else if (!SHA256.test(value.ownerImageDigest) || !SHA256.test(value.ownerContainerDigest)
|
|
1517
|
+
|| !SHA256.test(value.pendingDigestBefore) || value.pendingDigestAfter !== value.pendingDigestBefore
|
|
1518
|
+
|| value.approvalEpochAfterRestart !== value.approvalEpochBefore || !Number.isSafeInteger(value.continuationEpochAfter)
|
|
1519
|
+
|| value.continuationEpochAfter <= value.approvalEpochAfterRestart || !Number.isSafeInteger(value.restartCountBefore)
|
|
1520
|
+
|| value.restartCountAfter !== value.restartCountBefore + 1 || value.pendingRestored !== true || value.callbackAttempts !== 1
|
|
1521
|
+
|| value.mutationCount !== 1 || value.indeterminateRecoveryObserved !== true || value.attemptedRecoveryReopened !== true
|
|
1522
|
+
|| !SHA256.test(value.attemptedRecordDigest) || !SHA256.test(value.recoveredRecordDigest)
|
|
1523
|
+
|| value.attemptedRecordDigest === value.recoveredRecordDigest || value.indeterminateRetryCount !== 0) {
|
|
1524
|
+
throw new Error("interactive driver receipt is invalid")
|
|
1525
|
+
}
|
|
1526
|
+
return value
|
|
1527
|
+
}
|
|
1528
|
+
|
|
1529
|
+
function requireTimeoutReceipt(receipt, input) {
|
|
1530
|
+
const value = object(receipt, "timeout stale driver receipt")
|
|
1531
|
+
exactKeys(value, TIMEOUT_RECEIPT_KEYS, "timeout stale driver receipt")
|
|
1532
|
+
if (value.schemaVersion !== "sanctuary-timeout-stale-driver-receipt-v1" || value.phase !== "complete"
|
|
1533
|
+
|| value.label !== input.label || value.scenarioHandleDigest !== input.scenarioHandleDigest
|
|
1534
|
+
|| ![value.approvalIdDigest, value.checkpointDigest, value.suspendedSessionRevisionDigest, value.callbackDataDigest]
|
|
1535
|
+
.every((item) => typeof item === "string" && SHA256.test(item))
|
|
1536
|
+
|| !Number.isSafeInteger(value.approvalEpochBefore) || value.approvalEpochBefore < 0
|
|
1537
|
+
|| value.callbackAttempts !== 1 || value.distinctQueryCount !== 1 || value.settledCount !== 1
|
|
1538
|
+
|| value.claimCount !== 0 || value.mutationCount !== 0 || value.staleAcknowledged !== true || value.promptTerminal !== true) {
|
|
1539
|
+
throw new Error("timeout stale driver receipt is invalid")
|
|
1540
|
+
}
|
|
1541
|
+
return value
|
|
1542
|
+
}
|
|
1543
|
+
|
|
1544
|
+
function requirePreparedRestartReceipt(receipt, input) {
|
|
1545
|
+
const value = object(receipt, "restart continuation prepared receipt")
|
|
1546
|
+
exactKeys(value, PREPARED_RESTART_RECEIPT_KEYS, "restart continuation prepared receipt")
|
|
1547
|
+
if (value.schemaVersion !== "sanctuary-interactive-driver-receipt-v2" || (value.phase !== "prepared" && value.phase !== "attempted_or_indeterminate")
|
|
1548
|
+
|| value.label !== input.label || value.scenarioHandleDigest !== input.scenarioHandleDigest
|
|
1549
|
+
|| ![value.approvalIdDigest, value.checkpointDigest, value.suspendedSessionRevisionDigest, value.pendingDigestBefore,
|
|
1550
|
+
value.ownerImageDigest, value.ownerContainerDigest, value.attemptedRecordDigest, value.recoveredRecordDigest].every((item) => typeof item === "string" && SHA256.test(item))
|
|
1551
|
+
|| !Number.isSafeInteger(value.approvalEpochBefore) || value.approvalEpochBefore < 0 || value.indeterminateRecoveryObserved !== true
|
|
1552
|
+
|| value.attemptedRecoveryReopened !== true || value.attemptedRecordDigest === value.recoveredRecordDigest
|
|
1553
|
+
|| !Number.isSafeInteger(value.restartCountBefore) || value.restartCountBefore < 0) throw new Error("restart continuation prepared receipt is invalid")
|
|
1554
|
+
return value
|
|
1555
|
+
}
|
|
1556
|
+
|
|
1557
|
+
function runInteractiveRuntimeOperation(operation, value, dependencies = { run: spawnSync }) {
|
|
1558
|
+
const result = dependencies.run(DOCKER, ["exec", "-i", activeContainerId, ACCEPTANCE_ADAPTER], {
|
|
1559
|
+
cwd: "/", encoding: "utf8", timeout: 120_000, maxBuffer: MAX_REQUEST,
|
|
1560
|
+
env: { PATH: "/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" }, stdio: ["pipe", "pipe", "ignore"],
|
|
1561
|
+
input: JSON.stringify({ operation, ...value }),
|
|
1562
|
+
})
|
|
1563
|
+
if (result.error || result.status !== 0) throw new Error("interactive production runtime driver failed")
|
|
1564
|
+
try { return object(JSON.parse(result.stdout ?? ""), "interactive driver receipt") } catch { throw new Error("interactive driver receipt is invalid") }
|
|
1565
|
+
}
|
|
1566
|
+
|
|
1567
|
+
function runInteractiveDriver(input, dependencies = { run: spawnSync }) {
|
|
1568
|
+
const value = canonicalInteractiveRequest(input, "unit-16l-duplicate-callback")
|
|
1569
|
+
return requireInteractiveReceipt(runInteractiveRuntimeOperation("drive_duplicate_callbacks", value, dependencies), value)
|
|
1570
|
+
}
|
|
1571
|
+
|
|
1572
|
+
function runTimeoutStaleDriver(input, dependencies = { run: spawnSync }) {
|
|
1573
|
+
const value = canonicalInteractiveRequest(input, "unit-16k-timeout-stale")
|
|
1574
|
+
const response = runInteractiveRuntimeOperation("drive_timeout_stale", value, dependencies)
|
|
1575
|
+
if (response.state === "waiting") {
|
|
1576
|
+
exactKeys(response, ["state"], "timeout stale waiting response")
|
|
1577
|
+
return response
|
|
1578
|
+
}
|
|
1579
|
+
return requireTimeoutReceipt(response, value)
|
|
1580
|
+
}
|
|
1581
|
+
|
|
1582
|
+
async function restartButlerForAcceptance(input, dependencies = {
|
|
1583
|
+
snapshot: () => containerRestartSnapshot(expectedImageId), run: spawnSync,
|
|
1584
|
+
sleep: (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)),
|
|
1585
|
+
}) {
|
|
1586
|
+
canonicalInteractiveRequest(input, "unit-16m-restart-continuation")
|
|
1587
|
+
const before = object(await dependencies.snapshot(), "restart owner before")
|
|
1588
|
+
const result = dependencies.run(DOCKER, ["restart", activeContainerId], {
|
|
1589
|
+
cwd: "/", encoding: "utf8", timeout: 120_000, maxBuffer: 64 * 1024,
|
|
1590
|
+
env: { PATH: "/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" }, stdio: ["ignore", "ignore", "ignore"],
|
|
1591
|
+
})
|
|
1592
|
+
if (result.error || result.status !== 0) throw new Error("production butler restart failed")
|
|
1593
|
+
const ownerImage = text(before.imageId, "restart owner image", IMAGE_ID)
|
|
1594
|
+
const ownerContainer = text(before.containerId, "restart owner container", SHA256)
|
|
1595
|
+
if (!Number.isSafeInteger(before.restartCount) || before.restartCount < 0) throw new Error("restart owner count is invalid")
|
|
1596
|
+
const beforeLifecycleDigest = createHash("sha256").update(JSON.stringify(before)).digest("hex")
|
|
1597
|
+
for (let attempt = 0; attempt < 60; attempt += 1) {
|
|
1598
|
+
const after = object(await dependencies.snapshot(), "restart owner after")
|
|
1599
|
+
const afterLifecycleDigest = createHash("sha256").update(JSON.stringify(after)).digest("hex")
|
|
1600
|
+
if (after.running === true && after.health === "healthy" && after.containerId === ownerContainer && after.imageId === ownerImage
|
|
1601
|
+
&& after.restartCount === before.restartCount + 1 && afterLifecycleDigest !== beforeLifecycleDigest) {
|
|
1602
|
+
return {
|
|
1603
|
+
restarted: true,
|
|
1604
|
+
beforeLifecycleDigest,
|
|
1605
|
+
afterLifecycleDigest,
|
|
1606
|
+
restartInvocationCount: 1,
|
|
1607
|
+
ownerImageDigest: ownerImage.slice(7), ownerContainerDigest: ownerContainer,
|
|
1608
|
+
restartCountBefore: before.restartCount, restartCountAfter: after.restartCount,
|
|
1609
|
+
}
|
|
1610
|
+
}
|
|
1611
|
+
await dependencies.sleep(1_000)
|
|
1612
|
+
}
|
|
1613
|
+
throw new Error("production butler did not return with exact restart identity")
|
|
1614
|
+
}
|
|
1615
|
+
|
|
1616
|
+
function interactiveReceiptPath(scenarioHandleDigest) {
|
|
1617
|
+
return `${PRODUCTION_BUNDLE_SOURCE}/state/acceptance/interactive-driver-receipts/${scenarioHandleDigest}.json`
|
|
1618
|
+
}
|
|
1619
|
+
|
|
1620
|
+
function readInteractiveReceipt(input) {
|
|
1621
|
+
const file = interactiveReceiptPath(input.scenarioHandleDigest)
|
|
1622
|
+
let fd
|
|
1623
|
+
try { fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW) } catch (error) { if (error.code === "ENOENT") return null; throw error }
|
|
1624
|
+
try {
|
|
1625
|
+
const metadata = fstatSync(fd)
|
|
1626
|
+
if (!metadata.isFile() || metadata.uid !== 10001 || (metadata.mode & 0o777) !== 0o600 || metadata.size > MAX_REQUEST) throw new Error("interactive driver receipt metadata is invalid")
|
|
1627
|
+
return object(JSON.parse(readFileSync(fd, "utf8")), "interactive driver receipt")
|
|
1628
|
+
} finally { closeSync(fd) }
|
|
1629
|
+
}
|
|
1630
|
+
|
|
1631
|
+
function persistInteractiveReceipt(input, receipt) {
|
|
1632
|
+
const root = `${PRODUCTION_BUNDLE_SOURCE}/state/acceptance/interactive-driver-receipts`
|
|
1633
|
+
mkdirSync(root, { recursive: true, mode: 0o700 })
|
|
1634
|
+
chownSync(root, 10001, 10001)
|
|
1635
|
+
chmodSync(root, 0o700)
|
|
1636
|
+
const directory = openSync(root, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW)
|
|
1637
|
+
const directoryMetadata = fstatSync(directory)
|
|
1638
|
+
if (!directoryMetadata.isDirectory() || directoryMetadata.uid !== 10001 || (directoryMetadata.mode & 0o777) !== 0o700) {
|
|
1639
|
+
closeSync(directory)
|
|
1640
|
+
throw new Error("interactive driver receipt directory metadata is invalid")
|
|
1641
|
+
}
|
|
1642
|
+
const file = interactiveReceiptPath(input.scenarioHandleDigest)
|
|
1643
|
+
const temporary = `${file}.${process.pid}.${Date.now()}.tmp`
|
|
1644
|
+
try {
|
|
1645
|
+
const fd = openSync(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600)
|
|
1646
|
+
try { chownSync(temporary, 10001, 10001); writeFileSync(fd, `${JSON.stringify(receipt)}\n`); fsyncSync(fd) } finally { closeSync(fd) }
|
|
1647
|
+
renameSync(temporary, file)
|
|
1648
|
+
fsyncSync(directory)
|
|
1649
|
+
} finally { closeSync(directory) }
|
|
1650
|
+
}
|
|
1651
|
+
|
|
1652
|
+
async function driveDuplicateCallbacks(input, dependencies = {
|
|
1653
|
+
readReceipt: readInteractiveReceipt,
|
|
1654
|
+
persistReceipt: persistInteractiveReceipt,
|
|
1655
|
+
runtime: runInteractiveDriver,
|
|
1656
|
+
}) {
|
|
1657
|
+
const value = canonicalInteractiveRequest(input, "unit-16l-duplicate-callback")
|
|
1658
|
+
const existing = dependencies.readReceipt(value)
|
|
1659
|
+
if (existing) {
|
|
1660
|
+
if (existing.phase === "complete") return requireInteractiveReceipt(existing, value)
|
|
1661
|
+
throw new Error("duplicate callback drive requires inspect-before-retry")
|
|
1662
|
+
}
|
|
1663
|
+
dependencies.persistReceipt(value, { schemaVersion: "sanctuary-interactive-driver-receipt-v2", phase: "attempting", ...value })
|
|
1664
|
+
try {
|
|
1665
|
+
const receipt = requireInteractiveReceipt(await dependencies.runtime(value), value)
|
|
1666
|
+
dependencies.persistReceipt(value, receipt)
|
|
1667
|
+
return receipt
|
|
1668
|
+
} catch (error) {
|
|
1669
|
+
dependencies.persistReceipt(value, { schemaVersion: "sanctuary-interactive-driver-receipt-v2", phase: "attempted_or_indeterminate", ...value })
|
|
1670
|
+
throw error
|
|
1671
|
+
}
|
|
1672
|
+
}
|
|
1673
|
+
|
|
1674
|
+
async function driveTimeoutStale(input, dependencies = {
|
|
1675
|
+
readReceipt: readInteractiveReceipt,
|
|
1676
|
+
persistReceipt: persistInteractiveReceipt,
|
|
1677
|
+
runtime: runTimeoutStaleDriver,
|
|
1678
|
+
}) {
|
|
1679
|
+
const value = canonicalInteractiveRequest(input, "unit-16k-timeout-stale")
|
|
1680
|
+
const existing = dependencies.readReceipt(value)
|
|
1681
|
+
if (existing?.phase === "complete") return requireTimeoutReceipt(existing, value)
|
|
1682
|
+
if (existing && existing.phase !== "waiting") throw new Error("timeout stale drive requires inspect-before-retry")
|
|
1683
|
+
dependencies.persistReceipt(value, {
|
|
1684
|
+
schemaVersion: "sanctuary-timeout-stale-driver-receipt-v1",
|
|
1685
|
+
phase: existing ? "attempting" : "preparing",
|
|
1686
|
+
...value,
|
|
1687
|
+
})
|
|
1688
|
+
try {
|
|
1689
|
+
const response = await dependencies.runtime(value)
|
|
1690
|
+
if (response.state === "waiting") {
|
|
1691
|
+
exactKeys(response, ["state"], "timeout stale waiting response")
|
|
1692
|
+
dependencies.persistReceipt(value, { schemaVersion: "sanctuary-timeout-stale-driver-receipt-v1", phase: "waiting", ...value })
|
|
1693
|
+
return response
|
|
1694
|
+
}
|
|
1695
|
+
const receipt = requireTimeoutReceipt(response, value)
|
|
1696
|
+
dependencies.persistReceipt(value, receipt)
|
|
1697
|
+
return receipt
|
|
1698
|
+
} catch (error) {
|
|
1699
|
+
dependencies.persistReceipt(value, {
|
|
1700
|
+
schemaVersion: "sanctuary-timeout-stale-driver-receipt-v1",
|
|
1701
|
+
phase: existing ? "attempted_or_indeterminate" : "preparation_indeterminate",
|
|
1702
|
+
...value,
|
|
1703
|
+
})
|
|
1704
|
+
throw error
|
|
1705
|
+
}
|
|
1706
|
+
}
|
|
1707
|
+
|
|
1708
|
+
async function driveRestartContinuation(input, dependencies = {
|
|
1709
|
+
readReceipt: readInteractiveReceipt,
|
|
1710
|
+
persistReceipt: persistInteractiveReceipt,
|
|
1711
|
+
runtime: runInteractiveRuntimeOperation,
|
|
1712
|
+
restart: restartButlerForAcceptance,
|
|
1713
|
+
snapshot: () => containerRestartSnapshot(expectedImageId),
|
|
1714
|
+
sleep: (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)),
|
|
1715
|
+
}) {
|
|
1716
|
+
const value = canonicalInteractiveRequest(input, "unit-16m-restart-continuation")
|
|
1717
|
+
const existing = dependencies.readReceipt(value)
|
|
1718
|
+
if (existing) {
|
|
1719
|
+
if (existing.phase === "complete") return requireInteractiveReceipt(existing, value)
|
|
1720
|
+
let recoverable
|
|
1721
|
+
try { recoverable = requirePreparedRestartReceipt(existing, value) } catch { throw new Error("restart continuation requires inspect-before-retry") }
|
|
1722
|
+
const observed = object(await dependencies.snapshot(), "restart continuation recovery owner")
|
|
1723
|
+
if (observed.imageId !== `sha256:${recoverable.ownerImageDigest}` || observed.containerId !== recoverable.ownerContainerDigest
|
|
1724
|
+
|| observed.running !== true || observed.health !== "healthy" || observed.restartCount !== recoverable.restartCountBefore + 1) {
|
|
1725
|
+
throw new Error("restart continuation requires inspect-before-retry")
|
|
1726
|
+
}
|
|
1727
|
+
await waitForInteractiveRuntimeReady(value, dependencies)
|
|
1728
|
+
const reconciled = object(await dependencies.runtime("reconcile_restart_continuation", value), "restart continuation reconciliation")
|
|
1729
|
+
const receipt = { ...recoverable, ...reconciled, phase: "complete", restartCountAfter: observed.restartCount }
|
|
1730
|
+
delete receipt.restarted
|
|
1731
|
+
requireInteractiveReceipt(receipt, value)
|
|
1732
|
+
dependencies.persistReceipt(value, receipt)
|
|
1733
|
+
return receipt
|
|
1734
|
+
}
|
|
1735
|
+
const attempting = { schemaVersion: "sanctuary-interactive-driver-receipt-v2", phase: "attempting", ...value }
|
|
1736
|
+
dependencies.persistReceipt(value, attempting)
|
|
1737
|
+
let prepared
|
|
1738
|
+
try {
|
|
1739
|
+
prepared = object(await dependencies.runtime("prepare_restart_continuation", value), "restart continuation prepared receipt")
|
|
1740
|
+
const expected = ["schemaVersion", "phase", "label", "scenarioHandleDigest", "approvalIdDigest", "checkpointDigest", "suspendedSessionRevisionDigest", "approvalEpochBefore", "pendingDigestBefore", "indeterminateRecoveryObserved", "attemptedRecoveryReopened", "attemptedRecordDigest", "recoveredRecordDigest"]
|
|
1741
|
+
exactKeys(prepared, expected, "restart continuation prepared receipt")
|
|
1742
|
+
if (prepared.schemaVersion !== "sanctuary-interactive-driver-receipt-v2" || prepared.phase !== "prepared" || prepared.label !== value.label
|
|
1743
|
+
|| prepared.scenarioHandleDigest !== value.scenarioHandleDigest || ![prepared.approvalIdDigest, prepared.checkpointDigest, prepared.suspendedSessionRevisionDigest, prepared.pendingDigestBefore].every((item) => typeof item === "string" && SHA256.test(item))
|
|
1744
|
+
|| !Number.isSafeInteger(prepared.approvalEpochBefore) || prepared.approvalEpochBefore < 0 || prepared.indeterminateRecoveryObserved !== true
|
|
1745
|
+
|| prepared.attemptedRecoveryReopened !== true || !SHA256.test(prepared.attemptedRecordDigest) || !SHA256.test(prepared.recoveredRecordDigest)
|
|
1746
|
+
|| prepared.attemptedRecordDigest === prepared.recoveredRecordDigest) throw new Error("restart continuation prepared receipt is invalid")
|
|
1747
|
+
const ownerBefore = object(await dependencies.snapshot(), "restart continuation owner before")
|
|
1748
|
+
if (!IMAGE_ID.test(ownerBefore.imageId) || !SHA256.test(ownerBefore.containerId) || ownerBefore.running !== true || ownerBefore.health !== "healthy"
|
|
1749
|
+
|| !Number.isSafeInteger(ownerBefore.restartCount) || ownerBefore.restartCount < 0) throw new Error("restart continuation owner before is invalid")
|
|
1750
|
+
prepared = { ...prepared, ownerImageDigest: ownerBefore.imageId.slice(7), ownerContainerDigest: ownerBefore.containerId, restartCountBefore: ownerBefore.restartCount }
|
|
1751
|
+
dependencies.persistReceipt(value, prepared)
|
|
1752
|
+
const restarted = object(await dependencies.restart(value), "restart continuation owner restart")
|
|
1753
|
+
if (restarted.restarted !== true || restarted.restartInvocationCount !== 1 || !SHA256.test(restarted.ownerImageDigest) || !SHA256.test(restarted.ownerContainerDigest)
|
|
1754
|
+
|| restarted.ownerImageDigest !== prepared.ownerImageDigest || restarted.ownerContainerDigest !== prepared.ownerContainerDigest
|
|
1755
|
+
|| restarted.restartCountBefore !== prepared.restartCountBefore || restarted.restartCountAfter !== restarted.restartCountBefore + 1) throw new Error("restart continuation owner restart is invalid")
|
|
1756
|
+
await waitForInteractiveRuntimeReady(value, dependencies)
|
|
1757
|
+
const reconciled = object(await dependencies.runtime("reconcile_restart_continuation", value), "restart continuation reconciliation")
|
|
1758
|
+
const receipt = { ...prepared, ...reconciled, phase: "complete", ownerImageDigest: restarted.ownerImageDigest, ownerContainerDigest: restarted.ownerContainerDigest, restartCountBefore: restarted.restartCountBefore, restartCountAfter: restarted.restartCountAfter }
|
|
1759
|
+
requireInteractiveReceipt(receipt, value)
|
|
1760
|
+
dependencies.persistReceipt(value, receipt)
|
|
1761
|
+
return receipt
|
|
1762
|
+
} catch (error) {
|
|
1763
|
+
dependencies.persistReceipt(value, { ...(prepared ?? { schemaVersion: "sanctuary-interactive-driver-receipt-v2", ...value }), phase: "attempted_or_indeterminate" })
|
|
1764
|
+
throw error
|
|
1765
|
+
}
|
|
1766
|
+
}
|
|
1767
|
+
|
|
1768
|
+
async function dispatch(request, dependencies = {
|
|
1769
|
+
readBootId: () => readFileSync(BOOT_ID, "utf8"),
|
|
1770
|
+
containerSnapshot: () => containerSnapshot(expectedImageId),
|
|
1771
|
+
denialTargetSnapshot,
|
|
1772
|
+
containerOwnerSnapshot: () => containerOwnerSnapshot(expectedImageId),
|
|
1773
|
+
startHealthProbe,
|
|
1774
|
+
healthProbeStatus,
|
|
1775
|
+
recoverHealthProbe,
|
|
1776
|
+
healthProbeCoordinator,
|
|
1777
|
+
driveTimeoutStale,
|
|
1778
|
+
driveDuplicateCallbacks,
|
|
1779
|
+
driveRestartContinuation,
|
|
1780
|
+
ownerMutationCoordinator,
|
|
1781
|
+
interactiveRestartDriver,
|
|
1782
|
+
healthOwnerMutationActive,
|
|
1783
|
+
liveContainerProcessUser,
|
|
1784
|
+
liveContainerProcessIdentity,
|
|
1785
|
+
parseProcStartTime,
|
|
1786
|
+
productionProcessBindingDigest,
|
|
1787
|
+
rebootPreflightSnapshot: observeRebootPreflight,
|
|
1788
|
+
stopExactRebootOwner,
|
|
1789
|
+
verifyStoppedRebootOwner,
|
|
1790
|
+
commitHostReboot: () => new Promise((resolve, reject) => {
|
|
1791
|
+
const child = spawn(REBOOT, [], { cwd: "/", detached: true, stdio: "ignore", env: { PATH: "/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" } })
|
|
1792
|
+
child.once("error", reject)
|
|
1793
|
+
child.once("spawn", () => { child.unref(); resolve() })
|
|
1794
|
+
}),
|
|
1795
|
+
}) {
|
|
1796
|
+
const payload = object(request, "broker request")
|
|
1797
|
+
const operation = text(payload.operation, "operation")
|
|
1798
|
+
if (operation === "telegram_gateway_quiescence") {
|
|
1799
|
+
exactKeys(payload, ["operation", "targetId"], operation)
|
|
1800
|
+
if (payload.targetId !== TARGET_HOST) throw new Error("target host is invalid")
|
|
1801
|
+
return telegramGatewayQuiescence()
|
|
1802
|
+
}
|
|
1803
|
+
if (operation === "inventory_keys") {
|
|
1804
|
+
exactKeys(payload, ["operation", "targetServerId"], operation)
|
|
1805
|
+
if (payload.targetServerId !== TARGET_SERVER) throw new Error("target server is invalid")
|
|
1806
|
+
return { keys: inventoryRecords().map(publicRecord) }
|
|
1807
|
+
}
|
|
1808
|
+
if (operation === "read_key_record") {
|
|
1809
|
+
exactKeys(payload, ["operation", "targetServerId", "keyId"], operation)
|
|
1810
|
+
if (payload.targetServerId !== TARGET_SERVER) throw new Error("target server is invalid")
|
|
1811
|
+
const id = text(payload.keyId, "key id", KEY_ID)
|
|
1812
|
+
const matches = inventoryRecords().filter((record) => record.id === id)
|
|
1813
|
+
if (matches.length !== 1) throw new Error("exact key id is absent or ambiguous")
|
|
1814
|
+
return matches[0]
|
|
1815
|
+
}
|
|
1816
|
+
if (operation === "create_key") {
|
|
1817
|
+
exactKeys(payload, ["operation", "targetServerId", "name", "permissions"], operation)
|
|
1818
|
+
if (payload.targetServerId !== TARGET_SERVER) throw new Error("target server is invalid")
|
|
1819
|
+
const name = text(payload.name, "key name", KEY_NAME)
|
|
1820
|
+
const requested = permissions(payload.permissions)
|
|
1821
|
+
return await createUnraidKey(name, requested)
|
|
1822
|
+
}
|
|
1823
|
+
if (operation === "acknowledge_key_storage") {
|
|
1824
|
+
exactKeys(payload, ["operation", "targetServerId", "keyId"], operation)
|
|
1825
|
+
if (payload.targetServerId !== TARGET_SERVER) throw new Error("target server is invalid")
|
|
1826
|
+
return acknowledgeStoredUnraidKey(text(payload.keyId, "key id", KEY_ID))
|
|
1827
|
+
}
|
|
1828
|
+
if (operation === "revoke_key") {
|
|
1829
|
+
exactKeys(payload, ["operation", "targetServerId", "keyId"], operation)
|
|
1830
|
+
if (payload.targetServerId !== TARGET_SERVER) throw new Error("target server is invalid")
|
|
1831
|
+
const id = text(payload.keyId, "key id", KEY_ID)
|
|
1832
|
+
const matches = inventoryRecords().filter((record) => record.id === id)
|
|
1833
|
+
if (matches.length !== 1) throw new Error("exact key id is absent or ambiguous")
|
|
1834
|
+
const record = matches[0]
|
|
1835
|
+
persistRecovery(record)
|
|
1836
|
+
const deleted = runUnraid(["apikey", "--name", record.name, "--delete", "--json"])
|
|
1837
|
+
if (deleted.deleted !== 1 || !Array.isArray(deleted.keys) || deleted.keys.length !== 1
|
|
1838
|
+
|| deleted.keys[0]?.id !== id || deleted.keys[0]?.name !== record.name) throw new Error("exact key revoke attestation failed")
|
|
1839
|
+
return { revoked: true, id }
|
|
1840
|
+
}
|
|
1841
|
+
if (operation === "probe_revoked_key") {
|
|
1842
|
+
exactKeys(payload, ["operation", "targetServerId", "keyId"], operation)
|
|
1843
|
+
if (payload.targetServerId !== TARGET_SERVER) throw new Error("target server is invalid")
|
|
1844
|
+
return await probeRevokedUnraidKey(text(payload.keyId, "key id", KEY_ID))
|
|
1845
|
+
}
|
|
1846
|
+
if (operation === "request_reboot") {
|
|
1847
|
+
exactKeys(payload, ["operation", "targetId", "idempotencyKey", "preflightDigest", "processBindingDigest"], operation)
|
|
1848
|
+
if (payload.targetId !== TARGET_HOST) throw new Error("target host is invalid")
|
|
1849
|
+
const key = text(payload.idempotencyKey, "idempotency key", /^[0-9a-f]{32}$/u)
|
|
1850
|
+
const preflightDigest = text(payload.preflightDigest, "preflight digest", SHA256)
|
|
1851
|
+
const processBindingDigest = text(payload.processBindingDigest, "process binding digest", SHA256)
|
|
1852
|
+
const requestId = createHash("sha256").update(`sanctuary-reboot\0${key}`).digest("hex")
|
|
1853
|
+
const reservationId = createHash("sha256").update(`sanctuary-reboot-reservation\0${requestId}`).digest("hex")
|
|
1854
|
+
const reserve = dependencies.ownerMutationCoordinator?.reserveReboot
|
|
1855
|
+
if (!reserve) throw new Error("reboot reservation coordinator is unavailable")
|
|
1856
|
+
await reserve.call(dependencies.ownerMutationCoordinator, reservationId, processBindingDigest, async () => {
|
|
1857
|
+
const preflight = object(dependencies.rebootPreflightSnapshot(), "reboot preflight")
|
|
1858
|
+
if (preflight.digest !== preflightDigest) throw new Error("reboot preflight changed before commit")
|
|
1859
|
+
if (preflight.safe !== true) throw new Error("reboot preflight is unsafe")
|
|
1860
|
+
const owner = object(await dependencies.containerSnapshot(), "reboot reservation production owner")
|
|
1861
|
+
if (owner.processBindingDigest !== processBindingDigest) throw new Error("production process generation changed before reboot reservation")
|
|
1862
|
+
})
|
|
1863
|
+
const prebootId = text(dependencies.readBootId().trim(), "boot id", /^[A-Za-z0-9-]{4,128}$/u)
|
|
1864
|
+
return { accepted: true, targetId: TARGET_HOST, requestId, reservationId, prebootId, preflightDigest, processBindingDigest, staged: true }
|
|
1865
|
+
}
|
|
1866
|
+
if (operation === "stop_reboot_owner") {
|
|
1867
|
+
exactKeys(payload, ["operation", "targetId", "requestId", "reservationId", "processBindingDigest"], operation)
|
|
1868
|
+
if (payload.targetId !== TARGET_HOST) throw new Error("target host is invalid")
|
|
1869
|
+
const requestId = text(payload.requestId, "reboot request id", SHA256)
|
|
1870
|
+
const reservationId = text(payload.reservationId, "reboot reservation", SHA256)
|
|
1871
|
+
const processBindingDigest = text(payload.processBindingDigest, "process binding digest", SHA256)
|
|
1872
|
+
if (reservationId !== createHash("sha256").update(`sanctuary-reboot-reservation\0${requestId}`).digest("hex")) throw new Error("reboot request reservation binding is invalid")
|
|
1873
|
+
const stop = dependencies.ownerMutationCoordinator?.stopRebootOwner
|
|
1874
|
+
if (!stop || !dependencies.stopExactRebootOwner) throw new Error("exact reboot owner stop is unavailable")
|
|
1875
|
+
await stop.call(dependencies.ownerMutationCoordinator, reservationId, processBindingDigest, () => dependencies.stopExactRebootOwner(processBindingDigest))
|
|
1876
|
+
return { stopped: true, targetId: TARGET_HOST, requestId, reservationId, processBindingDigest }
|
|
1877
|
+
}
|
|
1878
|
+
if (operation === "commit_reboot") {
|
|
1879
|
+
exactKeys(payload, ["operation", "targetId", "requestId", "reservationId", "processBindingDigest"], operation)
|
|
1880
|
+
if (payload.targetId !== TARGET_HOST) throw new Error("target host is invalid")
|
|
1881
|
+
const requestId = text(payload.requestId, "reboot request id", SHA256)
|
|
1882
|
+
const reservationId = text(payload.reservationId, "reboot reservation", SHA256)
|
|
1883
|
+
const processBindingDigest = text(payload.processBindingDigest, "process binding digest", SHA256)
|
|
1884
|
+
if (reservationId !== createHash("sha256").update(`sanctuary-reboot-reservation\0${requestId}`).digest("hex")) throw new Error("reboot request reservation binding is invalid")
|
|
1885
|
+
const commit = dependencies.ownerMutationCoordinator?.commitReboot
|
|
1886
|
+
if (!commit) throw new Error("reboot reservation coordinator is unavailable")
|
|
1887
|
+
await commit.call(dependencies.ownerMutationCoordinator, reservationId, processBindingDigest, async (stoppedProof, markAttempted) => {
|
|
1888
|
+
await dependencies.verifyStoppedRebootOwner(stoppedProof)
|
|
1889
|
+
const preflight = object(dependencies.rebootPreflightSnapshot(), "final reboot preflight")
|
|
1890
|
+
if (preflight.safe !== true || preflight.arrayReady !== true || preflight.parityActive !== false || preflight.moverActive !== false || preflight.mutationActive !== false) {
|
|
1891
|
+
throw new Error("final reboot preflight is unsafe")
|
|
1892
|
+
}
|
|
1893
|
+
await dependencies.verifyStoppedRebootOwner(stoppedProof)
|
|
1894
|
+
markAttempted()
|
|
1895
|
+
await dependencies.commitHostReboot()
|
|
1896
|
+
})
|
|
1897
|
+
return { committed: true, targetId: TARGET_HOST, requestId, reservationId, processBindingDigest }
|
|
1898
|
+
}
|
|
1899
|
+
if (operation === "reboot_preflight_snapshot") {
|
|
1900
|
+
exactKeys(payload, ["operation", "targetId", "processBindingDigest"], operation)
|
|
1901
|
+
if (payload.targetId !== TARGET_HOST) throw new Error("target host is invalid")
|
|
1902
|
+
const processBindingDigest = text(payload.processBindingDigest, "process binding digest", SHA256)
|
|
1903
|
+
const owner = object(await dependencies.containerSnapshot(), "reboot preflight production owner")
|
|
1904
|
+
if (owner.processBindingDigest !== processBindingDigest) throw new Error("production process generation changed before reboot preflight")
|
|
1905
|
+
return { ...dependencies.rebootPreflightSnapshot(), processBindingDigest }
|
|
1906
|
+
}
|
|
1907
|
+
if (operation === "container_snapshot") {
|
|
1908
|
+
exactKeys(payload, ["operation", "targetId"], operation)
|
|
1909
|
+
if (payload.targetId !== TARGET_HOST) throw new Error("target host is invalid")
|
|
1910
|
+
return await dependencies.containerSnapshot()
|
|
1911
|
+
}
|
|
1912
|
+
if (operation === "denial_target_snapshot") {
|
|
1913
|
+
exactKeys(payload, ["operation", "targetId"], operation)
|
|
1914
|
+
if (payload.targetId !== TARGET_HOST) throw new Error("target host is invalid")
|
|
1915
|
+
if (!dependencies.denialTargetSnapshot) throw new Error("denial target snapshot is unavailable")
|
|
1916
|
+
return await dependencies.denialTargetSnapshot()
|
|
1917
|
+
}
|
|
1918
|
+
if (operation === "start_health_probe" || operation === "health_probe_status" || operation === "recover_health_probe") {
|
|
1919
|
+
exactKeys(payload, ["operation", "targetId", "label", "scenarioHandleDigest"], operation)
|
|
1920
|
+
if (payload.targetId !== TARGET_HOST) throw new Error("target host is invalid")
|
|
1921
|
+
const request = canonicalHealthProbeRequest({ label: payload.label, scenarioHandleDigest: payload.scenarioHandleDigest })
|
|
1922
|
+
if (operation === "start_health_probe") {
|
|
1923
|
+
const start = async () => {
|
|
1924
|
+
const snapshot = object(await dependencies.containerSnapshot(), "health probe production owner")
|
|
1925
|
+
if (snapshot.running !== true || snapshot.health !== "healthy") throw new Error("health probe requires a healthy production owner")
|
|
1926
|
+
const coordinates = healthProbeCoordinates(payload, snapshot)
|
|
1927
|
+
if (!dependencies.startHealthProbe) throw new Error("health probe start is unavailable")
|
|
1928
|
+
return await dependencies.startHealthProbe(coordinates)
|
|
1929
|
+
}
|
|
1930
|
+
if (dependencies.ownerMutationCoordinator) return await dependencies.ownerMutationCoordinator.healthStart(request.scenarioHandleDigest, start)
|
|
1931
|
+
return dependencies.healthProbeCoordinator ? await dependencies.healthProbeCoordinator.start(request.scenarioHandleDigest, start) : await start()
|
|
1932
|
+
}
|
|
1933
|
+
if (operation === "health_probe_status") {
|
|
1934
|
+
if (!dependencies.healthProbeStatus) throw new Error("health probe status is unavailable")
|
|
1935
|
+
const status = () => dependencies.healthProbeStatus(request)
|
|
1936
|
+
return dependencies.ownerMutationCoordinator ? await dependencies.ownerMutationCoordinator.healthOperation(request.scenarioHandleDigest, status) : await status()
|
|
1937
|
+
}
|
|
1938
|
+
const recover = async () => {
|
|
1939
|
+
if (!dependencies.recoverHealthProbe) throw new Error("health probe recovery is unavailable")
|
|
1940
|
+
const ownerSnapshot = object(await (dependencies.containerOwnerSnapshot ?? dependencies.containerSnapshot)(), "health probe production owner")
|
|
1941
|
+
const coordinates = healthProbeCoordinates(payload, ownerSnapshot)
|
|
1942
|
+
return await dependencies.recoverHealthProbe(coordinates)
|
|
1943
|
+
}
|
|
1944
|
+
if (dependencies.ownerMutationCoordinator) return await dependencies.ownerMutationCoordinator.healthRecover(request.scenarioHandleDigest, recover)
|
|
1945
|
+
return dependencies.healthProbeCoordinator ? await dependencies.healthProbeCoordinator.recover(request.scenarioHandleDigest, recover) : await recover()
|
|
1946
|
+
}
|
|
1947
|
+
if (operation === "drive_timeout_stale" || operation === "drive_duplicate_callbacks" || operation === "drive_restart_continuation") {
|
|
1948
|
+
exactKeys(payload, ["operation", "targetId", "label", "scenarioHandleDigest"], operation)
|
|
1949
|
+
if (payload.targetId !== TARGET_HOST) throw new Error("target host is invalid")
|
|
1950
|
+
const expectedLabel = operation === "drive_timeout_stale" ? "unit-16k-timeout-stale"
|
|
1951
|
+
: operation === "drive_duplicate_callbacks" ? "unit-16l-duplicate-callback" : "unit-16m-restart-continuation"
|
|
1952
|
+
const input = canonicalInteractiveRequest({ label: payload.label, scenarioHandleDigest: payload.scenarioHandleDigest }, expectedLabel)
|
|
1953
|
+
if (dependencies.healthOwnerMutationActive?.()) throw new Error("owner mutation refused while durable health probe artifacts are active")
|
|
1954
|
+
const drive = operation === "drive_timeout_stale" ? dependencies.driveTimeoutStale
|
|
1955
|
+
: operation === "drive_duplicate_callbacks" ? dependencies.driveDuplicateCallbacks : dependencies.driveRestartContinuation
|
|
1956
|
+
if (!drive) throw new Error("interactive production runtime driver is unavailable")
|
|
1957
|
+
const execute = async () => operation === "drive_timeout_stale"
|
|
1958
|
+
? await drive(input)
|
|
1959
|
+
: requireInteractiveReceipt(await drive(input), input)
|
|
1960
|
+
if (operation === "drive_restart_continuation") {
|
|
1961
|
+
if (!dependencies.interactiveRestartDriver) throw new Error("interactive restart driver is unavailable")
|
|
1962
|
+
const owned = () => dependencies.ownerMutationCoordinator ? dependencies.ownerMutationCoordinator.interactive(execute) : execute()
|
|
1963
|
+
const response = dependencies.interactiveRestartDriver.poll(input, owned)
|
|
1964
|
+
if (response.state === "waiting") Object.defineProperty(response, ASYNC_RESTART_SCENARIO, { value: input.scenarioHandleDigest })
|
|
1965
|
+
return response
|
|
1966
|
+
}
|
|
1967
|
+
return dependencies.ownerMutationCoordinator ? await dependencies.ownerMutationCoordinator.interactive(execute) : await execute()
|
|
1968
|
+
}
|
|
1969
|
+
throw new Error("host broker operation is not whitelisted")
|
|
1970
|
+
}
|
|
1971
|
+
|
|
1972
|
+
function writeClosedInventory(file) {
|
|
1973
|
+
const value = { keys: inventoryRecords().map(publicRecord) }
|
|
1974
|
+
const fd = openSync(file, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL, 0o600)
|
|
1975
|
+
try { writeFileSync(fd, `${JSON.stringify(value)}\n`); fsyncSync(fd) } finally { closeSync(fd) }
|
|
1976
|
+
chownSync(file, 0, 0)
|
|
1977
|
+
}
|
|
1978
|
+
|
|
1979
|
+
function createBrokerServer(dispatchRequest = dispatch) {
|
|
1980
|
+
const dispatchDrain = createDispatchDrain()
|
|
1981
|
+
const connections = new Set()
|
|
1982
|
+
const server = createServer({ allowHalfOpen: true }, (connection) => {
|
|
1983
|
+
connections.add(connection)
|
|
1984
|
+
connection.setTimeout(30_000, () => connection.destroy())
|
|
1985
|
+
connection.once("close", () => connections.delete(connection))
|
|
1986
|
+
let input = ""
|
|
1987
|
+
connection.setEncoding("utf8")
|
|
1988
|
+
connection.on("data", (chunk) => {
|
|
1989
|
+
input += chunk
|
|
1990
|
+
if (Buffer.byteLength(input) > MAX_REQUEST) connection.destroy()
|
|
1991
|
+
})
|
|
1992
|
+
connection.on("end", async () => {
|
|
1993
|
+
try {
|
|
1994
|
+
const result = await dispatchDrain.run(() => dispatchRequest(JSON.parse(input)))
|
|
1995
|
+
const scenario = result?.[ASYNC_RESTART_SCENARIO]
|
|
1996
|
+
const completed = scenario ? armRestartAfterResponseClosed(connection, scenario) : undefined
|
|
1997
|
+
connection.end(`${JSON.stringify({ ok: true, result })}\n`, completed)
|
|
1998
|
+
} catch { connection.end(`${JSON.stringify({ ok: false, error: "host operation failed" })}\n`) }
|
|
1999
|
+
})
|
|
2000
|
+
})
|
|
2001
|
+
const destroyConnections = () => { for (const connection of connections) connection.destroy() }
|
|
2002
|
+
return { server, dispatchDrain, destroyConnections }
|
|
2003
|
+
}
|
|
2004
|
+
|
|
2005
|
+
async function main() {
|
|
2006
|
+
if (process.getuid?.() !== 0) throw new Error("host broker must run as root")
|
|
2007
|
+
const [profileName, targetContainerId, socket, closedInventory, expectedImage, initialSnapshot] = process.argv.slice(2)
|
|
2008
|
+
if (!profileName || !targetContainerId || !socket || !closedInventory || !expectedImage || !initialSnapshot || process.argv.length !== 8) throw new Error("usage: broker <staging|final> <target-container-id> <socket> <closed-inventory> <expected-image-id> <initial-snapshot>")
|
|
2009
|
+
activeProfile = targetProfile(profileName)
|
|
2010
|
+
activeContainer = activeProfile.containerName
|
|
2011
|
+
activeContainerId = text(targetContainerId, "attested target container id", SHA256)
|
|
2012
|
+
expectedImageId = text(expectedImage, "expected image id", IMAGE_ID)
|
|
2013
|
+
rmSync(socket, { force: true })
|
|
2014
|
+
writeClosedInventory(closedInventory)
|
|
2015
|
+
const snapshot = await containerSnapshot(expectedImageId)
|
|
2016
|
+
const snapshotFd = openSync(initialSnapshot, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL, 0o600)
|
|
2017
|
+
try { writeFileSync(snapshotFd, `${JSON.stringify(snapshot)}\n`); fsyncSync(snapshotFd) } finally { closeSync(snapshotFd) }
|
|
2018
|
+
chownSync(initialSnapshot, 0, 0)
|
|
2019
|
+
const { server, dispatchDrain, destroyConnections } = createBrokerServer()
|
|
2020
|
+
server.listen(socket, () => {
|
|
2021
|
+
chownSync(socket, 0, 10001)
|
|
2022
|
+
chmodSync(socket, 0o660)
|
|
2023
|
+
})
|
|
2024
|
+
let shuttingDown = false
|
|
2025
|
+
const shutdown = async () => {
|
|
2026
|
+
if (shuttingDown) return
|
|
2027
|
+
shuttingDown = true
|
|
2028
|
+
server.close()
|
|
2029
|
+
destroyConnections()
|
|
2030
|
+
await dispatchDrain.stopAndDrain()
|
|
2031
|
+
await interactiveRestartDriver.stopAndDrain()
|
|
2032
|
+
let failed = false
|
|
2033
|
+
for (const record of [...activeHealthProbes.values()]) {
|
|
2034
|
+
try { await recoverHealthProbe(record.input) } catch { failed = true }
|
|
2035
|
+
}
|
|
2036
|
+
process.exitCode = failed ? 1 : 0
|
|
2037
|
+
}
|
|
2038
|
+
process.once("SIGTERM", () => { void shutdown() })
|
|
2039
|
+
process.once("SIGINT", () => { void shutdown() })
|
|
2040
|
+
}
|
|
2041
|
+
|
|
2042
|
+
if (process.argv[1]?.endsWith("sanctuary-unit16-host-broker.mjs")) {
|
|
2043
|
+
main().catch(() => { process.exitCode = 1 })
|
|
2044
|
+
}
|
|
2045
|
+
|
|
2046
|
+
export {
|
|
2047
|
+
acknowledgeStoredUnraidKey,
|
|
2048
|
+
assertStableContainerProcess,
|
|
2049
|
+
attestHealthProbeProcessAbsent,
|
|
2050
|
+
armRestartAfterResponseClosed,
|
|
2051
|
+
completeHealthProbeFromReceipt,
|
|
2052
|
+
createDispatchDrain,
|
|
2053
|
+
createBrokerServer,
|
|
2054
|
+
createHealthProbeOperationCoordinator,
|
|
2055
|
+
createOwnerMutationCoordinator,
|
|
2056
|
+
createUnraidKey,
|
|
2057
|
+
createInteractiveRestartDriver,
|
|
2058
|
+
dispatch,
|
|
2059
|
+
denialTargetSnapshot,
|
|
2060
|
+
healthProbeArtifactDisposition,
|
|
2061
|
+
healthProbeDockerArgs,
|
|
2062
|
+
healthProbeOperationBudgets,
|
|
2063
|
+
finalizeHealthProbeAfterAttestation,
|
|
2064
|
+
healthOwnerMutationActive,
|
|
2065
|
+
liveContainerProcessUser,
|
|
2066
|
+
liveContainerProcessIdentity,
|
|
2067
|
+
parseProcStartTime,
|
|
2068
|
+
productionProcessBindingDigest,
|
|
2069
|
+
observeRebootPreflight,
|
|
2070
|
+
parseVaultStatus,
|
|
2071
|
+
optionalStoppedContainerExact,
|
|
2072
|
+
probeRevokedUnraidKey,
|
|
2073
|
+
queryGraphqlAutostart,
|
|
2074
|
+
readBoundedProcStatus,
|
|
2075
|
+
driveDuplicateCallbacks,
|
|
2076
|
+
driveTimeoutStale,
|
|
2077
|
+
driveRestartContinuation,
|
|
2078
|
+
restartButlerForAcceptance,
|
|
2079
|
+
runInteractiveDriver,
|
|
2080
|
+
recoverAfterHealthProbeTermination,
|
|
2081
|
+
requireStableHealthProbeOwner,
|
|
2082
|
+
requireHealthProbeCompleteAttestation,
|
|
2083
|
+
terminateHealthProbeChild,
|
|
2084
|
+
}
|