@cohortapp/agent-sdk 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/commands/init-agent.md +104 -0
- package/.claude/commands/init-maestro.md +1187 -0
- package/.claude/settings.json +161 -0
- package/.env.example +216 -0
- package/README.md +632 -0
- package/agents/browser-operator/agent.md +52 -0
- package/agents/calendar-ops/agent.md +50 -0
- package/agents/communications/agent.md +96 -0
- package/agents/decision-log/agent.md +65 -0
- package/agents/desktop-operator/agent.md +59 -0
- package/agents/gmail-operator/agent.md +62 -0
- package/agents/inbound-dispatcher/agent.md +66 -0
- package/agents/inbox-processor/agent.md +39 -0
- package/agents/pmo-execution/agent.md +60 -0
- package/agents/session-spawner/agent.md +64 -0
- package/agents/slack-operator/agent.md +60 -0
- package/agents/whatsapp-operator/agent.md +60 -0
- package/agents/workflow-automation/agent.md +61 -0
- package/archetypes/altitudes/c-suite.yaml +58 -0
- package/archetypes/altitudes/founder.yaml +68 -0
- package/archetypes/altitudes/senior-manager.yaml +63 -0
- package/archetypes/altitudes/svp.yaml +60 -0
- package/archetypes/altitudes/vp.yaml +50 -0
- package/archetypes/archetype.schema.json +77 -0
- package/archetypes/base.yaml +47 -0
- package/archetypes/capabilities/commercial-leader.yaml +159 -0
- package/archetypes/capabilities/compliance-officer.yaml +159 -0
- package/archetypes/capabilities/executive-operator.yaml +169 -0
- package/archetypes/capabilities/finance-leader.yaml +162 -0
- package/archetypes/capabilities/operations-leader.yaml +154 -0
- package/archetypes/capabilities/product-leader.yaml +148 -0
- package/archetypes/capabilities/technical-leader.yaml +146 -0
- package/archetypes/functions/commercial-leader.yaml +62 -0
- package/archetypes/functions/compliance-officer.yaml +64 -0
- package/archetypes/functions/executive-operator.yaml +70 -0
- package/archetypes/functions/finance-leader.yaml +70 -0
- package/archetypes/functions/operations-leader.yaml +62 -0
- package/archetypes/functions/product-leader.yaml +61 -0
- package/archetypes/functions/technical-leader.yaml +57 -0
- package/bin/cohort-mcp.mjs +81 -0
- package/bin/maestro.mjs +3516 -0
- package/bin/maestro.test.mjs +1015 -0
- package/desktop-control/README.md +56 -0
- package/desktop-control/app-profiles/gmail.yaml +120 -0
- package/desktop-control/app-profiles/slack.yaml +315 -0
- package/desktop-control/app-profiles/whatsapp.yaml +107 -0
- package/docs/architecture/agent-topology.md +2239 -0
- package/docs/architecture/archetype-agent-factory.md +110 -0
- package/docs/architecture/collective-memory-and-org-mesh.md +115 -0
- package/docs/architecture/continuous-monitoring.md +221 -0
- package/docs/architecture/mcp-capability-map.md +585 -0
- package/docs/architecture/system-architecture.md +1272 -0
- package/docs/company-context/README.md +40 -0
- package/docs/guides/agent-persona-setup.md +600 -0
- package/docs/guides/agents-observe-setup.md +64 -0
- package/docs/guides/billing-console-keys.md +88 -0
- package/docs/guides/ccxray-diagnostics.md +65 -0
- package/docs/guides/channel-bus.md +127 -0
- package/docs/guides/claude-mem-setup.md +79 -0
- package/docs/guides/claude-pace-setup.md +56 -0
- package/docs/guides/claudraband-sessions.md +98 -0
- package/docs/guides/clawteam-swarm.md +116 -0
- package/docs/guides/code-review-graph-setup.md +86 -0
- package/docs/guides/email-setup.md +431 -0
- package/docs/guides/mac-mini.md +119 -0
- package/docs/guides/media-generation-setup.md +349 -0
- package/docs/guides/model-routing.md +162 -0
- package/docs/guides/observability-otel.md +265 -0
- package/docs/guides/org-onboarding.md +132 -0
- package/docs/guides/outbound-governance-setup.md +437 -0
- package/docs/guides/pdf-generation-setup.md +315 -0
- package/docs/guides/poller-daemon-setup.md +563 -0
- package/docs/guides/rag-context-setup.md +459 -0
- package/docs/guides/self-optimization-pattern.md +82 -0
- package/docs/guides/setup-wizard.md +178 -0
- package/docs/guides/slack-setup.md +350 -0
- package/docs/guides/telegram-setup.md +227 -0
- package/docs/guides/twilio-subaccounts-setup.md +223 -0
- package/docs/guides/verification.md +128 -0
- package/docs/guides/voice-mode.md +188 -0
- package/docs/guides/voice-sms-setup.md +698 -0
- package/docs/guides/webhook-relay-setup.md +349 -0
- package/docs/guides/whatsapp-setup.md +288 -0
- package/docs/prompts/board-pack-cover-template.md +36 -0
- package/docs/prompts/decision-recommendation-template.md +88 -0
- package/docs/prompts/followup-message-template.md +141 -0
- package/docs/prompts/investor-letter-template.md +52 -0
- package/docs/prompts/morning-brief-template.md +82 -0
- package/docs/prompts/presentation-template.md +58 -0
- package/docs/prompts/weekly-strategic-memo-template.md +104 -0
- package/docs/research/hallucinated-tool-output-investigation.md +151 -0
- package/docs/runbooks/backup-restore.md +205 -0
- package/docs/runbooks/cohort-cutover.md +129 -0
- package/docs/runbooks/fleet-operations.md +200 -0
- package/docs/runbooks/incident-response.md +226 -0
- package/docs/runbooks/mac-mini-bootstrap.md +431 -0
- package/docs/runbooks/perpetual-operations.md +509 -0
- package/docs/runbooks/recovery-and-failover.md +260 -0
- package/framework-features.json +267 -0
- package/ingest/README.md +87 -0
- package/lib/action-executor.js +689 -0
- package/lib/action-executor.test.mjs +871 -0
- package/lib/agent-root.mjs +37 -0
- package/lib/archetype.mjs +236 -0
- package/lib/archetype.test.mjs +132 -0
- package/lib/autonomy.mjs +114 -0
- package/lib/autonomy.test.mjs +66 -0
- package/lib/backlog.mjs +358 -0
- package/lib/backlog.test.mjs +266 -0
- package/lib/budget-guard.mjs +279 -0
- package/lib/budget-guard.test.mjs +291 -0
- package/lib/cadence-bus-schedule.test.mjs +194 -0
- package/lib/cadence-bus.mjs +1120 -0
- package/lib/cadence-bus.test.mjs +720 -0
- package/lib/cadences.mjs +205 -0
- package/lib/cadences.test.mjs +125 -0
- package/lib/capability.mjs +154 -0
- package/lib/capability.test.mjs +78 -0
- package/lib/channels/base-adapter.mjs +719 -0
- package/lib/channels/base-adapter.test.mjs +590 -0
- package/lib/channels/channel.mjs +128 -0
- package/lib/channels/channels.test.mjs +371 -0
- package/lib/channels/contract.mjs +215 -0
- package/lib/channels/contract.test.mjs +137 -0
- package/lib/channels/conversation-resolver.mjs +95 -0
- package/lib/channels/gmail/adapter.mjs +87 -0
- package/lib/channels/inbox-item.mjs +255 -0
- package/lib/channels/inbox-item.test.mjs +335 -0
- package/lib/channels/index.mjs +94 -0
- package/lib/channels/orgmail/adapter.mjs +353 -0
- package/lib/channels/orgmail/adapter.test.mjs +311 -0
- package/lib/channels/pairing.mjs +363 -0
- package/lib/channels/pairing.test.mjs +270 -0
- package/lib/channels/registry.mjs +164 -0
- package/lib/channels/slack/adapter.mjs +317 -0
- package/lib/channels/slack-adapter.test.mjs +212 -0
- package/lib/channels/sms/adapter.mjs +43 -0
- package/lib/channels/telegram/adapter.mjs +432 -0
- package/lib/channels/telegram-adapter.test.mjs +306 -0
- package/lib/channels/voice/adapter.mjs +301 -0
- package/lib/channels/voice/adapter.test.mjs +278 -0
- package/lib/channels/whatsapp/adapter-baileys.mjs +587 -0
- package/lib/channels/whatsapp/adapter-baileys.test.mjs +359 -0
- package/lib/channels/whatsapp/adapter-twilio.mjs +65 -0
- package/lib/channels/whatsapp/baileys-typing.test.mjs +154 -0
- package/lib/charter.mjs +256 -0
- package/lib/charter.test.mjs +89 -0
- package/lib/claude-bin.mjs +134 -0
- package/lib/claude-bin.test.mjs +75 -0
- package/lib/collective/capture.mjs +185 -0
- package/lib/collective/capture.test.mjs +121 -0
- package/lib/collective/cards.mjs +201 -0
- package/lib/collective/cards.test.mjs +114 -0
- package/lib/collective/config.mjs +186 -0
- package/lib/collective/config.test.mjs +123 -0
- package/lib/collective/global-config.mjs +113 -0
- package/lib/collective/global-config.test.mjs +75 -0
- package/lib/collective/presence.mjs +201 -0
- package/lib/collective/presence.test.mjs +95 -0
- package/lib/collective/recall.mjs +215 -0
- package/lib/collective/recall.test.mjs +116 -0
- package/lib/comms/send-gate.mjs +554 -0
- package/lib/comms/send-gate.test.mjs +577 -0
- package/lib/comms.mjs +67 -0
- package/lib/comms.test.mjs +41 -0
- package/lib/diagnostics/alerts.mjs +424 -0
- package/lib/diagnostics/alerts.test.mjs +318 -0
- package/lib/diagnostics/backup-freshness.mjs +188 -0
- package/lib/diagnostics/backup-freshness.test.mjs +185 -0
- package/lib/diagnostics/counters.mjs +269 -0
- package/lib/diagnostics/counters.test.mjs +206 -0
- package/lib/diagnostics/events.mjs +188 -0
- package/lib/diagnostics/events.test.mjs +290 -0
- package/lib/diagnostics/otel.mjs +237 -0
- package/lib/diagnostics/otel.test.mjs +196 -0
- package/lib/diagnostics/trace.mjs +216 -0
- package/lib/diagnostics/trace.test.mjs +251 -0
- package/lib/env-compat.mjs +74 -0
- package/lib/env-compat.test.mjs +104 -0
- package/lib/feature-init.mjs +331 -0
- package/lib/fs-atomic.mjs +112 -0
- package/lib/fs-atomic.test.mjs +72 -0
- package/lib/fs-ownership.mjs +111 -0
- package/lib/fs-ownership.test.mjs +158 -0
- package/lib/hooks/bus.mjs +347 -0
- package/lib/hooks/bus.test.mjs +387 -0
- package/lib/index.js +16 -0
- package/lib/learning/config.mjs +106 -0
- package/lib/learning/config.test.mjs +75 -0
- package/lib/learning/counters.mjs +156 -0
- package/lib/learning/counters.test.mjs +69 -0
- package/lib/learning/curator-consolidate.test.mjs +238 -0
- package/lib/learning/curator.mjs +453 -0
- package/lib/learning/curator.test.mjs +106 -0
- package/lib/learning/log.mjs +40 -0
- package/lib/learning/reflect.mjs +534 -0
- package/lib/learning/reflect.test.mjs +0 -0
- package/lib/learning/session-index.mjs +352 -0
- package/lib/learning/session-index.test.mjs +125 -0
- package/lib/learning/skill-writer.mjs +474 -0
- package/lib/learning/skill-writer.test.mjs +210 -0
- package/lib/mcp/server.mjs +328 -0
- package/lib/mcp/server.test.mjs +400 -0
- package/lib/model-router/auth-profiles.mjs +758 -0
- package/lib/model-router/auth-profiles.test.mjs +580 -0
- package/lib/model-router/catalog/anthropic.yaml +153 -0
- package/lib/model-router/catalog/deepseek.yaml +86 -0
- package/lib/model-router/catalog/moonshot.yaml +81 -0
- package/lib/model-router/catalog/qwen.yaml +114 -0
- package/lib/model-router/catalog.mjs +925 -0
- package/lib/model-router/catalog.test.mjs +385 -0
- package/lib/model-router/economics.mjs +564 -0
- package/lib/model-router/economics.test.mjs +344 -0
- package/lib/model-router/failover.mjs +298 -0
- package/lib/model-router/failover.test.mjs +439 -0
- package/lib/model-router/health.mjs +453 -0
- package/lib/model-router/health.test.mjs +338 -0
- package/lib/model-router/integration-coverage.test.mjs +829 -0
- package/lib/model-router/integration.test.mjs +564 -0
- package/lib/model-router/ledger.mjs +402 -0
- package/lib/model-router/ledger.test.mjs +382 -0
- package/lib/model-router/llm-task.mjs +515 -0
- package/lib/model-router/llm-task.test.mjs +392 -0
- package/lib/model-router/org-credentials.mjs +260 -0
- package/lib/model-router/org-credentials.test.mjs +265 -0
- package/lib/model-router/pricing-refresh.mjs +463 -0
- package/lib/model-router/pricing-refresh.test.mjs +286 -0
- package/lib/model-router/reconcile.mjs +429 -0
- package/lib/model-router/reconcile.test.mjs +316 -0
- package/lib/model-router/repair.mjs +471 -0
- package/lib/model-router/repair.test.mjs +180 -0
- package/lib/model-router/resolve.mjs +1206 -0
- package/lib/model-router/spawn.mjs +497 -0
- package/lib/model-router/spawn.test.mjs +425 -0
- package/lib/model-router/taxonomy.mjs +893 -0
- package/lib/model-router/taxonomy.test.mjs +410 -0
- package/lib/model-router.mjs +677 -0
- package/lib/model-router.test.mjs +907 -0
- package/lib/org/activity.mjs +211 -0
- package/lib/org/activity.test.mjs +134 -0
- package/lib/org/approvals.mjs +448 -0
- package/lib/org/approvals.test.mjs +216 -0
- package/lib/org/awareness.mjs +222 -0
- package/lib/org/awareness.test.mjs +159 -0
- package/lib/org/board.mjs +229 -0
- package/lib/org/board.test.mjs +177 -0
- package/lib/org/bootstrap-context.mjs +169 -0
- package/lib/org/bootstrap-context.test.mjs +153 -0
- package/lib/org/client.mjs +1628 -0
- package/lib/org/client.test.mjs +1107 -0
- package/lib/org/cohort-client.mjs +67 -0
- package/lib/org/cohort-client.test.mjs +126 -0
- package/lib/org/cost-sync.mjs +227 -0
- package/lib/org/cost-sync.test.mjs +153 -0
- package/lib/org/doctor.mjs +212 -0
- package/lib/org/doctor.test.mjs +212 -0
- package/lib/org/handoff.mjs +293 -0
- package/lib/org/handoff.test.mjs +269 -0
- package/lib/org/integration-tools.mjs +182 -0
- package/lib/org/integration-tools.test.mjs +160 -0
- package/lib/org/keys.mjs +131 -0
- package/lib/org/keys.test.mjs +92 -0
- package/lib/org/knowledge.mjs +463 -0
- package/lib/org/knowledge.test.mjs +319 -0
- package/lib/org/leases.mjs +335 -0
- package/lib/org/leases.test.mjs +235 -0
- package/lib/org/mesh-integration.test.mjs +127 -0
- package/lib/org/mesh.mjs +459 -0
- package/lib/org/mesh.test.mjs +345 -0
- package/lib/org/messaging.mjs +503 -0
- package/lib/org/messaging.test.mjs +238 -0
- package/lib/org/policy.mjs +345 -0
- package/lib/org/policy.test.mjs +237 -0
- package/lib/org/protocol.checksum +1 -0
- package/lib/org/protocol.checksum.test.mjs +90 -0
- package/lib/org/protocol.mjs +967 -0
- package/lib/org/protocol.test.mjs +264 -0
- package/lib/org/registry.mjs +194 -0
- package/lib/org/registry.test.mjs +100 -0
- package/lib/org/tool-surface-integration.test.mjs +120 -0
- package/lib/org/tool-surface.mjs +2535 -0
- package/lib/org/tool-surface.test.mjs +589 -0
- package/lib/org/ui-parity.mjs +3236 -0
- package/lib/org/ui-parity.test.mjs +348 -0
- package/lib/org/verify.mjs +176 -0
- package/lib/org/verify.test.mjs +194 -0
- package/lib/rag/embed.mjs +188 -0
- package/lib/rag/indexer.mjs +425 -0
- package/lib/rag/rag.test.mjs +505 -0
- package/lib/rag/search.mjs +475 -0
- package/lib/rate-guard.mjs +246 -0
- package/lib/rate-guard.test.mjs +201 -0
- package/lib/render.mjs +112 -0
- package/lib/render.test.mjs +68 -0
- package/lib/resource-governor.mjs +297 -0
- package/lib/resource-governor.test.mjs +262 -0
- package/lib/scheduling/dynamic-jobs.mjs +675 -0
- package/lib/scheduling/dynamic-jobs.test.mjs +344 -0
- package/lib/scheduling/jitter.mjs +0 -0
- package/lib/scheduling/jitter.test.mjs +140 -0
- package/lib/secrets/broker.mjs +315 -0
- package/lib/secrets/broker.test.mjs +280 -0
- package/lib/secrets/providers.mjs +461 -0
- package/lib/secrets/providers.test.mjs +274 -0
- package/lib/security/audit-engine.mjs +684 -0
- package/lib/security/audit-engine.test.mjs +389 -0
- package/lib/security/coerce-args.mjs +552 -0
- package/lib/security/coerce-args.test.mjs +281 -0
- package/lib/security/dangerous-tools.mjs +97 -0
- package/lib/security/dangerous-tools.test.mjs +68 -0
- package/lib/security/external-content.mjs +145 -0
- package/lib/security/external-content.test.mjs +67 -0
- package/lib/security/redact.mjs +592 -0
- package/lib/security/redact.test.mjs +441 -0
- package/lib/security/secret-equal.mjs +73 -0
- package/lib/security/secret-equal.test.mjs +55 -0
- package/lib/session-permissions.mjs +101 -0
- package/lib/session-permissions.test.mjs +100 -0
- package/lib/setup/claude-probe.mjs +74 -0
- package/lib/setup/completeness.mjs +175 -0
- package/lib/setup/completeness.test.mjs +110 -0
- package/lib/setup/context-pack.mjs +173 -0
- package/lib/setup/context-pack.test.mjs +89 -0
- package/lib/setup/enrich.mjs +277 -0
- package/lib/setup/enrich.test.mjs +115 -0
- package/lib/setup/enroll-from-cohort.mjs +441 -0
- package/lib/setup/enroll-from-cohort.test.mjs +233 -0
- package/lib/setup/integration.test.mjs +162 -0
- package/lib/setup/io.mjs +360 -0
- package/lib/setup/io.test.mjs +77 -0
- package/lib/setup/run-generator.mjs +81 -0
- package/lib/setup/runner.mjs +244 -0
- package/lib/setup/runner.test.mjs +132 -0
- package/lib/setup/sections/comms.mjs +173 -0
- package/lib/setup/sections/company.mjs +120 -0
- package/lib/setup/sections/enrich.mjs +138 -0
- package/lib/setup/sections/identity.mjs +182 -0
- package/lib/setup/sections/identity.test.mjs +140 -0
- package/lib/setup/sections/learning.mjs +153 -0
- package/lib/setup/sections/learning.test.mjs +81 -0
- package/lib/setup/sections/messaging.mjs +219 -0
- package/lib/setup/sections/messaging.test.mjs +127 -0
- package/lib/setup/sections/model.mjs +102 -0
- package/lib/setup/sections/operating-model.mjs +78 -0
- package/lib/setup/sections/org.mjs +475 -0
- package/lib/setup/sections/org.test.mjs +313 -0
- package/lib/setup/sections/orgmail.mjs +173 -0
- package/lib/setup/sections/orgmail.test.mjs +118 -0
- package/lib/setup/sections/recovery.mjs +159 -0
- package/lib/setup/sections/recovery.test.mjs +98 -0
- package/lib/setup/sections/tools.mjs +132 -0
- package/lib/setup/sections/verify.mjs +97 -0
- package/lib/setup/sot.mjs +205 -0
- package/lib/setup/sot.test.mjs +81 -0
- package/lib/setup/state.mjs +151 -0
- package/lib/setup/state.test.mjs +92 -0
- package/lib/singleton.js +229 -0
- package/lib/singleton.test.mjs +135 -0
- package/lib/telemetry/alerts.mjs +216 -0
- package/lib/telemetry/alerts.test.mjs +109 -0
- package/lib/telemetry/collect.mjs +512 -0
- package/lib/telemetry/collect.test.mjs +202 -0
- package/lib/tool-definitions-integration.test.mjs +83 -0
- package/lib/tool-definitions.js +738 -0
- package/lib/tool-definitions.test.mjs +437 -0
- package/lib/util/fetch-timeout.mjs +136 -0
- package/lib/util/fetch-timeout.test.mjs +202 -0
- package/lib/util/reconnect.mjs +343 -0
- package/lib/util/reconnect.test.mjs +369 -0
- package/lib/util/unhandled.mjs +205 -0
- package/lib/util/unhandled.test.mjs +216 -0
- package/lib/voice/context-loader.mjs +466 -0
- package/lib/voice/index.mjs +100 -0
- package/lib/voice/openai-realtime.mjs +510 -0
- package/lib/voice/outbound.mjs +542 -0
- package/lib/voice/outbound.test.mjs +69 -0
- package/lib/voice/post-call-brief.mjs +428 -0
- package/lib/voice/provider.mjs +52 -0
- package/lib/voice/session-rotation.mjs +257 -0
- package/lib/voice/stt.mjs +161 -0
- package/lib/voice/stt.test.mjs +226 -0
- package/lib/voice/tool-bridge.mjs +370 -0
- package/lib/voice/tts.mjs +104 -0
- package/lib/voice/twilio-sip-bridge.mjs +288 -0
- package/lib/voice/voice.test.mjs +990 -0
- package/mcp/README.md +80 -0
- package/package.json +151 -0
- package/plugins/maestro-skills/plugin.json +139 -0
- package/plugins/maestro-skills/skills/agents-observe.md +110 -0
- package/plugins/maestro-skills/skills/board-deck.md +68 -0
- package/plugins/maestro-skills/skills/books-close.md +77 -0
- package/plugins/maestro-skills/skills/brand-steward.md +121 -0
- package/plugins/maestro-skills/skills/calendar-plan.md +57 -0
- package/plugins/maestro-skills/skills/call-working-sessions.md +124 -0
- package/plugins/maestro-skills/skills/ccxray-diagnostics.md +91 -0
- package/plugins/maestro-skills/skills/claude-pace.md +61 -0
- package/plugins/maestro-skills/skills/code-review-graph.md +99 -0
- package/plugins/maestro-skills/skills/crm-pipeline.md +65 -0
- package/plugins/maestro-skills/skills/decision-brief.md +89 -0
- package/plugins/maestro-skills/skills/directory-hygiene.md +125 -0
- package/plugins/maestro-skills/skills/draft-comms.md +84 -0
- package/plugins/maestro-skills/skills/evening-wrap.md +53 -0
- package/plugins/maestro-skills/skills/files-find.md +65 -0
- package/plugins/maestro-skills/skills/generative-ui.md +228 -0
- package/plugins/maestro-skills/skills/hiring-triage.md +74 -0
- package/plugins/maestro-skills/skills/inbox-triage.md +61 -0
- package/plugins/maestro-skills/skills/mail-triage.md +86 -0
- package/plugins/maestro-skills/skills/morning-brief.md +54 -0
- package/plugins/maestro-skills/skills/native-artifacts.md +157 -0
- package/plugins/maestro-skills/skills/org-board.md +133 -0
- package/plugins/maestro-skills/skills/org-credential.md +68 -0
- package/plugins/maestro-skills/skills/org-recall.md +81 -0
- package/plugins/maestro-skills/skills/pipeline-review.md +76 -0
- package/plugins/maestro-skills/skills/regulatory-status.md +81 -0
- package/plugins/maestro-skills/skills/router-why.md +78 -0
- package/plugins/maestro-skills/skills/schedule-meeting.md +91 -0
- package/plugins/maestro-skills/skills/session-search.md +71 -0
- package/plugins/maestro-skills/skills/set-reminder.md +93 -0
- package/plugins/maestro-skills/skills/slack-followup.md +64 -0
- package/plugins/maestro-skills/skills/team-activity.md +86 -0
- package/plugins/maestro-skills/skills/weekly-memo.md +70 -0
- package/policies/action-classification.yaml +114 -0
- package/policies/ai-disclosure.yaml +294 -0
- package/policies/communication-style.md +139 -0
- package/policies/information-barriers.yaml +118 -0
- package/policies/prompt-injection-defence.yaml +138 -0
- package/public/assets/icon-dark.png +0 -0
- package/public/assets/icon-dark.svg +9 -0
- package/public/assets/icon-light.svg +9 -0
- package/public/assets/logo-dark.svg +15 -0
- package/public/assets/logo-light.svg +15 -0
- package/scaffold/.mcp.json +7 -0
- package/scaffold/CLAUDE.md +368 -0
- package/scaffold/config/agent.json +55 -0
- package/scaffold/config/agent.ts +76 -0
- package/scaffold/config/agent.ts.example +89 -0
- package/scaffold/config/alerts.yaml +23 -0
- package/scaffold/config/allowlist.yaml.example +25 -0
- package/scaffold/config/caller-id-map.yaml +46 -0
- package/scaffold/config/collective.yaml +49 -0
- package/scaffold/config/company.json +20 -0
- package/scaffold/config/known-agents.json +6 -0
- package/scaffold/config/learning.yaml +55 -0
- package/scaffold/config/model-routing.yaml.example +104 -0
- package/scaffold/config/org.yaml +25 -0
- package/scaffold/config/orgmail.yaml.example +19 -0
- package/scaffold/config/recovery.yaml +72 -0
- package/scaffold/config/secrets.yaml +27 -0
- package/scaffold/config/slack.yaml.example +35 -0
- package/scaffold/config/telegram.yaml.example +38 -0
- package/scaffold/config/voice.yaml.example +89 -0
- package/scaffold/config/whatsapp.yaml.example +39 -0
- package/schedules/README.md +49 -0
- package/schedules/triggers/backlog-executor.md +102 -0
- package/schedules/triggers/brand-steward.md +72 -0
- package/schedules/triggers/daily-evening-wrap.md +159 -0
- package/schedules/triggers/daily-midday-sweep.md +58 -0
- package/schedules/triggers/daily-morning-brief.md +55 -0
- package/schedules/triggers/directory-hygiene.md +81 -0
- package/schedules/triggers/dynamic-jobs.md +40 -0
- package/schedules/triggers/inbox-processor.md +115 -0
- package/schedules/triggers/meeting-action-capture.md +60 -0
- package/schedules/triggers/meeting-prep.md +69 -0
- package/schedules/triggers/messaging-inbound.md +50 -0
- package/schedules/triggers/org-pulse.md +24 -0
- package/schedules/triggers/quarterly-self-assessment.md +54 -0
- package/schedules/triggers/weekly-engineering-health.md +37 -0
- package/schedules/triggers/weekly-execution.md +65 -0
- package/schedules/triggers/weekly-hiring.md +53 -0
- package/schedules/triggers/weekly-priorities.md +38 -0
- package/schedules/triggers/weekly-strategic-memo.md +124 -0
- package/scripts/archive-email.sh +55 -0
- package/scripts/cadence/cadence-status.mjs +36 -0
- package/scripts/cadence/enqueue-cadence-tick.mjs +174 -0
- package/scripts/cadence/enqueue-cadence-tick.test.mjs +187 -0
- package/scripts/cadence/launchd-cadence-wrapper.sh +85 -0
- package/scripts/cadence/launchd-cloud-relay-wrapper.sh +95 -0
- package/scripts/cadence/launchd-socket-mode-wrapper.sh +95 -0
- package/scripts/ci/check-docs-accuracy.mjs +493 -0
- package/scripts/ci/check-docs-accuracy.test.mjs +409 -0
- package/scripts/ci/check-exports-exist.mjs +140 -0
- package/scripts/ci/check-files-exist.mjs +107 -0
- package/scripts/ci/check-no-build-artifacts.mjs +111 -0
- package/scripts/ci/check-no-build-artifacts.test.mjs +71 -0
- package/scripts/ci/check-no-confidential.mjs +198 -0
- package/scripts/ci/check-no-conflict-markers.mjs +169 -0
- package/scripts/ci/check-no-residual-identity.mjs +163 -0
- package/scripts/ci/check-no-residual-identity.test.mjs +89 -0
- package/scripts/ci/check-tarball-fidelity.mjs +205 -0
- package/scripts/ci/check-unresolved-tokens.mjs +83 -0
- package/scripts/ci/check.mjs +109 -0
- package/scripts/ci/check.test.mjs +194 -0
- package/scripts/ci/run-coverage.mjs +82 -0
- package/scripts/ci/run-tests.mjs +71 -0
- package/scripts/cloud-relay/README.md +59 -0
- package/scripts/cloud-relay/index.mjs +233 -0
- package/scripts/cloud-relay/package.json +15 -0
- package/scripts/cloud-relay/railway.json +13 -0
- package/scripts/cloud-relay/voice/README.md +94 -0
- package/scripts/cloud-relay/voice/package-lock.json +39 -0
- package/scripts/cloud-relay/voice/package.json +16 -0
- package/scripts/cloud-relay/voice/railway.json +13 -0
- package/scripts/cloud-relay/voice/server.mjs +532 -0
- package/scripts/collective/hook-runner.mjs +211 -0
- package/scripts/collective/hook-runner.test.mjs +90 -0
- package/scripts/collective/org-pulse.mjs +72 -0
- package/scripts/collective/org-sync.mjs +61 -0
- package/scripts/collective/recall.mjs +45 -0
- package/scripts/collective/who.mjs +30 -0
- package/scripts/comms-monitor.sh +288 -0
- package/scripts/configure-whatsapp-sandbox.sh +201 -0
- package/scripts/continuous-monitor.sh +91 -0
- package/scripts/cost/fleet-digest.mjs +407 -0
- package/scripts/cost/fleet-digest.test.mjs +207 -0
- package/scripts/cost/track-claude-usage.mjs +169 -0
- package/scripts/daemon/agent-daemon.mjs +989 -0
- package/scripts/daemon/agent-daemon.test.mjs +525 -0
- package/scripts/daemon/cadence-consumer-governance.test.mjs +220 -0
- package/scripts/daemon/cadence-consumer.mjs +1080 -0
- package/scripts/daemon/cadence-consumer.test.mjs +770 -0
- package/scripts/daemon/cadence-handlers.mjs +1121 -0
- package/scripts/daemon/cadence-handlers.test.mjs +617 -0
- package/scripts/daemon/classifier.mjs +704 -0
- package/scripts/daemon/classifier.test.mjs +238 -0
- package/scripts/daemon/classify-kind.mjs +54 -0
- package/scripts/daemon/classify-kind.test.mjs +40 -0
- package/scripts/daemon/context-compiler.mjs +605 -0
- package/scripts/daemon/context-compiler.test.mjs +300 -0
- package/scripts/daemon/dispatcher-cooldown.test.mjs +122 -0
- package/scripts/daemon/dispatcher-governance.test.mjs +886 -0
- package/scripts/daemon/dispatcher.mjs +1516 -0
- package/scripts/daemon/health.mjs +72 -0
- package/scripts/daemon/inbox-deferral.mjs +210 -0
- package/scripts/daemon/inbox-deferral.test.mjs +242 -0
- package/scripts/daemon/integration.test.mjs +149 -0
- package/scripts/daemon/launchd-wrapper-generic.sh +96 -0
- package/scripts/daemon/launchd-wrapper-slack-events.sh +37 -0
- package/scripts/daemon/launchd-wrapper.sh +91 -0
- package/scripts/daemon/lib/session-router.mjs +274 -0
- package/scripts/daemon/lib/session-router.test.mjs +295 -0
- package/scripts/daemon/maestro-daemon.mjs +275 -0
- package/scripts/daemon/prompt-builder.mjs +685 -0
- package/scripts/daemon/prompt-builder.test.mjs +213 -0
- package/scripts/daemon/responder.mjs +854 -0
- package/scripts/daemon/session-lock.mjs +721 -0
- package/scripts/daemon/session-lock.test.mjs +252 -0
- package/scripts/daemon/session-outcomes.mjs +640 -0
- package/scripts/daemon/session-outcomes.test.mjs +533 -0
- package/scripts/daemon/typing-registry.mjs +90 -0
- package/scripts/daemon/typing-registry.test.mjs +77 -0
- package/scripts/daemon/voice-webhook-server.mjs +804 -0
- package/scripts/decisions/capture-decision.mjs +116 -0
- package/scripts/disclosure_assessment.py +873 -0
- package/scripts/disclosure_boundaries.py +562 -0
- package/scripts/email-signature-principal.html +52 -0
- package/scripts/email-signature.html +60 -0
- package/scripts/email_quote_thread.py +167 -0
- package/scripts/email_thread_dedup.py +362 -0
- package/scripts/emergency-stop.sh +81 -0
- package/scripts/healthcheck.sh +116 -0
- package/scripts/hooks/block-mcp-cohort-send.sh +15 -0
- package/scripts/hooks/block-mcp-slack-send.sh +7 -0
- package/scripts/hooks/post-action-log.sh +126 -0
- package/scripts/hooks/pre-send-audit.sh +174 -0
- package/scripts/hooks/pre-send-audit.test.mjs +215 -0
- package/scripts/hooks/session-end-log.sh +27 -0
- package/scripts/hooks/session-start-banner.sh +115 -0
- package/scripts/huddle/audio-bridge.mjs +664 -0
- package/scripts/huddle/boot-slack-cdp.sh +102 -0
- package/scripts/huddle/huddle-controller.mjs +942 -0
- package/scripts/huddle/huddle-server.mjs +1229 -0
- package/scripts/huddle/launch-slack.sh +232 -0
- package/scripts/huddle/openai-realtime-bridge.mjs +462 -0
- package/scripts/huddle/package-lock.json +62 -0
- package/scripts/huddle/package.json +22 -0
- package/scripts/huddle/setup-audio.sh +239 -0
- package/scripts/huddle/start-call.mjs +318 -0
- package/scripts/huddle/test-pipeline.mjs +263 -0
- package/scripts/learning/consolidate-skills.mjs +72 -0
- package/scripts/learning/session-search.mjs +125 -0
- package/scripts/llm_email_dedup.py +442 -0
- package/scripts/local-triggers/generate-plists.sh +432 -0
- package/scripts/local-triggers/generate-plists.test.mjs +413 -0
- package/scripts/local-triggers/install-all.sh +49 -0
- package/scripts/local-triggers/plists/.gitkeep +0 -0
- package/scripts/local-triggers/run-trigger.sh +63 -0
- package/scripts/local-triggers/templates/rag-reindex.plist.template +47 -0
- package/scripts/local-triggers/templates/voice-relay-poller.plist.template +54 -0
- package/scripts/local-triggers/templates/voice-tunnel.plist.template +55 -0
- package/scripts/local-triggers/templates/voice-webhook.plist.template +51 -0
- package/scripts/maintenance/backup-to-cloud.sh +124 -0
- package/scripts/maintenance/health-check.sh +377 -0
- package/scripts/media-generation/README.md +105 -0
- package/scripts/media-generation/gemini-image-client.mjs +173 -0
- package/scripts/media-generation/generate-assets.mjs +289 -0
- package/scripts/media-generation/veo-video-client.mjs +219 -0
- package/scripts/org/send-orgmail.mjs +227 -0
- package/scripts/outbound-dedup-cleanup.sh +43 -0
- package/scripts/outbound-dedup.sh +477 -0
- package/scripts/outbound_dedup.py +115 -0
- package/scripts/parse-voice-transcript.mjs +481 -0
- package/scripts/pdf-generation/README.md +63 -0
- package/scripts/pdf-generation/build-document.mjs +247 -0
- package/scripts/pdf-generation/templates/board-pack.latex +136 -0
- package/scripts/pdf-generation/templates/corporate-letter.latex +126 -0
- package/scripts/pdf-generation/templates/memo.latex +114 -0
- package/scripts/poll-slack-events.sh +35 -0
- package/scripts/poller/calendar-poller.mjs +12 -0
- package/scripts/poller/gmail-poller.mjs +192 -0
- package/scripts/poller/imap-client.mjs +289 -0
- package/scripts/poller/inbox-scan-poller.mjs +156 -0
- package/scripts/poller/inbox-scan-poller.test.mjs +231 -0
- package/scripts/poller/index.mjs +73 -0
- package/scripts/poller/intra-session-check.mjs +285 -0
- package/scripts/poller/lib/cloud-relay-dedup.mjs +88 -0
- package/scripts/poller/lib/cloud-relay-dedup.test.mjs +133 -0
- package/scripts/poller/lib/slash-command-handlers.mjs +177 -0
- package/scripts/poller/secondary-gmail-poller.mjs +132 -0
- package/scripts/poller/slack-cloud-relay-client.mjs +368 -0
- package/scripts/poller/slack-poller.mjs +854 -0
- package/scripts/poller/slack-socket-mode.mjs +917 -0
- package/scripts/poller/slack-socket-mode.test.mjs +753 -0
- package/scripts/poller/trigger.mjs +75 -0
- package/scripts/poller/utils.mjs +371 -0
- package/scripts/poller/voice-cloud-relay-client.mjs +179 -0
- package/scripts/poller/voice-poller.mjs +236 -0
- package/scripts/poller-launchd/install.sh +66 -0
- package/scripts/poller-launchd/poller.plist.template +40 -0
- package/scripts/poller-launchd/whatsapp-handler.plist.template +39 -0
- package/scripts/post-interaction-indexer.py +1598 -0
- package/scripts/pre-draft-context.py +994 -0
- package/scripts/pre_draft_lookup.py +258 -0
- package/scripts/rag/build-index.mjs +47 -0
- package/scripts/rag/ingest.mjs +111 -0
- package/scripts/rag/search.mjs +119 -0
- package/scripts/rag-indexer.py +629 -0
- package/scripts/restore-from-backup.sh +248 -0
- package/scripts/restore-from-backup.test.mjs +178 -0
- package/scripts/resume-operations.sh +80 -0
- package/scripts/search-secondary-inbox.py +181 -0
- package/scripts/secondary-inbox-poller.py +437 -0
- package/scripts/self-optimization/compute-metrics.py +398 -0
- package/scripts/send-email-as-principal.py +369 -0
- package/scripts/send-email-threaded.py +392 -0
- package/scripts/send-email-with-attachment.py +377 -0
- package/scripts/send-email.sh +131 -0
- package/scripts/send-sms.sh +175 -0
- package/scripts/send-whatsapp.sh +292 -0
- package/scripts/session-start.sh +106 -0
- package/scripts/setup/boot-claude-session.sh +94 -0
- package/scripts/setup/configure-macos.sh +674 -0
- package/scripts/setup/configure-twilio-sip-trunk.mjs +207 -0
- package/scripts/setup/configure-voice-tunnel.mjs +182 -0
- package/scripts/setup/generate-agent-env.mjs +92 -0
- package/scripts/setup/generate-agent-package-json.mjs +222 -0
- package/scripts/setup/generate-agent-package-json.test.mjs +143 -0
- package/scripts/setup/generate-autonomy.mjs +60 -0
- package/scripts/setup/generate-backlog.mjs +101 -0
- package/scripts/setup/generate-cadences.mjs +92 -0
- package/scripts/setup/generate-capability.mjs +162 -0
- package/scripts/setup/generate-charter.mjs +76 -0
- package/scripts/setup/generate-comms.mjs +59 -0
- package/scripts/setup/generate-company.mjs +92 -0
- package/scripts/setup/init-agent.sh +547 -0
- package/scripts/setup/init-agent.test.mjs +151 -0
- package/scripts/setup/init-archetype.mjs +74 -0
- package/scripts/setup/init-backup.mjs +54 -0
- package/scripts/setup/init-cadence-bus.mjs +60 -0
- package/scripts/setup/init-channel-bus.mjs +46 -0
- package/scripts/setup/init-cost-tracking.mjs +45 -0
- package/scripts/setup/init-decision-capture.mjs +66 -0
- package/scripts/setup/init-known-agents.mjs +57 -0
- package/scripts/setup/init-learning.mjs +70 -0
- package/scripts/setup/init-memory-executive.mjs +45 -0
- package/scripts/setup/init-model-router.mjs +124 -0
- package/scripts/setup/init-rag.mjs +174 -0
- package/scripts/setup/init-session-router.mjs +38 -0
- package/scripts/setup/init-slack-socket-mode.mjs +260 -0
- package/scripts/setup/init-telegram.mjs +165 -0
- package/scripts/setup/init-voice-realtime.mjs +204 -0
- package/scripts/setup/init-whatsapp-baileys.mjs +77 -0
- package/scripts/setup/install-dev-tools.sh +150 -0
- package/scripts/setup/lib/install-plist.mjs +95 -0
- package/scripts/setup/migrate-agent-to-sot.mjs +192 -0
- package/scripts/setup/render-environment-yaml.mjs +133 -0
- package/scripts/slack-events-ctl.sh +177 -0
- package/scripts/slack-events-server.mjs +1045 -0
- package/scripts/slack-react.mjs +89 -0
- package/scripts/slack-responded.sh +232 -0
- package/scripts/slack-send.sh +287 -0
- package/scripts/slack-typing.mjs +196 -0
- package/scripts/slack-upload-v2.py +95 -0
- package/scripts/sms-handler.mjs +450 -0
- package/scripts/spawn-session.sh +120 -0
- package/scripts/sync-protocol.mjs +217 -0
- package/scripts/system-verify.sh +184 -0
- package/scripts/test-email-thread-dedup.py +239 -0
- package/scripts/test-information-barriers.py +484 -0
- package/scripts/test-llm-email-dedup.py +251 -0
- package/scripts/test-pre-draft-integration.py +203 -0
- package/scripts/test-rag-phase2.sh +442 -0
- package/scripts/test-rag-search.sh +251 -0
- package/scripts/test-voice-parser.mjs +316 -0
- package/scripts/user-context-search.py +659 -0
- package/scripts/validate_outbound.py +1504 -0
- package/scripts/watchdog/ai.maestro.memory-watchdog.plist +41 -0
- package/scripts/watchdog/force-reboot.sh +157 -0
- package/scripts/watchdog/memory-watchdog.sh +473 -0
- package/scripts/whatsapp-handler.mjs +538 -0
- package/teams/desktop-operations.yaml +34 -0
- package/teams/executive-office.yaml +27 -0
- package/teams/legal-and-regulatory.yaml +24 -0
- package/teams/platform-and-engineering.yaml +23 -0
- package/teams/strategy-and-growth.yaml +29 -0
- package/workflows/continuous/backlog-executor.yaml +141 -0
- package/workflows/continuous/inbound-monitor.yaml +168 -0
- package/workflows/daily/applicant-triage.yaml +197 -0
- package/workflows/daily/comms-triage.yaml +80 -0
- package/workflows/daily/evening-wrap.yaml +105 -0
- package/workflows/daily/morning-brief.yaml +164 -0
- package/workflows/daily/slack-followup-sweep.yaml +87 -0
- package/workflows/event-driven/README.md +50 -0
- package/workflows/event-driven/agent-failure-investigation.yaml +137 -0
- package/workflows/event-driven/pr-review.yaml +107 -0
- package/workflows/monthly/board-readiness.yaml +76 -0
- package/workflows/quarterly/strategic-scenario-analysis.yaml +85 -0
- package/workflows/session-protocol.md +171 -0
- package/workflows/weekly/engineering-health.yaml +154 -0
- package/workflows/weekly/hiring-review.yaml +169 -0
- package/workflows/weekly/rollup-pipeline-review.yaml +76 -0
- package/workflows/weekly/strategic-memo.yaml +79 -0
|
@@ -0,0 +1,1080 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Maestro — Cadence Consumer
|
|
3
|
+
*
|
|
4
|
+
* The persistent main session's drain loop for the cadence bus.
|
|
5
|
+
*
|
|
6
|
+
* Lifecycle: launched once per agent process (inside maestro-daemon.mjs).
|
|
7
|
+
* On every tick it:
|
|
8
|
+
*
|
|
9
|
+
* 1. Recovers any stale claims (events whose handler crashed mid-flight).
|
|
10
|
+
* 2. Drains inbox/, atomically claiming each event.
|
|
11
|
+
* 3. Routes the event through cadence-handlers.mjs:
|
|
12
|
+
* - inline → run handler in-process, complete.
|
|
13
|
+
* - guarded → run cheap pre-check; if "no work", complete inline,
|
|
14
|
+
* else escalate to a sub-session.
|
|
15
|
+
* - escalate → spawn a sub-session running the cadence's trigger
|
|
16
|
+
* prompt under schedules/triggers/<name>.md.
|
|
17
|
+
* Unknown cadences with a prompt on disk default to escalate; without
|
|
18
|
+
* a prompt they go straight to dlq with a clear error.
|
|
19
|
+
* 4. Respects .emergency-stop: while present, the loop logs a heartbeat
|
|
20
|
+
* but never spawns a sub-session and never processes events. Existing
|
|
21
|
+
* claims remain on disk so they can be resumed once the stop is lifted.
|
|
22
|
+
* 5. Writes a heartbeat to state/cadence-bus/health.json on every cycle
|
|
23
|
+
* so doctor / healthcheck can confirm liveness.
|
|
24
|
+
*
|
|
25
|
+
* The consumer never spawns more than ONE sub-session at a time on its own.
|
|
26
|
+
* That bound exists because the parent process is the single owner of
|
|
27
|
+
* cadence housekeeping; if your workflow needs higher parallelism, lean on
|
|
28
|
+
* the existing daemon dispatcher (which is purpose-built for inbox items),
|
|
29
|
+
* not on multiplying cadence consumers.
|
|
30
|
+
*
|
|
31
|
+
* Public API:
|
|
32
|
+
* startConsumer(opts) → { stop(), getStats(), tickOnce() }
|
|
33
|
+
*
|
|
34
|
+
* Options:
|
|
35
|
+
* agentRoot override AGENT_ROOT (tests).
|
|
36
|
+
* pollMs drain interval (default 2_000).
|
|
37
|
+
* heartbeatMs health.json refresh interval (default 15_000).
|
|
38
|
+
* recoveryMs stale-claim sweep interval (default 5 * 60_000).
|
|
39
|
+
* spawnSession injected spawner for tests; defaults to a real
|
|
40
|
+
* `claude --print <permission-args> <prompt>` child_process
|
|
41
|
+
* .spawn. <permission-args> comes from
|
|
42
|
+
* lib/session-permissions.mjs#sessionPermissionArgs and is
|
|
43
|
+
* `--dangerously-skip-permissions` by default, or a scoped
|
|
44
|
+
* per-cadence `--allowedTools <list>` when the operator sets
|
|
45
|
+
* MAESTRO_SCOPED_PERMISSIONS=1 (audit H1).
|
|
46
|
+
* maxSpawnMs hard timeout per sub-session (default 30 * 60_000).
|
|
47
|
+
* logger optional fn({ ts, level, …rest }) → void for tests.
|
|
48
|
+
* now injectable clock fn() → ms (tests); used when re-stamping
|
|
49
|
+
* in-flight claim mtimes so the stale-claim sweep can't sweep
|
|
50
|
+
* a live escalate (audit L6).
|
|
51
|
+
*/
|
|
52
|
+
|
|
53
|
+
import { existsSync, readFileSync, writeFileSync, mkdirSync, appendFileSync, openSync, closeSync, statSync, unlinkSync, utimesSync } from "node:fs";
|
|
54
|
+
import { join } from "node:path";
|
|
55
|
+
import { spawn } from "node:child_process";
|
|
56
|
+
import { homedir } from "node:os";
|
|
57
|
+
|
|
58
|
+
import {
|
|
59
|
+
ensureBusDirs,
|
|
60
|
+
claimNextTick,
|
|
61
|
+
completeTick,
|
|
62
|
+
failTick,
|
|
63
|
+
requeueTick,
|
|
64
|
+
recoverStaleClaims,
|
|
65
|
+
sweepRetention,
|
|
66
|
+
writeHealth,
|
|
67
|
+
getBusPaths,
|
|
68
|
+
logBusEvent,
|
|
69
|
+
busDepth,
|
|
70
|
+
} from "../../lib/cadence-bus.mjs";
|
|
71
|
+
import { resolveClaudeBin as sharedResolveClaude, augmentedPath, daemonClaudeArgs } from "../../lib/claude-bin.mjs";
|
|
72
|
+
import { getCadenceDef } from "./cadence-handlers.mjs";
|
|
73
|
+
import { sessionPermissionArgs } from "../../lib/session-permissions.mjs";
|
|
74
|
+
import { renderTemplate, buildContext } from "../../lib/render.mjs";
|
|
75
|
+
// Model router (opt-in). A `schema_version: 2` config routes each cadence
|
|
76
|
+
// sub-session's model through resolveChain (SPEC §6.2): the RouteDecision
|
|
77
|
+
// supplies the --model flag, the session-retarget env (third-party backends),
|
|
78
|
+
// the spawn knobs (--max-turns / --effort / --agents Haiku-Explore map), and a
|
|
79
|
+
// decision_id for the ledger/audit join. Deferrable cadences route on the BATCH
|
|
80
|
+
// lane (-50%) before the budget ladder degrades the chain. No config / v1 config
|
|
81
|
+
// / any router error → the historical stock-CLI cadence spawn, unchanged.
|
|
82
|
+
import {
|
|
83
|
+
loadRoutingConfig,
|
|
84
|
+
resolveChain,
|
|
85
|
+
} from "../../lib/model-router.mjs";
|
|
86
|
+
import { buildChildEnv } from "../../lib/model-router/spawn.mjs";
|
|
87
|
+
import { budgetLadder, batchLaneFor, spawnKnobsFor } from "../../lib/model-router/economics.mjs";
|
|
88
|
+
import * as budgetGuardForBand from "../../lib/budget-guard.mjs";
|
|
89
|
+
// WS4 — the consumer is one of the spawn sources, so it honours the SAME
|
|
90
|
+
// resource governor + shared 429 breaker + daily-budget essential-only mode as
|
|
91
|
+
// the dispatcher. A non-ADMIT decision REQUEUES the tick (decision:"deferred")
|
|
92
|
+
// WITHOUT calling failTick — a governor/rate deferral is an upstream gate, not
|
|
93
|
+
// a per-event failure, so it must never burn the event's retry budget.
|
|
94
|
+
import * as resourceGovernor from "../../lib/resource-governor.mjs";
|
|
95
|
+
import * as rateGuardModule from "../../lib/rate-guard.mjs";
|
|
96
|
+
import * as budgetGuardModule from "../../lib/budget-guard.mjs";
|
|
97
|
+
|
|
98
|
+
const RATE_PROVIDER = process.env.MAESTRO_RATE_PROVIDER || "anthropic";
|
|
99
|
+
|
|
100
|
+
// ---------------------------------------------------------------------------
|
|
101
|
+
// Defaults
|
|
102
|
+
// ---------------------------------------------------------------------------
|
|
103
|
+
|
|
104
|
+
const DEFAULT_POLL_MS = 2_000;
|
|
105
|
+
const DEFAULT_HEARTBEAT_MS = 15_000;
|
|
106
|
+
const DEFAULT_RECOVERY_MS = 5 * 60_000;
|
|
107
|
+
const DEFAULT_SPAWN_TIMEOUT_MS = 30 * 60_000;
|
|
108
|
+
|
|
109
|
+
// Concurrency: at most one sub-session at a time per cadence consumer.
|
|
110
|
+
// Cadence events are not realtime — queueing one tick behind another is
|
|
111
|
+
// preferable to thrashing Claude / hitting usage limits.
|
|
112
|
+
const MAX_CONCURRENT_SUB_SESSIONS = 1;
|
|
113
|
+
|
|
114
|
+
// Retry policy. Most cadence failures are systemic (broken prompt, bad
|
|
115
|
+
// auth, transient API errors) — 5 retries doesn't help, it just amplifies
|
|
116
|
+
// the burn. 2 retries with exponential back-off is the right balance.
|
|
117
|
+
const DEFAULT_MAX_ATTEMPTS = 2;
|
|
118
|
+
const BACKOFF_SCHEDULE_MS = [0, 30_000, 120_000]; // 1st retry +30s, 2nd retry +2m
|
|
119
|
+
|
|
120
|
+
// Circuit breaker — when 3 same-cadence failures land in a row, stop
|
|
121
|
+
// spawning that cadence for 30 minutes. Prevents launchd-rate runaway.
|
|
122
|
+
const CIRCUIT_OPEN_THRESHOLD = 3;
|
|
123
|
+
const CIRCUIT_OPEN_DURATION_MS = 30 * 60_000;
|
|
124
|
+
|
|
125
|
+
// ---------------------------------------------------------------------------
|
|
126
|
+
// Helpers
|
|
127
|
+
// ---------------------------------------------------------------------------
|
|
128
|
+
|
|
129
|
+
function isEmergencyStop(paths) {
|
|
130
|
+
return existsSync(paths.emergencyStop);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function defaultLogger(entry) {
|
|
134
|
+
// The default logger ALSO writes to logs/cadence-bus/<date>.jsonl via
|
|
135
|
+
// logBusEvent, but mirrors important events to stderr so launchd's
|
|
136
|
+
// StandardErrorPath captures them.
|
|
137
|
+
if (entry.level === "error" || entry.level === "warn") {
|
|
138
|
+
try { process.stderr.write(`[cadence-consumer] ${JSON.stringify(entry)}\n`); }
|
|
139
|
+
catch { /* ignore */ }
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// Claude binary resolution moved to lib/claude-bin.mjs (shared by
|
|
144
|
+
// dispatcher, responder, and this consumer). See that file for the
|
|
145
|
+
// candidate search order.
|
|
146
|
+
const resolveClaudeBin = sharedResolveClaude;
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Parse the `claude --print --output-format json` result object from a captured
|
|
150
|
+
* stdout file to recover the run's REAL token usage + authoritative cost
|
|
151
|
+
* (recovery C1). The CLI emits a single JSON object:
|
|
152
|
+
* { result, usage: { input_tokens, output_tokens, cache_read_input_tokens, ... },
|
|
153
|
+
* total_cost_usd, ... }
|
|
154
|
+
* Some CLI versions wrap it differently or prepend a banner, so we read the file
|
|
155
|
+
* and JSON.parse the LAST balanced top-level object (the JSON tail is the
|
|
156
|
+
* result). Returns { ok, inputTokens, outputTokens, cacheReadTokens?,
|
|
157
|
+
* totalCostUsd?, model? } on success, or { ok:false, reason } so the caller can
|
|
158
|
+
* record a parse-failure marker instead of fabricating zeros.
|
|
159
|
+
*
|
|
160
|
+
* @param {string} stdoutPath
|
|
161
|
+
* @returns {{ok:true,inputTokens:number,outputTokens:number,cacheReadTokens?:number,totalCostUsd?:number,model?:string}|{ok:false,reason:string}}
|
|
162
|
+
*/
|
|
163
|
+
export function parseUsageFromStdout(stdoutPath) {
|
|
164
|
+
let raw;
|
|
165
|
+
try { raw = readFileSync(stdoutPath, "utf-8"); }
|
|
166
|
+
catch (err) { return { ok: false, reason: `read-failed: ${err.message}` }; }
|
|
167
|
+
const trimmed = (raw || "").trim();
|
|
168
|
+
if (!trimmed) return { ok: false, reason: "empty-stdout" };
|
|
169
|
+
|
|
170
|
+
let obj = null;
|
|
171
|
+
// Fast path: the whole file is the JSON object.
|
|
172
|
+
try { obj = JSON.parse(trimmed); }
|
|
173
|
+
catch {
|
|
174
|
+
// Tolerant path: a leading banner / log line can precede the JSON tail. The
|
|
175
|
+
// CLI's result object is the FIRST balanced top-level object, so slice from
|
|
176
|
+
// the first "{" to the last "}". (lastIndexOf("{") would grab a nested
|
|
177
|
+
// object's opening brace and break the parse.)
|
|
178
|
+
const start = trimmed.indexOf("{");
|
|
179
|
+
const end = trimmed.lastIndexOf("}");
|
|
180
|
+
if (start !== -1 && end !== -1 && end > start) {
|
|
181
|
+
try { obj = JSON.parse(trimmed.slice(start, end + 1)); }
|
|
182
|
+
catch { obj = null; }
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
if (!obj || typeof obj !== "object") return { ok: false, reason: "no-json-object" };
|
|
186
|
+
|
|
187
|
+
const usage = obj.usage && typeof obj.usage === "object" ? obj.usage : null;
|
|
188
|
+
if (!usage) return { ok: false, reason: "no-usage-field" };
|
|
189
|
+
|
|
190
|
+
const inputTokens = Number(usage.input_tokens);
|
|
191
|
+
const outputTokens = Number(usage.output_tokens);
|
|
192
|
+
if (!Number.isFinite(inputTokens) || !Number.isFinite(outputTokens)) {
|
|
193
|
+
return { ok: false, reason: "non-numeric-tokens" };
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
const out = { ok: true, inputTokens, outputTokens };
|
|
197
|
+
const cacheRead = Number(usage.cache_read_input_tokens);
|
|
198
|
+
if (Number.isFinite(cacheRead)) out.cacheReadTokens = cacheRead;
|
|
199
|
+
const totalCost = Number(obj.total_cost_usd);
|
|
200
|
+
if (Number.isFinite(totalCost) && totalCost >= 0) out.totalCostUsd = totalCost;
|
|
201
|
+
// The CLI may report the resolved model (e.g. "claude-sonnet-4-6"); map it to
|
|
202
|
+
// the tracker's coarse class so pricing lookup stays correct.
|
|
203
|
+
if (typeof obj.model === "string" && obj.model) {
|
|
204
|
+
out.model = /opus/i.test(obj.model) ? "opus" : /haiku/i.test(obj.model) ? "haiku" : "sonnet";
|
|
205
|
+
}
|
|
206
|
+
return out;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// Lazy + cached routing config per agentRoot (invalidated only on process
|
|
210
|
+
// restart, matching the dispatcher). A misconfigured YAML doesn't break cadence
|
|
211
|
+
// spawns — we fall back to the stock-CLI path.
|
|
212
|
+
const _cadenceRoutingCache = new Map(); // agentRoot -> config | null
|
|
213
|
+
function getCadenceRoutingConfig(agentRoot) {
|
|
214
|
+
if (_cadenceRoutingCache.has(agentRoot)) return _cadenceRoutingCache.get(agentRoot);
|
|
215
|
+
let cfg = null;
|
|
216
|
+
try {
|
|
217
|
+
cfg = loadRoutingConfig(agentRoot) || null;
|
|
218
|
+
} catch (err) {
|
|
219
|
+
process.stderr.write(`[cadence-consumer] model-router config invalid, using stock CLI: ${err.message}\n`);
|
|
220
|
+
cfg = null;
|
|
221
|
+
}
|
|
222
|
+
_cadenceRoutingCache.set(agentRoot, cfg);
|
|
223
|
+
return cfg;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* Route a cadence sub-session's model through the v2 router. Returns a normalised
|
|
228
|
+
* target { modelFlag, envForSpawn, decisionId, backend, model, transport,
|
|
229
|
+
* maxTurns, effort, agentsJson, explain } or null when there's nothing to route
|
|
230
|
+
* (no config, v1 config, kill switch, or any router error) — the caller then
|
|
231
|
+
* keeps the historical stock-CLI cadence spawn (no --model, keychain OAuth).
|
|
232
|
+
*
|
|
233
|
+
* NEVER throws. Deferrable cadences (digests, learning, rollups) route on the
|
|
234
|
+
* BATCH lane; the budget band (from budget-guard) degrades the chain per §6.4.
|
|
235
|
+
*
|
|
236
|
+
* Exported as a test seam (and for doctor/introspection); the spawn path is the
|
|
237
|
+
* only production caller.
|
|
238
|
+
*/
|
|
239
|
+
export function routeCadenceSpawn(agentRoot, cadence) {
|
|
240
|
+
const cfg = getCadenceRoutingConfig(agentRoot);
|
|
241
|
+
if (!cfg || cfg.schema_version !== 2) return null;
|
|
242
|
+
try {
|
|
243
|
+
const taskClass = `cadence.${cadence}`;
|
|
244
|
+
let band = 0;
|
|
245
|
+
try {
|
|
246
|
+
const st = budgetGuardForBand.dailyStatus({ agentRoot });
|
|
247
|
+
band = budgetLadder(st.spentUSD, st.capUSD).band;
|
|
248
|
+
} catch { /* band 0 on read failure */ }
|
|
249
|
+
const req = {
|
|
250
|
+
agent_role: "cadence",
|
|
251
|
+
source: "cadence",
|
|
252
|
+
priority: "normal",
|
|
253
|
+
task_class: taskClass,
|
|
254
|
+
// Cadence work runs over internal queues / repo state, not live inbound
|
|
255
|
+
// user DMs — internal data class (SPEC §7.6). Deferrable → batch lane.
|
|
256
|
+
data_class: "internal",
|
|
257
|
+
lane: batchLaneFor({ task_class: taskClass, priority: "normal" }) ? "batch" : "realtime",
|
|
258
|
+
budget_band: band,
|
|
259
|
+
harness_hint: "session",
|
|
260
|
+
};
|
|
261
|
+
const decision = resolveChain(req, { config: cfg, agentRoot });
|
|
262
|
+
if (!decision || !decision.chosen) return null;
|
|
263
|
+
const knobs = spawnKnobsFor(taskClass);
|
|
264
|
+
const sa = decision.spawnArgs || {};
|
|
265
|
+
return {
|
|
266
|
+
modelFlag: sa.modelFlag || decision.chosen.model || null,
|
|
267
|
+
envForSpawn: decision.envForSpawn || {},
|
|
268
|
+
decisionId: decision.decision_id || null,
|
|
269
|
+
backend: decision.chosen.provider || null,
|
|
270
|
+
model: decision.chosen.model || null,
|
|
271
|
+
transport: decision.chosen.transport || null,
|
|
272
|
+
maxTurns: sa.maxTurns != null ? sa.maxTurns : knobs.maxTurns,
|
|
273
|
+
effort: sa.effort != null ? sa.effort : knobs.effort,
|
|
274
|
+
agentsJson: sa.agentsJson != null ? sa.agentsJson : knobs.agentsJson,
|
|
275
|
+
explain: decision.explain || null,
|
|
276
|
+
lane: decision.lane || req.lane,
|
|
277
|
+
};
|
|
278
|
+
} catch (err) {
|
|
279
|
+
process.stderr.write(`[cadence-consumer] resolveChain failed for ${cadence}, using stock CLI: ${err.message}\n`);
|
|
280
|
+
return null;
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/**
|
|
285
|
+
* Spawn a sub-session running the cadence's trigger prompt and resolve
|
|
286
|
+
* with { exit_code, durationMs, stderr_tail }. Reads the prompt at call
|
|
287
|
+
* time so the latest version (possibly upgraded between ticks) is always
|
|
288
|
+
* used.
|
|
289
|
+
*
|
|
290
|
+
* Robustness: stdout + stderr are tee'd to logs/cadence-bus/subsessions/
|
|
291
|
+
* so non-zero exits remain diagnosable after the fact. The last ~4 KB of
|
|
292
|
+
* stderr is also captured in-memory and surfaced on the failure event.
|
|
293
|
+
*/
|
|
294
|
+
function realSpawnSession({ agentRoot, cadence, promptPath, timeoutMs, log }) {
|
|
295
|
+
return new Promise((resolveOut) => {
|
|
296
|
+
const fullPrompt = join(agentRoot, promptPath);
|
|
297
|
+
if (!existsSync(fullPrompt)) {
|
|
298
|
+
resolveOut({ ok: false, exit_code: -2, error: `prompt not found: ${promptPath}` });
|
|
299
|
+
return;
|
|
300
|
+
}
|
|
301
|
+
let body;
|
|
302
|
+
try { body = readFileSync(fullPrompt, "utf-8"); }
|
|
303
|
+
catch (err) {
|
|
304
|
+
resolveOut({ ok: false, exit_code: -3, error: `prompt read failed: ${err.message}` });
|
|
305
|
+
return;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
// Render {{agent.*}} / {{company.*}} identity tokens at spawn time so framework
|
|
309
|
+
// triggers stay generic in the package and resolve to THIS agent + company
|
|
310
|
+
// (see lib/render.mjs). Synchronous, from config/agent.json + config/company.json.
|
|
311
|
+
// No-op on token-free text, so this is behaviour-preserving for any not-yet-
|
|
312
|
+
// tokenised trigger.
|
|
313
|
+
try {
|
|
314
|
+
const agentCfg = JSON.parse(readFileSync(join(agentRoot, "config/agent.json"), "utf-8"));
|
|
315
|
+
let companyCfg = {};
|
|
316
|
+
try { companyCfg = JSON.parse(readFileSync(join(agentRoot, "config/company.json"), "utf-8")); } catch { /* optional */ }
|
|
317
|
+
body = renderTemplate(body, buildContext(agentCfg, null, companyCfg));
|
|
318
|
+
} catch { /* no/invalid config/agent.json — leave body verbatim */ }
|
|
319
|
+
|
|
320
|
+
const bin = resolveClaudeBin();
|
|
321
|
+
// Permission scoping (audit H1). sessionPermissionArgs() defaults to the
|
|
322
|
+
// historical "--dangerously-skip-permissions" unless the operator opts in
|
|
323
|
+
// with MAESTRO_SCOPED_PERMISSIONS=1, in which case it returns a per-cadence
|
|
324
|
+
// "--allowedTools <list>" form. daemonClaudeArgs() (--strict-mcp-config /
|
|
325
|
+
// --bare) is still appended separately so it applies in both modes.
|
|
326
|
+
const permissionArgs = sessionPermissionArgs({ cadence, mode: getCadenceDef(cadence)?.mode });
|
|
327
|
+
|
|
328
|
+
// Route the cadence sub-session's model through the v2 router (opt-in). When
|
|
329
|
+
// a backend is chosen we pass its --model flag + spawn knobs; otherwise the
|
|
330
|
+
// argv stays exactly as before (no --model → stock CLI default model).
|
|
331
|
+
const target = routeCadenceSpawn(agentRoot, cadence);
|
|
332
|
+
const routedArgs = [];
|
|
333
|
+
if (target) {
|
|
334
|
+
if (target.maxTurns != null && Number.isFinite(Number(target.maxTurns))) {
|
|
335
|
+
routedArgs.push("--max-turns", String(target.maxTurns));
|
|
336
|
+
}
|
|
337
|
+
if (target.effort) routedArgs.push("--effort", String(target.effort));
|
|
338
|
+
if (target.agentsJson) {
|
|
339
|
+
routedArgs.push("--agents", typeof target.agentsJson === "string" ? target.agentsJson : JSON.stringify(target.agentsJson));
|
|
340
|
+
}
|
|
341
|
+
if (target.modelFlag) routedArgs.push("--model", String(target.modelFlag));
|
|
342
|
+
log({ level: "info", stage: "cadence_routed", cadence, decision_id: target.decisionId, backend: target.backend, model: target.model, transport: target.transport, lane: target.lane, explain: target.explain });
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
// --output-format json makes stdout a single JSON object carrying the real
|
|
346
|
+
// token usage (and total_cost_usd) for this run. The cadence sub-session's
|
|
347
|
+
// stdout is redirected to stdoutPath (below) and nothing parses it for
|
|
348
|
+
// CONTENT — only this function reads it back, to record a TRUTHFUL cost row
|
|
349
|
+
// (recovery C1). Without this the ledger logged input/output tokens 0 → the
|
|
350
|
+
// fleet's only spend control summed $0 forever.
|
|
351
|
+
// The cadence PROMPT (body) stays the LAST positional arg; routed model +
|
|
352
|
+
// knob flags slot before it (after the daemon args), mirroring the dispatcher.
|
|
353
|
+
const args = ["--print", "--output-format", "json", ...permissionArgs, ...daemonClaudeArgs(), ...routedArgs, body];
|
|
354
|
+
// PATH augmented via lib/claude-bin.mjs so subsession can find jq/node.
|
|
355
|
+
let env = {
|
|
356
|
+
...process.env,
|
|
357
|
+
AGENT_ROOT: agentRoot,
|
|
358
|
+
AGENT_DIR: agentRoot,
|
|
359
|
+
PATH: augmentedPath(),
|
|
360
|
+
};
|
|
361
|
+
// v2 third-party retarget: build the child env through the execution layer's
|
|
362
|
+
// §7.3 allowlist scrub so no foreign credential leaks into the third-party
|
|
363
|
+
// session (buildChildEnv injects ANTHROPIC_API_KEY="" for a retarget). We
|
|
364
|
+
// preserve the cadence-specific env (AGENT_ROOT/AGENT_DIR/PATH) on top.
|
|
365
|
+
const retargeting = !!(target && target.envForSpawn && target.envForSpawn.ANTHROPIC_BASE_URL);
|
|
366
|
+
if (retargeting) {
|
|
367
|
+
env = {
|
|
368
|
+
...buildChildEnv(env, target.envForSpawn),
|
|
369
|
+
AGENT_ROOT: agentRoot,
|
|
370
|
+
AGENT_DIR: agentRoot,
|
|
371
|
+
PATH: augmentedPath(),
|
|
372
|
+
};
|
|
373
|
+
} else {
|
|
374
|
+
// Auth handling. Claude Code authenticates via macOS Keychain
|
|
375
|
+
// (OAuth from the user's Pro/Max subscription) when no API key is
|
|
376
|
+
// set, OR via the ANTHROPIC_API_KEY env var when one is present.
|
|
377
|
+
// If the env key is present BUT looks like a placeholder / empty
|
|
378
|
+
// string, we strip it so claude can fall back to Keychain OAuth.
|
|
379
|
+
// Set MAESTRO_PREFER_SUBSCRIPTION_AUTH=1 in .env to always strip
|
|
380
|
+
// the API key (force subscription auth) — useful when the agent
|
|
381
|
+
// owns a Claude Code Pro/Max subscription and shouldn't burn API
|
|
382
|
+
// credits for routine ticks.
|
|
383
|
+
const preferSubscription = process.env.MAESTRO_PREFER_SUBSCRIPTION_AUTH === "1";
|
|
384
|
+
const apiKey = env.ANTHROPIC_API_KEY || "";
|
|
385
|
+
if (preferSubscription || !apiKey.trim() || /^(your-api-key|placeholder|xxx+|sk-ant-xxx)/i.test(apiKey)) {
|
|
386
|
+
delete env.ANTHROPIC_API_KEY;
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
const started = Date.now();
|
|
390
|
+
|
|
391
|
+
// Per-run log file. Pattern is short enough to be tail-friendly.
|
|
392
|
+
const logsDir = join(agentRoot, "logs", "cadence-bus", "subsessions");
|
|
393
|
+
mkdirSync(logsDir, { recursive: true });
|
|
394
|
+
const date = new Date().toISOString().slice(0, 10);
|
|
395
|
+
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
396
|
+
const stdoutPath = join(logsDir, `${date}-${cadence}-${stamp}.stdout.log`);
|
|
397
|
+
const stderrPath = join(logsDir, `${date}-${cadence}-${stamp}.stderr.log`);
|
|
398
|
+
const stdoutFd = openSync(stdoutPath, "a");
|
|
399
|
+
const stderrFd = openSync(stderrPath, "a");
|
|
400
|
+
|
|
401
|
+
log({ level: "info", stage: "subsession_spawn", cadence, bin, stdout: stdoutPath, stderr: stderrPath });
|
|
402
|
+
|
|
403
|
+
let child;
|
|
404
|
+
try {
|
|
405
|
+
// stdio:
|
|
406
|
+
// 0 ignore (claude --print reads prompt from argv, not stdin)
|
|
407
|
+
// 1 → file (capture stdout for later inspection)
|
|
408
|
+
// 2 → file (capture stderr — critical for diagnosing exit-1)
|
|
409
|
+
child = spawn(bin, args, { cwd: agentRoot, env, stdio: ["ignore", stdoutFd, stderrFd] });
|
|
410
|
+
} catch (err) {
|
|
411
|
+
try { closeSync(stdoutFd); closeSync(stderrFd); } catch { /* */ }
|
|
412
|
+
resolveOut({ ok: false, exit_code: -4, error: `spawn failed: ${err.message}` });
|
|
413
|
+
return;
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
const timer = setTimeout(() => {
|
|
417
|
+
log({ level: "warn", stage: "subsession_timeout", cadence, timeout_ms: timeoutMs });
|
|
418
|
+
try { child.kill("SIGTERM"); } catch { /* ignore */ }
|
|
419
|
+
// Give it a beat to die before SIGKILL.
|
|
420
|
+
setTimeout(() => { try { child.kill("SIGKILL"); } catch { /* ignore */ } }, 5_000);
|
|
421
|
+
}, timeoutMs);
|
|
422
|
+
|
|
423
|
+
child.on("exit", (code, signal) => {
|
|
424
|
+
clearTimeout(timer);
|
|
425
|
+
try { closeSync(stdoutFd); closeSync(stderrFd); } catch { /* */ }
|
|
426
|
+
const durationMs = Date.now() - started;
|
|
427
|
+
const exit_code = typeof code === "number" ? code : (signal ? -1 : -5);
|
|
428
|
+
|
|
429
|
+
// Pull tail of stderr (and stdout if stderr empty) for the failure
|
|
430
|
+
// surface. Best-effort; we never block on file size.
|
|
431
|
+
let stderrTail = "";
|
|
432
|
+
try {
|
|
433
|
+
const body = readFileSync(stderrPath, "utf-8");
|
|
434
|
+
stderrTail = body.slice(-4096);
|
|
435
|
+
if (!stderrTail.trim()) {
|
|
436
|
+
const so = readFileSync(stdoutPath, "utf-8");
|
|
437
|
+
stderrTail = so.slice(-4096);
|
|
438
|
+
}
|
|
439
|
+
} catch { /* file may not exist if spawn ENOENT before fd-redirect */ }
|
|
440
|
+
|
|
441
|
+
// Record a TRUTHFUL cost-ledger row (recovery C1). We parse the run's
|
|
442
|
+
// `--output-format json` object from stdoutPath to read the REAL token
|
|
443
|
+
// usage (and the CLI's authoritative total_cost_usd) and pass those to the
|
|
444
|
+
// tracker, which derives estimated_usd. If parsing fails we pass NO token
|
|
445
|
+
// flags (tracker records 0 then) and log a parse-failure marker — we never
|
|
446
|
+
// fabricate counts.
|
|
447
|
+
const usage = parseUsageFromStdout(stdoutPath);
|
|
448
|
+
try {
|
|
449
|
+
const trackerPath = join(agentRoot, "scripts/cost/track-claude-usage.mjs");
|
|
450
|
+
if (existsSync(trackerPath)) {
|
|
451
|
+
const trackerArgs = [
|
|
452
|
+
trackerPath, "record",
|
|
453
|
+
"--cadence", cadence,
|
|
454
|
+
"--source", "cadence-consumer",
|
|
455
|
+
// Prefer the CLI-reported resolved model class; else the routed
|
|
456
|
+
// model flag (v2); else the historical sonnet default.
|
|
457
|
+
"--model", usage.model || (target && target.model) || "sonnet",
|
|
458
|
+
"--duration-ms", String(durationMs),
|
|
459
|
+
"--exit", String(exit_code),
|
|
460
|
+
];
|
|
461
|
+
// Carry the v2 RouteDecision id onto the ledger row (SPEC §4.7) so a
|
|
462
|
+
// cadence spend row joins its routing decision. Absent on stock-CLI.
|
|
463
|
+
if (target && target.decisionId) trackerArgs.push("--decision-id", String(target.decisionId));
|
|
464
|
+
if (usage.ok) {
|
|
465
|
+
trackerArgs.push("--input-tokens", String(usage.inputTokens));
|
|
466
|
+
trackerArgs.push("--output-tokens", String(usage.outputTokens));
|
|
467
|
+
if (usage.cacheReadTokens != null) trackerArgs.push("--cache-read-tokens", String(usage.cacheReadTokens));
|
|
468
|
+
if (usage.totalCostUsd != null) trackerArgs.push("--total-cost-usd", String(usage.totalCostUsd));
|
|
469
|
+
} else {
|
|
470
|
+
// Parse failure: do NOT pass fabricated zeros — omit token flags
|
|
471
|
+
// (tracker defaults them to 0) and surface the gap in the bus log so
|
|
472
|
+
// a systematic parse regression is visible rather than silently $0.
|
|
473
|
+
log({ level: "warn", stage: "cost_usage_parse_failed", cadence, reason: usage.reason, stdout: stdoutPath });
|
|
474
|
+
}
|
|
475
|
+
spawn(process.execPath, trackerArgs, { stdio: "ignore", env: { ...env, AGENT_ROOT: agentRoot } }).unref();
|
|
476
|
+
}
|
|
477
|
+
} catch { /* cost tracking is best-effort */ }
|
|
478
|
+
|
|
479
|
+
// Clean up empty log files so the directory doesn't accumulate
|
|
480
|
+
// hundreds of zero-byte successes.
|
|
481
|
+
try {
|
|
482
|
+
|
|
483
|
+
if (statSync(stdoutPath).size === 0) unlinkSync(stdoutPath);
|
|
484
|
+
if (statSync(stderrPath).size === 0) unlinkSync(stderrPath);
|
|
485
|
+
} catch { /* */ }
|
|
486
|
+
|
|
487
|
+
resolveOut({
|
|
488
|
+
ok: exit_code === 0,
|
|
489
|
+
exit_code,
|
|
490
|
+
signal: signal || null,
|
|
491
|
+
duration_ms: durationMs,
|
|
492
|
+
stderr_tail: stderrTail || null,
|
|
493
|
+
stdout_path: stdoutPath,
|
|
494
|
+
stderr_path: stderrPath,
|
|
495
|
+
});
|
|
496
|
+
});
|
|
497
|
+
|
|
498
|
+
child.on("error", (err) => {
|
|
499
|
+
clearTimeout(timer);
|
|
500
|
+
try { closeSync(stdoutFd); closeSync(stderrFd); } catch { /* */ }
|
|
501
|
+
const durationMs = Date.now() - started;
|
|
502
|
+
resolveOut({ ok: false, exit_code: -6, error: err.message, duration_ms: durationMs });
|
|
503
|
+
});
|
|
504
|
+
});
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
// ---------------------------------------------------------------------------
|
|
508
|
+
// Public API
|
|
509
|
+
// ---------------------------------------------------------------------------
|
|
510
|
+
|
|
511
|
+
/**
|
|
512
|
+
* Start the consumer. Returns control handles (stop, getStats, tickOnce).
|
|
513
|
+
* Caller must `await stop()` to flush state on shutdown.
|
|
514
|
+
*/
|
|
515
|
+
export function startConsumer(opts = {}) {
|
|
516
|
+
const agentRoot = opts.agentRoot || process.env.AGENT_ROOT || process.env.AGENT_DIR || process.cwd();
|
|
517
|
+
const paths = ensureBusDirs(agentRoot);
|
|
518
|
+
|
|
519
|
+
const pollMs = opts.pollMs ?? DEFAULT_POLL_MS;
|
|
520
|
+
const heartbeatMs = opts.heartbeatMs ?? DEFAULT_HEARTBEAT_MS;
|
|
521
|
+
const recoveryMs = opts.recoveryMs ?? DEFAULT_RECOVERY_MS;
|
|
522
|
+
const maxSpawnMs = opts.maxSpawnMs ?? DEFAULT_SPAWN_TIMEOUT_MS;
|
|
523
|
+
const spawnSession = opts.spawnSession || realSpawnSession;
|
|
524
|
+
const userLogger = opts.logger;
|
|
525
|
+
// Test / tuning hooks for the reliability layer.
|
|
526
|
+
const backoffSchedule = opts.backoffSchedule || BACKOFF_SCHEDULE_MS;
|
|
527
|
+
const circuitThreshold = opts.circuitThreshold ?? CIRCUIT_OPEN_THRESHOLD;
|
|
528
|
+
const circuitDurationMs = opts.circuitDurationMs ?? CIRCUIT_OPEN_DURATION_MS;
|
|
529
|
+
const maxAttempts = opts.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
|
|
530
|
+
// WS4 governance modules (injectable for hermetic tests).
|
|
531
|
+
const governor = opts.governor || resourceGovernor;
|
|
532
|
+
const rateGuard = opts.rateGuard || rateGuardModule;
|
|
533
|
+
const budgetGuard = opts.budgetGuard || budgetGuardModule;
|
|
534
|
+
|
|
535
|
+
/**
|
|
536
|
+
* Admission gate for an escalation. Folds the shared 429 breaker + the
|
|
537
|
+
* resource governor (+ budget essential-only) into one decision. Fail-open:
|
|
538
|
+
* any throw → admit (Invariant: a governance bug must not wedge cadences).
|
|
539
|
+
* Returns { admit:boolean, reason }.
|
|
540
|
+
*/
|
|
541
|
+
function governanceGate() {
|
|
542
|
+
try {
|
|
543
|
+
const rb = rateGuard.checkRateLimit(RATE_PROVIDER, { agentRoot });
|
|
544
|
+
if (!rb.allowed) return { admit: false, reason: "rate-limited" };
|
|
545
|
+
let mode = null;
|
|
546
|
+
try { if (budgetGuard.dailyStatus({ agentRoot }).essentialOnly) mode = "essential-only"; }
|
|
547
|
+
catch { /* budget read best-effort */ }
|
|
548
|
+
const adm = governor.admit({ source: "cadence", mode }, governor.defaultDeps({ agentRoot }));
|
|
549
|
+
if (adm.decision !== "ADMIT") return { admit: false, reason: adm.reason || adm.decision };
|
|
550
|
+
return { admit: true };
|
|
551
|
+
} catch {
|
|
552
|
+
return { admit: true, reason: "governor-error-fail-open" };
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
const stats = {
|
|
557
|
+
started_at: new Date().toISOString(),
|
|
558
|
+
received: 0,
|
|
559
|
+
inline: 0,
|
|
560
|
+
escalated: 0,
|
|
561
|
+
skipped_emergency_stop: 0,
|
|
562
|
+
skipped_circuit_open: 0,
|
|
563
|
+
skipped_backoff: 0,
|
|
564
|
+
dlq: 0,
|
|
565
|
+
retries: 0,
|
|
566
|
+
spawn_failures: 0,
|
|
567
|
+
last_event_id: null,
|
|
568
|
+
last_decision: null,
|
|
569
|
+
};
|
|
570
|
+
|
|
571
|
+
let stopping = false;
|
|
572
|
+
let activeTick = null;
|
|
573
|
+
let timers = [];
|
|
574
|
+
let activeSubSessions = 0;
|
|
575
|
+
|
|
576
|
+
// Audit L6: stale-claim recovery in lib/cadence-bus.mjs is mtime-based and
|
|
577
|
+
// cannot distinguish a crashed handler from a slow-but-live one. A
|
|
578
|
+
// long-running escalate keeps its claim in claimed/ for the whole sub-session
|
|
579
|
+
// run (default cap 30m). recoverStaleClaims' default threshold is also 30m,
|
|
580
|
+
// so a sub-session that runs near the cap could have its claim swept while
|
|
581
|
+
// still executing — producing a SECOND sub-session for the same cadence tick.
|
|
582
|
+
//
|
|
583
|
+
// The consumer is the single owner of cadence escalation, so it knows
|
|
584
|
+
// exactly which claim ids are in-flight. We track them here and, immediately
|
|
585
|
+
// before every recovery sweep, re-stamp those files' mtimes to "now". That
|
|
586
|
+
// keeps a known-active claim out of the stale window without touching the
|
|
587
|
+
// mtime fallback that still recovers genuinely crashed handlers (whose ids
|
|
588
|
+
// are NOT in this set). We deliberately filter consumer-side rather than
|
|
589
|
+
// change recoverStaleClaims' signature (it lives in lib/cadence-bus.mjs,
|
|
590
|
+
// owned by another change-set).
|
|
591
|
+
const inFlightClaimIds = new Set();
|
|
592
|
+
|
|
593
|
+
// Injectable clock so tests can drive the stale window deterministically.
|
|
594
|
+
const nowMs = typeof opts.now === "function" ? opts.now : Date.now;
|
|
595
|
+
|
|
596
|
+
/**
|
|
597
|
+
* Re-stamp the mtime of every in-flight claim file to the current time so
|
|
598
|
+
* the next mtime-based stale sweep treats it as fresh. Best-effort: a missing
|
|
599
|
+
* file (already completed/recovered) or a permission error is ignored — the
|
|
600
|
+
* mtime fallback still protects genuinely crashed handlers.
|
|
601
|
+
*/
|
|
602
|
+
function keepActiveClaimsFresh() {
|
|
603
|
+
if (inFlightClaimIds.size === 0) return;
|
|
604
|
+
const when = new Date(nowMs());
|
|
605
|
+
for (const id of inFlightClaimIds) {
|
|
606
|
+
try {
|
|
607
|
+
const claimedPath = join(paths.claimed, `${id}.json`);
|
|
608
|
+
if (existsSync(claimedPath)) utimesSync(claimedPath, when, when);
|
|
609
|
+
} catch { /* best-effort; mtime fallback still applies if we miss one */ }
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
/**
|
|
614
|
+
* Recover stale claims, first protecting the consumer's known-active claims
|
|
615
|
+
* (audit L6). All call sites route through here so the active-id guard is
|
|
616
|
+
* applied uniformly.
|
|
617
|
+
*/
|
|
618
|
+
function recoverStaleClaimsGuarded() {
|
|
619
|
+
keepActiveClaimsFresh();
|
|
620
|
+
return recoverStaleClaims(agentRoot);
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
// Per-cadence reliability state. Tracks consecutive failure count and
|
|
624
|
+
// the earliest moment we'll allow another spawn for that cadence.
|
|
625
|
+
// Persists nothing — circuit state is in-memory only. On daemon restart
|
|
626
|
+
// we get a fresh slate; that's intentional (operators expect a restart
|
|
627
|
+
// to mean "try again now").
|
|
628
|
+
const cadenceState = new Map(); // cadence → { failures, openUntil, nextAllowedAt }
|
|
629
|
+
|
|
630
|
+
function getCadenceState(cadence) {
|
|
631
|
+
let s = cadenceState.get(cadence);
|
|
632
|
+
if (!s) { s = { failures: 0, openUntil: 0, nextAllowedAt: 0 }; cadenceState.set(cadence, s); }
|
|
633
|
+
return s;
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
function recordSubsessionSuccess(cadence) {
|
|
637
|
+
const s = getCadenceState(cadence);
|
|
638
|
+
s.failures = 0;
|
|
639
|
+
s.openUntil = 0;
|
|
640
|
+
s.nextAllowedAt = 0;
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
function recordSubsessionFailure(cadence) {
|
|
644
|
+
const s = getCadenceState(cadence);
|
|
645
|
+
s.failures += 1;
|
|
646
|
+
// Exponential back-off honouring the (test-overridable) schedule.
|
|
647
|
+
const idx = Math.min(s.failures, backoffSchedule.length - 1);
|
|
648
|
+
s.nextAllowedAt = Date.now() + backoffSchedule[idx];
|
|
649
|
+
if (s.failures >= circuitThreshold) {
|
|
650
|
+
s.openUntil = Date.now() + circuitDurationMs;
|
|
651
|
+
log({ level: "error", stage: "circuit_opened", cadence, failures: s.failures, open_until: new Date(s.openUntil).toISOString() });
|
|
652
|
+
writeCircuitFile();
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
function writeCircuitFile() {
|
|
657
|
+
// Persist the open-circuit snapshot so doctor + the operator can see
|
|
658
|
+
// which cadences are currently held back without scraping logs.
|
|
659
|
+
const open = {};
|
|
660
|
+
for (const [cad, s] of cadenceState.entries()) {
|
|
661
|
+
if (s.openUntil > Date.now()) {
|
|
662
|
+
open[cad] = { failures: s.failures, open_until: new Date(s.openUntil).toISOString() };
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
const path = join(agentRoot, "state/cadence-bus/circuit-open.json");
|
|
666
|
+
try {
|
|
667
|
+
if (Object.keys(open).length === 0) {
|
|
668
|
+
// Remove the file when nothing is open.
|
|
669
|
+
|
|
670
|
+
try { unlinkSync(path); } catch { /* */ }
|
|
671
|
+
} else {
|
|
672
|
+
writeFileSync(path, JSON.stringify({ generated: new Date().toISOString(), open }, null, 2) + "\n");
|
|
673
|
+
}
|
|
674
|
+
} catch { /* best-effort */ }
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
function isCadenceAllowed(cadence) {
|
|
678
|
+
const s = getCadenceState(cadence);
|
|
679
|
+
const now = Date.now();
|
|
680
|
+
if (s.openUntil > now) return { allowed: false, reason: "circuit-open", retry_at: s.openUntil };
|
|
681
|
+
if (s.nextAllowedAt > now) return { allowed: false, reason: "backoff", retry_at: s.nextAllowedAt };
|
|
682
|
+
// Circuit closes automatically when openUntil passes.
|
|
683
|
+
if (s.openUntil && s.openUntil <= now) {
|
|
684
|
+
s.openUntil = 0;
|
|
685
|
+
s.failures = 0;
|
|
686
|
+
log({ level: "info", stage: "circuit_closed", cadence });
|
|
687
|
+
writeCircuitFile();
|
|
688
|
+
}
|
|
689
|
+
return { allowed: true };
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
function log(entry) {
|
|
693
|
+
const enriched = { ts: new Date().toISOString(), ...entry };
|
|
694
|
+
logBusEvent(agentRoot, enriched);
|
|
695
|
+
if (userLogger) {
|
|
696
|
+
try { userLogger(enriched); } catch { /* never crash on logging */ }
|
|
697
|
+
} else {
|
|
698
|
+
defaultLogger(enriched);
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
function heartbeat() {
|
|
703
|
+
writeHealth(agentRoot, {
|
|
704
|
+
stats: { ...stats, depth: busDepth(agentRoot) },
|
|
705
|
+
active_subsessions: activeSubSessions,
|
|
706
|
+
stopping,
|
|
707
|
+
});
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
async function escalate(event) {
|
|
711
|
+
// Circuit-breaker / back-off gate. If this cadence is currently held
|
|
712
|
+
// back, requeue without spawning. The event keeps its attempt count
|
|
713
|
+
// because the failure was upstream (not a per-event problem).
|
|
714
|
+
const gate = isCadenceAllowed(event.cadence);
|
|
715
|
+
if (!gate.allowed) {
|
|
716
|
+
log({
|
|
717
|
+
level: "warn",
|
|
718
|
+
stage: gate.reason === "circuit-open" ? "skipped_circuit_open" : "skipped_backoff",
|
|
719
|
+
id: event.id,
|
|
720
|
+
cadence: event.cadence,
|
|
721
|
+
retry_at: new Date(gate.retry_at).toISOString(),
|
|
722
|
+
});
|
|
723
|
+
if (gate.reason === "circuit-open") stats.skipped_circuit_open += 1;
|
|
724
|
+
else stats.skipped_backoff += 1;
|
|
725
|
+
// Put the event back in inbox unchanged. Attempt accounting is
|
|
726
|
+
// single-sourced in failTick (audit M5), so we no longer decrement here
|
|
727
|
+
// — a held-back-by-circuit re-queue is not a failed attempt and must
|
|
728
|
+
// not drift the count. Use the atomic requeueTick (audit M6) instead of
|
|
729
|
+
// a bare writeFileSync so a crash can't leave a half-written inbox file.
|
|
730
|
+
requeueTick(agentRoot, event);
|
|
731
|
+
return { ok: false, decision: gate.reason };
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
// WS4 governance gate — beside the concurrency cap. If the host is under
|
|
735
|
+
// memory/load pressure, the 429 breaker is open, or the daily budget is
|
|
736
|
+
// exhausted (essential-only), DEFER this cadence tick: requeue it unchanged
|
|
737
|
+
// and DO NOT call failTick (a governor/rate deferral is an upstream gate,
|
|
738
|
+
// not a per-event failure — burning retry budget here would eventually DLQ
|
|
739
|
+
// a perfectly good cadence just because the box was busy).
|
|
740
|
+
{
|
|
741
|
+
const gov = governanceGate();
|
|
742
|
+
if (!gov.admit) {
|
|
743
|
+
log({
|
|
744
|
+
level: "info",
|
|
745
|
+
stage: "escalate_deferred",
|
|
746
|
+
reason: gov.reason,
|
|
747
|
+
id: event.id,
|
|
748
|
+
cadence: event.cadence,
|
|
749
|
+
});
|
|
750
|
+
requeueTick(agentRoot, event);
|
|
751
|
+
stats.retries += 1;
|
|
752
|
+
stats.last_decision = "deferred";
|
|
753
|
+
return { ok: false, decision: "deferred" };
|
|
754
|
+
}
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
if (activeSubSessions >= MAX_CONCURRENT_SUB_SESSIONS) {
|
|
758
|
+
// Re-queue and try again next tick. Single-owner cadence consumer
|
|
759
|
+
// means this can only happen when a prior tick is still running —
|
|
760
|
+
// queue depth is the right back-pressure signal.
|
|
761
|
+
log({
|
|
762
|
+
level: "info",
|
|
763
|
+
stage: "escalate_deferred",
|
|
764
|
+
id: event.id,
|
|
765
|
+
cadence: event.cadence,
|
|
766
|
+
active_subsessions: activeSubSessions,
|
|
767
|
+
});
|
|
768
|
+
// Re-queue unchanged — concurrent-spawn isn't a per-event failure, so
|
|
769
|
+
// it must not touch the attempt count (single-sourced in failTick,
|
|
770
|
+
// audit M5). Atomic requeueTick (audit M6) replaces the bare
|
|
771
|
+
// writeFileSync that could leave a half-written inbox file.
|
|
772
|
+
requeueTick(agentRoot, event);
|
|
773
|
+
stats.retries += 1;
|
|
774
|
+
return { ok: false, decision: "deferred" };
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
const def = getCadenceDef(event.cadence);
|
|
778
|
+
let promptPath = def?.prompt;
|
|
779
|
+
if (!promptPath) {
|
|
780
|
+
// Unknown cadence — try the conventional location.
|
|
781
|
+
const conventional = `schedules/triggers/${event.cadence}.md`;
|
|
782
|
+
if (existsSync(join(agentRoot, conventional))) {
|
|
783
|
+
promptPath = conventional;
|
|
784
|
+
log({ level: "warn", stage: "escalate_unknown_cadence", id: event.id, cadence: event.cadence, prompt: conventional });
|
|
785
|
+
} else {
|
|
786
|
+
log({ level: "error", stage: "escalate_no_prompt", id: event.id, cadence: event.cadence });
|
|
787
|
+
failTick(agentRoot, event.id, `no handler and no prompt at ${conventional}`, { terminal: true });
|
|
788
|
+
stats.dlq += 1;
|
|
789
|
+
return { ok: false, decision: "dlq-no-prompt" };
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
activeSubSessions += 1;
|
|
794
|
+
// Audit L6: mark this claim in-flight so the periodic stale-claim sweep
|
|
795
|
+
// (which can run concurrently with a long-running sub-session) does not
|
|
796
|
+
// treat its claimed/<id>.json as crashed and re-queue it under us.
|
|
797
|
+
inFlightClaimIds.add(event.id);
|
|
798
|
+
let result;
|
|
799
|
+
try {
|
|
800
|
+
log({ level: "info", stage: "escalating", id: event.id, cadence: event.cadence, prompt: promptPath, reason: event.metadata?.reason || "registry policy" });
|
|
801
|
+
result = await spawnSession({
|
|
802
|
+
agentRoot,
|
|
803
|
+
cadence: event.cadence,
|
|
804
|
+
promptPath,
|
|
805
|
+
timeoutMs: maxSpawnMs,
|
|
806
|
+
log,
|
|
807
|
+
});
|
|
808
|
+
} finally {
|
|
809
|
+
activeSubSessions -= 1;
|
|
810
|
+
inFlightClaimIds.delete(event.id);
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
if (result.ok) {
|
|
814
|
+
completeTick(agentRoot, event.id, {
|
|
815
|
+
decision: "escalated",
|
|
816
|
+
cadence: event.cadence,
|
|
817
|
+
prompt: promptPath,
|
|
818
|
+
exit_code: result.exit_code,
|
|
819
|
+
duration_ms: result.duration_ms,
|
|
820
|
+
stdout_path: result.stdout_path || null,
|
|
821
|
+
stderr_path: result.stderr_path || null,
|
|
822
|
+
});
|
|
823
|
+
recordSubsessionSuccess(event.cadence);
|
|
824
|
+
// WS4: a clean sub-session closes the shared 429 breaker.
|
|
825
|
+
try { rateGuard.recordSuccess(RATE_PROVIDER, { agentRoot }); } catch { /* */ }
|
|
826
|
+
stats.escalated += 1;
|
|
827
|
+
stats.last_decision = "escalated";
|
|
828
|
+
return { ok: true, decision: "escalated", exit_code: result.exit_code };
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
// WS4: if the sub-session failed with a rate-limit / overload signal, open
|
|
832
|
+
// the shared breaker and REQUEUE the tick unchanged (decision:"deferred")
|
|
833
|
+
// instead of failTick — a 429 is an upstream gate, not a per-event failure,
|
|
834
|
+
// so it must not burn this cadence's retry budget toward the DLQ.
|
|
835
|
+
if (rateGuard.classifyStderr(result.stderr_tail || result.error || "")) {
|
|
836
|
+
try {
|
|
837
|
+
const rec = rateGuard.recordRateLimit(RATE_PROVIDER, { agentRoot });
|
|
838
|
+
log({ level: "warn", stage: "subsession_rate_limited", id: event.id, cadence: event.cadence, open_until: rec.openUntil });
|
|
839
|
+
} catch { /* */ }
|
|
840
|
+
// L2: a 429 is a SHARED, provider-side rate-limit signal — the shared
|
|
841
|
+
// breaker (recordRateLimit above) already throttles every cadence. It is
|
|
842
|
+
// NOT evidence that THIS cadence is broken, so we must NOT call
|
|
843
|
+
// recordSubsessionFailure here: doing so would advance the per-cadence
|
|
844
|
+
// circuit toward open and leave a perfectly-healthy cadence circuit-open
|
|
845
|
+
// even after the shared breaker clears. Requeue unchanged (no failTick,
|
|
846
|
+
// no circuit trip); the shared breaker gates re-escalation.
|
|
847
|
+
requeueTick(agentRoot, event);
|
|
848
|
+
stats.retries += 1;
|
|
849
|
+
stats.last_decision = "deferred";
|
|
850
|
+
return { ok: false, decision: "deferred" };
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
// Failure path: log + cap retries low. The exact stderr tail comes
|
|
854
|
+
// from the spawn helper so we never DLQ "blind" again.
|
|
855
|
+
const stderrTail = (result.stderr_tail || "").trim().split("\n").slice(-3).join(" | ");
|
|
856
|
+
log({
|
|
857
|
+
level: "error",
|
|
858
|
+
stage: "subsession_failed",
|
|
859
|
+
id: event.id,
|
|
860
|
+
cadence: event.cadence,
|
|
861
|
+
exit_code: result.exit_code,
|
|
862
|
+
duration_ms: result.duration_ms,
|
|
863
|
+
error: result.error || stderrTail || `exit ${result.exit_code}`,
|
|
864
|
+
stderr_path: result.stderr_path || null,
|
|
865
|
+
});
|
|
866
|
+
stats.spawn_failures += 1;
|
|
867
|
+
recordSubsessionFailure(event.cadence);
|
|
868
|
+
const reason = result.error || (stderrTail ? `exit ${result.exit_code}: ${stderrTail}` : `exit ${result.exit_code}`);
|
|
869
|
+
const outcome = failTick(agentRoot, event.id, reason, { maxAttempts });
|
|
870
|
+
if (outcome?.destination === "dlq") stats.dlq += 1;
|
|
871
|
+
else stats.retries += 1;
|
|
872
|
+
return { ok: false, decision: outcome?.destination || "failed" };
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
// NOTE: do not name this `process` — function declarations are hoisted
|
|
876
|
+
// and would shadow the global `process` object inside startConsumer.
|
|
877
|
+
async function processEvent(event) {
|
|
878
|
+
stats.received += 1;
|
|
879
|
+
stats.last_event_id = event.id;
|
|
880
|
+
|
|
881
|
+
const def = getCadenceDef(event.cadence);
|
|
882
|
+
if (def?.mode === "inline" && typeof def.handler === "function") {
|
|
883
|
+
try {
|
|
884
|
+
const out = await def.handler({ event, agentRoot, log });
|
|
885
|
+
completeTick(agentRoot, event.id, {
|
|
886
|
+
decision: "inline",
|
|
887
|
+
cadence: event.cadence,
|
|
888
|
+
handler_result: out,
|
|
889
|
+
});
|
|
890
|
+
stats.inline += 1;
|
|
891
|
+
stats.last_decision = "inline";
|
|
892
|
+
return { decision: "inline", result: out };
|
|
893
|
+
} catch (err) {
|
|
894
|
+
log({ level: "error", stage: "inline_handler_threw", id: event.id, cadence: event.cadence, error: err.message });
|
|
895
|
+
const outcome = failTick(agentRoot, event.id, `inline-handler-threw: ${err.message}`);
|
|
896
|
+
if (outcome?.destination === "dlq") stats.dlq += 1;
|
|
897
|
+
return { decision: "failed", error: err.message };
|
|
898
|
+
}
|
|
899
|
+
}
|
|
900
|
+
|
|
901
|
+
if (def?.mode === "guarded" && typeof def.guard === "function") {
|
|
902
|
+
let guardOut;
|
|
903
|
+
try {
|
|
904
|
+
guardOut = await def.guard({ event, agentRoot, log });
|
|
905
|
+
} catch (err) {
|
|
906
|
+
log({ level: "error", stage: "guard_threw", id: event.id, cadence: event.cadence, error: err.message });
|
|
907
|
+
failTick(agentRoot, event.id, `guard-threw: ${err.message}`);
|
|
908
|
+
stats.retries += 1;
|
|
909
|
+
return { decision: "failed", error: err.message };
|
|
910
|
+
}
|
|
911
|
+
if (guardOut?.decision === "inline") {
|
|
912
|
+
completeTick(agentRoot, event.id, {
|
|
913
|
+
decision: "inline_via_guard",
|
|
914
|
+
cadence: event.cadence,
|
|
915
|
+
guard_result: guardOut,
|
|
916
|
+
});
|
|
917
|
+
stats.inline += 1;
|
|
918
|
+
stats.last_decision = "inline_via_guard";
|
|
919
|
+
return { decision: "inline_via_guard", result: guardOut };
|
|
920
|
+
}
|
|
921
|
+
// Guard says escalate — record WHY so the archived event shows the
|
|
922
|
+
// substantive pre-check that justified spawning a sub-session. Persist
|
|
923
|
+
// the annotated metadata to claimed/<id>.json so completeTick picks it
|
|
924
|
+
// up when it archives to processed/.
|
|
925
|
+
event.metadata = {
|
|
926
|
+
...(event.metadata || {}),
|
|
927
|
+
reason: guardOut?.reason || "guard:escalate",
|
|
928
|
+
guard: guardOut,
|
|
929
|
+
};
|
|
930
|
+
try {
|
|
931
|
+
const claimedPath = join(paths.claimed, `${event.id}.json`);
|
|
932
|
+
writeFileSync(claimedPath, JSON.stringify(event, null, 2) + "\n");
|
|
933
|
+
} catch (err) {
|
|
934
|
+
log({ level: "warn", stage: "persist_metadata_failed", id: event.id, error: err.message });
|
|
935
|
+
}
|
|
936
|
+
return escalate(event);
|
|
937
|
+
}
|
|
938
|
+
|
|
939
|
+
// No registry entry, or registry says escalate.
|
|
940
|
+
return escalate(event);
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
async function tickOnce() {
|
|
944
|
+
if (stopping) return { processed: 0 };
|
|
945
|
+
if (isEmergencyStop(paths)) {
|
|
946
|
+
stats.skipped_emergency_stop += 1;
|
|
947
|
+
heartbeat();
|
|
948
|
+
log({ level: "warn", stage: "emergency_stop_active" });
|
|
949
|
+
return { processed: 0, emergency_stop: true };
|
|
950
|
+
}
|
|
951
|
+
// Recover stale claims occasionally — done by the periodic timer too,
|
|
952
|
+
// but every tick is cheap because it short-circuits on empty claimed/.
|
|
953
|
+
// Routed through the guarded wrapper so an in-flight claim is never swept
|
|
954
|
+
// (audit L6).
|
|
955
|
+
recoverStaleClaimsGuarded();
|
|
956
|
+
|
|
957
|
+
let processed = 0;
|
|
958
|
+
let escalatedThisTick = 0;
|
|
959
|
+
// Drain inline events as much as the consumer can in one tick; cap
|
|
960
|
+
// sub-session escalations at 1 per tick so a fast-failing cadence
|
|
961
|
+
// can't burn a whole minute's worth of retries inside a single poll.
|
|
962
|
+
// The next poll (DEFAULT_POLL_MS later) will pick up where we left off.
|
|
963
|
+
while (!stopping) {
|
|
964
|
+
const claim = claimNextTick(agentRoot);
|
|
965
|
+
if (!claim) break;
|
|
966
|
+
const event = claim.event;
|
|
967
|
+
activeTick = event.id;
|
|
968
|
+
let didEscalate = false;
|
|
969
|
+
try {
|
|
970
|
+
const def = getCadenceDef(event.cadence);
|
|
971
|
+
const willEscalate = !def || (def.mode !== "inline" && (def.mode !== "guarded" || true));
|
|
972
|
+
// Roughly: if it's not a registry-inline cadence, we MAY escalate.
|
|
973
|
+
// We don't yet know if the guard will say inline; processEvent
|
|
974
|
+
// will tell us via stats. Use the escalated stats delta as the
|
|
975
|
+
// signal that an actual sub-session ran this iteration.
|
|
976
|
+
const before = stats.escalated + stats.spawn_failures + stats.skipped_circuit_open + stats.skipped_backoff;
|
|
977
|
+
await processEvent(event);
|
|
978
|
+
const after = stats.escalated + stats.spawn_failures + stats.skipped_circuit_open + stats.skipped_backoff;
|
|
979
|
+
if (after > before) didEscalate = true;
|
|
980
|
+
// Silence unused var warning.
|
|
981
|
+
void willEscalate;
|
|
982
|
+
} finally {
|
|
983
|
+
activeTick = null;
|
|
984
|
+
}
|
|
985
|
+
processed += 1;
|
|
986
|
+
if (didEscalate) escalatedThisTick += 1;
|
|
987
|
+
// Hard cap: at most ONE sub-session spawn per tick. Inline ticks
|
|
988
|
+
// keep draining freely (they're cheap).
|
|
989
|
+
if (escalatedThisTick >= 1) break;
|
|
990
|
+
if (processed >= 16) break; // soft batch cap
|
|
991
|
+
}
|
|
992
|
+
return { processed };
|
|
993
|
+
}
|
|
994
|
+
|
|
995
|
+
// Background loops
|
|
996
|
+
const pollTimer = setInterval(() => {
|
|
997
|
+
tickOnce().catch((err) => {
|
|
998
|
+
log({ level: "error", stage: "tick_threw", error: err?.message || String(err) });
|
|
999
|
+
});
|
|
1000
|
+
}, pollMs);
|
|
1001
|
+
pollTimer.unref?.();
|
|
1002
|
+
timers.push(pollTimer);
|
|
1003
|
+
|
|
1004
|
+
const hbTimer = setInterval(() => heartbeat(), heartbeatMs);
|
|
1005
|
+
hbTimer.unref?.();
|
|
1006
|
+
timers.push(hbTimer);
|
|
1007
|
+
|
|
1008
|
+
const recoveryTimer = setInterval(() => {
|
|
1009
|
+
try {
|
|
1010
|
+
const out = recoverStaleClaimsGuarded();
|
|
1011
|
+
if (out.scanned > 0) {
|
|
1012
|
+
log({ level: "info", stage: "periodic_recovery", ...out });
|
|
1013
|
+
}
|
|
1014
|
+
} catch (err) {
|
|
1015
|
+
log({ level: "error", stage: "periodic_recovery_failed", error: err.message });
|
|
1016
|
+
}
|
|
1017
|
+
// Retention sweep (audit M8) shares the recovery cadence. Isolated in its
|
|
1018
|
+
// own try/catch so a sweep failure can never take down stale-claim
|
|
1019
|
+
// recovery — the more critical of the two.
|
|
1020
|
+
try {
|
|
1021
|
+
const swept = sweepRetention(agentRoot);
|
|
1022
|
+
if (
|
|
1023
|
+
swept.processedDirsDeleted || swept.dlqDeleted || swept.failedDeleted ||
|
|
1024
|
+
swept.queueRotated || swept.logsDeleted
|
|
1025
|
+
) {
|
|
1026
|
+
log({ level: "info", stage: "periodic_retention", ...swept });
|
|
1027
|
+
}
|
|
1028
|
+
} catch (err) {
|
|
1029
|
+
log({ level: "error", stage: "periodic_retention_failed", error: err.message });
|
|
1030
|
+
}
|
|
1031
|
+
}, recoveryMs);
|
|
1032
|
+
recoveryTimer.unref?.();
|
|
1033
|
+
timers.push(recoveryTimer);
|
|
1034
|
+
|
|
1035
|
+
// Initial sweep + heartbeat
|
|
1036
|
+
recoverStaleClaimsGuarded();
|
|
1037
|
+
heartbeat();
|
|
1038
|
+
log({ level: "info", stage: "consumer_started", pollMs, heartbeatMs, recoveryMs });
|
|
1039
|
+
|
|
1040
|
+
async function stop() {
|
|
1041
|
+
if (stopping) return;
|
|
1042
|
+
stopping = true;
|
|
1043
|
+
for (const t of timers) clearInterval(t);
|
|
1044
|
+
timers = [];
|
|
1045
|
+
log({ level: "info", stage: "consumer_stopping", active_tick: activeTick });
|
|
1046
|
+
heartbeat();
|
|
1047
|
+
}
|
|
1048
|
+
|
|
1049
|
+
function getStats() {
|
|
1050
|
+
return { ...stats, depth: busDepth(agentRoot), agent_root: agentRoot };
|
|
1051
|
+
}
|
|
1052
|
+
|
|
1053
|
+
return {
|
|
1054
|
+
stop,
|
|
1055
|
+
getStats,
|
|
1056
|
+
tickOnce,
|
|
1057
|
+
_paths: getBusPaths(agentRoot),
|
|
1058
|
+
// Test hooks (audit L6). Let tests drive the active-id guard and the
|
|
1059
|
+
// guarded recovery deterministically without spawning a real sub-session.
|
|
1060
|
+
_recoverStaleClaimsGuarded: recoverStaleClaimsGuarded,
|
|
1061
|
+
_markInFlight: (id) => inFlightClaimIds.add(id),
|
|
1062
|
+
_clearInFlight: (id) => inFlightClaimIds.delete(id),
|
|
1063
|
+
};
|
|
1064
|
+
}
|
|
1065
|
+
|
|
1066
|
+
// Allow `node scripts/daemon/cadence-consumer.mjs` to run the consumer
|
|
1067
|
+
// standalone (useful for development and for migrations that need to
|
|
1068
|
+
// drain the bus without starting the full daemon).
|
|
1069
|
+
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
1070
|
+
const consumer = startConsumer({});
|
|
1071
|
+
const shutdown = async (sig) => {
|
|
1072
|
+
process.stderr.write(`[cadence-consumer] caught ${sig}, stopping…\n`);
|
|
1073
|
+
await consumer.stop();
|
|
1074
|
+
process.exit(0);
|
|
1075
|
+
};
|
|
1076
|
+
process.on("SIGTERM", () => shutdown("SIGTERM"));
|
|
1077
|
+
process.on("SIGINT", () => shutdown("SIGINT"));
|
|
1078
|
+
// Keep the process alive — unref'd timers won't otherwise.
|
|
1079
|
+
setInterval(() => {}, 60_000);
|
|
1080
|
+
}
|