@vellumai/assistant 0.6.3 → 0.6.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.prettierignore +5 -0
- package/ARCHITECTURE.md +298 -39
- package/Dockerfile +14 -3
- package/README.md +3 -4
- package/bun.lock +13 -16
- package/docs/architecture/integrations.md +1 -20
- package/docs/architecture/security.md +16 -16
- package/docs/backup-troubleshooting.md +52 -0
- package/docs/browser-use-architecture-phase2.md +174 -0
- package/docs/error-handling.md +111 -0
- package/docs/skills.md +10 -10
- package/docs/stt-provider-onboarding.md +121 -0
- package/knip.json +20 -3
- package/node_modules/@vellumai/ces-contracts/bun.lock +8 -6
- package/node_modules/@vellumai/ces-contracts/package.json +5 -4
- package/node_modules/@vellumai/ces-contracts/src/__tests__/trust-rules.test.ts +471 -0
- package/node_modules/@vellumai/ces-contracts/src/trust-rules.ts +398 -4
- package/node_modules/@vellumai/credential-storage/bun.lock +2 -2
- package/node_modules/@vellumai/credential-storage/package.json +2 -2
- package/node_modules/@vellumai/credential-storage/src/oauth-runtime.ts +20 -2
- package/node_modules/@vellumai/egress-proxy/bun.lock +2 -2
- package/node_modules/@vellumai/egress-proxy/package.json +2 -2
- package/openapi.yaml +1094 -72
- package/package.json +9 -8
- package/scripts/generate-openapi.ts +50 -12
- package/scripts/test.sh +73 -18
- package/src/__tests__/agent-image-optimize.test.ts +28 -0
- package/src/__tests__/agent-loop-callsite-precedence.test.ts +318 -0
- package/src/__tests__/agent-loop-sentry-hygiene.test.ts +137 -0
- package/src/__tests__/agent-loop.test.ts +235 -1
- package/src/__tests__/anthropic-error-formatting.test.ts +98 -0
- package/src/__tests__/anthropic-provider.test.ts +434 -12
- package/src/__tests__/approval-cascade.test.ts +31 -10
- package/src/__tests__/approval-routes-http.test.ts +134 -10
- package/src/__tests__/assistant-attachments.test.ts +44 -0
- package/src/__tests__/assistant-feature-flags-integration.test.ts +29 -0
- package/src/__tests__/auto-analysis-end-to-end.test.ts +550 -0
- package/src/__tests__/auto-analysis-prompt.test.ts +50 -0
- package/src/__tests__/browser-fill-credential.test.ts +12 -1
- package/src/__tests__/browser-identifier-parity-guard.test.ts +53 -0
- package/src/__tests__/browser-skill-baseline-tool-payload.test.ts +23 -33
- package/src/__tests__/browser-skill-endstate.test.ts +52 -159
- package/src/__tests__/btw-routes.test.ts +54 -1
- package/src/__tests__/call-controller.test.ts +582 -22
- package/src/__tests__/call-site-routing-provider.test.ts +214 -0
- package/src/__tests__/catalog-cache.test.ts +27 -4
- package/src/__tests__/catalog-files.test.ts +138 -0
- package/src/__tests__/channel-approval-routes.test.ts +4 -4
- package/src/__tests__/channel-invite-transport.test.ts +2 -2
- package/src/__tests__/channel-readiness-routes.test.ts +16 -20
- package/src/__tests__/channel-readiness-service.test.ts +12 -7
- package/src/__tests__/channel-reply-delivery.test.ts +300 -2
- package/src/__tests__/checker.test.ts +576 -502
- package/src/__tests__/clawhub-files.test.ts +347 -0
- package/src/__tests__/cli-command-risk-guard.test.ts +30 -33
- package/src/__tests__/commit-message-enrichment-service.test.ts +36 -19
- package/src/__tests__/compaction-circuit-breaker.test.ts +336 -0
- package/src/__tests__/compaction.benchmark.test.ts +1 -1
- package/src/__tests__/config-analysis.test.ts +83 -0
- package/src/__tests__/config-loader-backfill.test.ts +174 -0
- package/src/__tests__/config-loader-corrupt.test.ts +183 -0
- package/src/__tests__/config-loader-quarantine-bulletin.test.ts +202 -0
- package/src/__tests__/config-schema-cmd.test.ts +11 -5
- package/src/__tests__/config-schema.test.ts +1458 -198
- package/src/__tests__/config-watcher-cleanup-throttle.test.ts +339 -0
- package/src/__tests__/config-watcher.test.ts +45 -10
- package/src/__tests__/contact-store-user-file.test.ts +511 -0
- package/src/__tests__/contacts-write.test.ts +197 -0
- package/src/__tests__/context-token-estimator.test.ts +191 -1
- package/src/__tests__/context-window-manager.test.ts +618 -2
- package/src/__tests__/conversation-abort-tool-results.test.ts +32 -16
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +62 -17
- package/src/__tests__/conversation-agent-loop.test.ts +510 -84
- package/src/__tests__/conversation-attachments.test.ts +1 -1
- package/src/__tests__/conversation-confirmation-signals.test.ts +165 -9
- package/src/__tests__/conversation-error.test.ts +102 -1
- package/src/__tests__/conversation-history-web-search.test.ts +17 -4
- package/src/__tests__/conversation-init.benchmark.test.ts +42 -1
- package/src/__tests__/conversation-launcher-skill-regression.test.ts +51 -0
- package/src/__tests__/conversation-lifecycle.test.ts +336 -0
- package/src/__tests__/conversation-list-source.test.ts +145 -0
- package/src/__tests__/conversation-load-history-repair.test.ts +27 -10
- package/src/__tests__/conversation-pre-run-repair.test.ts +32 -16
- package/src/__tests__/conversation-process-callsite.test.ts +306 -0
- package/src/__tests__/conversation-provider-retry-repair.test.ts +32 -16
- package/src/__tests__/conversation-queue.test.ts +932 -76
- package/src/__tests__/conversation-routes-disk-view.test.ts +299 -1
- package/src/__tests__/conversation-routes-slash-commands.test.ts +31 -3
- package/src/__tests__/conversation-runtime-assembly.test.ts +2790 -55
- package/src/__tests__/conversation-runtime-workspace.test.ts +12 -12
- package/src/__tests__/conversation-skill-tools.test.ts +12 -143
- package/src/__tests__/conversation-slash-commands.test.ts +33 -0
- package/src/__tests__/conversation-slash-queue.test.ts +120 -34
- package/src/__tests__/conversation-slash-unknown.test.ts +32 -16
- package/src/__tests__/conversation-speed-override.test.ts +30 -11
- package/src/__tests__/conversation-surfaces-standalone-payloads.test.ts +1035 -0
- package/src/__tests__/conversation-surfaces-standalone.test.ts +630 -0
- package/src/__tests__/conversation-title-service.test.ts +2 -2
- package/src/__tests__/conversation-tool-setup-batch-authorized.test.ts +226 -0
- package/src/__tests__/conversation-unread-route.test.ts +2 -2
- package/src/__tests__/conversation-usage.test.ts +3 -1
- package/src/__tests__/conversation-workspace-cache-state.test.ts +31 -10
- package/src/__tests__/conversation-workspace-injection.test.ts +45 -15
- package/src/__tests__/conversation-workspace-tool-tracking.test.ts +46 -16
- package/src/__tests__/credential-broker-browser-fill.test.ts +110 -0
- package/src/__tests__/credential-health-service.test.ts +352 -0
- package/src/__tests__/credential-security-invariants.test.ts +8 -3
- package/src/__tests__/credential-storage-oauth-compat.test.ts +18 -0
- package/src/__tests__/credential-storage-static-compat.test.ts +28 -0
- package/src/__tests__/credential-vault-unit.test.ts +495 -3
- package/src/__tests__/credentials-cli.test.ts +32 -16
- package/src/__tests__/cross-provider-web-search.test.ts +230 -35
- package/src/__tests__/daemon-server-persist-and-process-callsite.test.ts +92 -0
- package/src/__tests__/delete-propagation.test.ts +437 -0
- package/src/__tests__/deterministic-verification-control-plane.test.ts +10 -1
- package/src/__tests__/device-id.test.ts +112 -0
- package/src/__tests__/dm-backfill.test.ts +417 -0
- package/src/__tests__/dm-persistence.test.ts +227 -0
- package/src/__tests__/docker-signing-key-bootstrap.test.ts +167 -4
- package/src/__tests__/dynamic-skill-workflow-prompt.test.ts +1 -3
- package/src/__tests__/edit-propagation.test.ts +280 -0
- package/src/__tests__/email-html-renderer.test.ts +71 -0
- package/src/__tests__/email-invite-adapter.test.ts +36 -32
- package/src/__tests__/emit-event-signal.test.ts +71 -0
- package/src/__tests__/ephemeral-permissions.test.ts +93 -3
- package/src/__tests__/estimator-calibration-integration.test.ts +208 -0
- package/src/__tests__/estimator-calibration.test.ts +213 -0
- package/src/__tests__/extension-id-sync-guard.test.ts +101 -15
- package/src/__tests__/file-write-tool.test.ts +151 -1
- package/src/__tests__/filing-service.test.ts +255 -0
- package/src/__tests__/fixtures/mock-chrome-extension.ts +11 -0
- package/src/__tests__/gateway-only-enforcement.test.ts +206 -1
- package/src/__tests__/gateway-only-guard.test.ts +0 -1
- package/src/__tests__/gemini-provider.test.ts +64 -3
- package/src/__tests__/get-skill-detail-audit.test.ts +325 -0
- package/src/__tests__/guardian-grant-minting.test.ts +8 -0
- package/src/__tests__/headless-browser-interactions.test.ts +44 -1
- package/src/__tests__/headless-browser-mode.test.ts +614 -0
- package/src/__tests__/headless-browser-navigate.test.ts +142 -5
- package/src/__tests__/headless-browser-read-tools.test.ts +11 -0
- package/src/__tests__/headless-browser-snapshot.test.ts +10 -0
- package/src/__tests__/heartbeat-service.test.ts +166 -32
- package/src/__tests__/home-state-routes.test.ts +162 -0
- package/src/__tests__/host-bash-proxy.test.ts +0 -5
- package/src/__tests__/host-browser-e2e-cloud.test.ts +138 -4
- package/src/__tests__/host-browser-e2e-self-hosted.test.ts +4 -4
- package/src/__tests__/host-browser-ws-events-e2e.test.ts +103 -0
- package/src/__tests__/host-cu-proxy.test.ts +0 -5
- package/src/__tests__/host-shell-tool.test.ts +124 -18
- package/src/__tests__/http-user-message-parity.test.ts +29 -1
- package/src/__tests__/identity-intro-cache.test.ts +40 -10
- package/src/__tests__/inbound-slack-persistence.test.ts +340 -0
- package/src/__tests__/init-feature-flag-overrides.test.ts +38 -112
- package/src/__tests__/intent-routing.test.ts +1 -40
- package/src/__tests__/jobs-store-upsert-debounced.test.ts +141 -0
- package/src/__tests__/llm-catalog-parity.test.ts +174 -0
- package/src/__tests__/llm-context-normalization.test.ts +609 -0
- package/src/__tests__/llm-context-route-provider.test.ts +86 -5
- package/src/__tests__/llm-resolver.test.ts +214 -0
- package/src/__tests__/llm-schema.test.ts +223 -0
- package/src/__tests__/llm-usage-store.test.ts +363 -0
- package/src/__tests__/managed-proxy-context.test.ts +6 -2
- package/src/__tests__/media-stream-output.test.ts +555 -0
- package/src/__tests__/media-stream-parser.test.ts +374 -0
- package/src/__tests__/media-stream-server-integration.test.ts +1234 -0
- package/src/__tests__/media-stream-stt-session.test.ts +588 -0
- package/src/__tests__/media-turn-detector.test.ts +440 -0
- package/src/__tests__/message-queue.test.ts +125 -0
- package/src/__tests__/messaging-skill-split.test.ts +3 -34
- package/src/__tests__/migration-export-http.test.ts +6 -6
- package/src/__tests__/migration-import-commit-http.test.ts +8 -6
- package/src/__tests__/migration-import-from-url.test.ts +684 -0
- package/src/__tests__/migration-import-preflight-http.test.ts +6 -5
- package/src/__tests__/migration-validate-http.test.ts +3 -3
- package/src/__tests__/mock-gateway-ipc.ts +151 -0
- package/src/__tests__/model-intents.test.ts +10 -84
- package/src/__tests__/notification-decision-fallback.test.ts +0 -10
- package/src/__tests__/notification-decision-identity.test.ts +0 -9
- package/src/__tests__/notification-decision-recipient-context.test.ts +0 -9
- package/src/__tests__/oauth-apps-routes.test.ts +1 -0
- package/src/__tests__/oauth-cli.test.ts +2 -0
- package/src/__tests__/oauth-connect-orchestrator.test.ts +2 -0
- package/src/__tests__/oauth-provider-serializer.test.ts +1 -0
- package/src/__tests__/oauth-providers-routes.test.ts +2 -0
- package/src/__tests__/oauth-store.test.ts +95 -7
- package/src/__tests__/oauth2-gateway-transport.test.ts +257 -9
- package/src/__tests__/oauth2-refresh-retry.test.ts +279 -0
- package/src/__tests__/onboarding-template-contract.test.ts +6 -13
- package/src/__tests__/openai-provider.test.ts +183 -0
- package/src/__tests__/openai-responses-cutover-guard.test.ts +184 -0
- package/src/__tests__/openai-responses-provider.test.ts +1501 -0
- package/src/__tests__/openrouter-provider-only.test.ts +135 -0
- package/src/__tests__/openrouter-token-estimation.test.ts +100 -0
- package/src/__tests__/outbound-slack-persistence.test.ts +293 -0
- package/src/__tests__/permission-checker-host-gate.test.ts +1 -1
- package/src/__tests__/permission-mode.test.ts +16 -0
- package/src/__tests__/permission-types.test.ts +0 -1
- package/src/__tests__/persona-resolver.test.ts +251 -0
- package/src/__tests__/pkb-autoinject.test.ts +37 -1
- package/src/__tests__/platform-bash-auto-approve.test.ts +5 -1
- package/src/__tests__/platform.test.ts +92 -1
- package/src/__tests__/post-turn-tool-result-truncation.test.ts +47 -0
- package/src/__tests__/prechat-onboarding-contract.test.ts +267 -0
- package/src/__tests__/pricing.test.ts +224 -3
- package/src/__tests__/profiler-routes.test.ts +1 -1
- package/src/__tests__/provider-commit-message-generator.test.ts +14 -84
- package/src/__tests__/provider-env-vars-scope.test.ts +52 -0
- package/src/__tests__/provider-error-scenarios.test.ts +135 -6
- package/src/__tests__/provider-managed-proxy-integration.test.ts +42 -11
- package/src/__tests__/provider-registry-ollama.test.ts +1 -2
- package/src/__tests__/proxy-approval-callback.test.ts +0 -1
- package/src/__tests__/qdrant-manager.test.ts +29 -8
- package/src/__tests__/reaction-persistence.test.ts +560 -0
- package/src/__tests__/regenerate-fire-and-forget-trace.test.ts +194 -0
- package/src/__tests__/relationship-state-contract.test.ts +175 -0
- package/src/__tests__/relay-server.test.ts +424 -6
- package/src/__tests__/require-fresh-approval.test.ts +1 -1
- package/src/__tests__/retry-openrouter-only-normalization.test.ts +136 -0
- package/src/__tests__/retry-thinking-tool-choice.test.ts +226 -0
- package/src/__tests__/risk-classifier-parity.test.ts +230 -0
- package/src/__tests__/sanitize-config-for-transfer.test.ts +78 -1
- package/src/__tests__/search-skills-unified.test.ts +118 -0
- package/src/__tests__/secret-ingress-http.test.ts +28 -0
- package/src/__tests__/secret-prompter-channel-fallback.test.ts +125 -0
- package/src/__tests__/secret-routes-managed-proxy.test.ts +2 -3
- package/src/__tests__/secret-scanner-executor.test.ts +5 -1
- package/src/__tests__/secure-keys.test.ts +107 -0
- package/src/__tests__/send-endpoint-busy.test.ts +34 -2
- package/src/__tests__/sequence-store.test.ts +1 -1
- package/src/__tests__/server-history-render.test.ts +80 -0
- package/src/__tests__/settings-routes.test.ts +201 -0
- package/src/__tests__/shell-parser-property.test.ts +13 -13
- package/src/__tests__/skill-cache-store.test.ts +182 -0
- package/src/__tests__/skill-load-feature-flag.test.ts +1 -0
- package/src/__tests__/skills-file-content-endpoint.test.ts +276 -145
- package/src/__tests__/skills-files-catalog-fallback.test.ts +381 -93
- package/src/__tests__/skills.test.ts +19 -30
- package/src/__tests__/skillssh-files.test.ts +446 -0
- package/src/__tests__/slack-app-setup-skill-regression.test.ts +3 -1
- package/src/__tests__/slack-block-formatting.test.ts +110 -0
- package/src/__tests__/slack-channel-config.test.ts +564 -1
- package/src/__tests__/slack-skill.test.ts +3 -8
- package/src/__tests__/starter-bundle.test.ts +35 -0
- package/src/__tests__/stt-catalog-parity.test.ts +282 -0
- package/src/__tests__/stt-stream-session.test.ts +535 -0
- package/src/__tests__/subagent-call-site-routing.test.ts +280 -0
- package/src/__tests__/suggestion-routes.test.ts +160 -3
- package/src/__tests__/system-prompt.test.ts +126 -53
- package/src/__tests__/task-runner.test.ts +3 -1
- package/src/__tests__/tcc-sandbox-deny.test.ts +198 -0
- package/src/__tests__/telephony-stt-routing.test.ts +329 -0
- package/src/__tests__/terminal-tools.test.ts +26 -7
- package/src/__tests__/test-preload.ts +18 -0
- package/src/__tests__/test-support/browser-skill-harness.ts +2 -49
- package/src/__tests__/thread-backfill.test.ts +941 -0
- package/src/__tests__/tool-execution-pipeline.benchmark.test.ts +2 -2
- package/src/__tests__/tool-executor-lifecycle-events.test.ts +10 -6
- package/src/__tests__/tool-executor-shell-integration.test.ts +4 -0
- package/src/__tests__/tool-executor.test.ts +88 -113
- package/src/__tests__/tool-result-truncation.test.ts +36 -0
- package/src/__tests__/trust-store.test.ts +442 -103
- package/src/__tests__/trusted-contact-approval-notifier.test.ts +1 -1
- package/src/__tests__/tts-catalog-parity.test.ts +345 -0
- package/src/__tests__/twilio-routes-twiml.test.ts +512 -114
- package/src/__tests__/twilio-routes.test.ts +376 -0
- package/src/__tests__/unicode.test.ts +293 -0
- package/src/__tests__/update-bulletin-job.test.ts +389 -0
- package/src/__tests__/usage-cache-backfill-migration.test.ts +3 -1
- package/src/__tests__/usage-routes.test.ts +25 -4
- package/src/__tests__/user-reference.test.ts +46 -61
- package/src/__tests__/verification-control-plane-policy.test.ts +5 -22
- package/src/__tests__/voice-config-update.test.ts +403 -0
- package/src/__tests__/voice-quality.test.ts +434 -19
- package/src/__tests__/voice-session-bridge.test.ts +39 -0
- package/src/__tests__/volume-security-guard.test.ts +3 -2
- package/src/__tests__/web-search-history.test.ts +337 -0
- package/src/__tests__/workspace-heartbeat-service.test.ts +7 -0
- package/src/__tests__/workspace-migration-033-stt-service-explicit-config.test.ts +547 -0
- package/src/__tests__/workspace-migration-034-remove-calls-voice-transcription-provider.test.ts +596 -0
- package/src/__tests__/workspace-migration-039-drop-legacy-llm-keys.test.ts +343 -0
- package/src/__tests__/workspace-migration-043-release-notes-latex-rendering.test.ts +202 -0
- package/src/__tests__/workspace-migration-045-release-notes-meet-avatar.test.ts +210 -0
- package/src/__tests__/workspace-migration-drop-user-md.test.ts +368 -0
- package/src/__tests__/workspace-migration-meets.test.ts +244 -0
- package/src/__tests__/workspace-migration-seed-device-id.test.ts +14 -20
- package/src/__tests__/workspace-migration-unify-llm-callsite-configs.test.ts +841 -0
- package/src/__tests__/workspace-policy.test.ts +1 -11
- package/src/acp/client-handler.ts +1 -2
- package/src/agent/image-optimize.ts +24 -12
- package/src/agent/loop.ts +251 -19
- package/src/avatar/resvg-lazy.test.ts +136 -0
- package/src/avatar/resvg-lazy.ts +82 -9
- package/src/avatar/traits-png-sync.ts +21 -1
- package/src/backup/__tests__/backup-key.test.ts +152 -0
- package/src/backup/__tests__/backup-worker.test.ts +767 -0
- package/src/backup/__tests__/list-snapshots.test.ts +87 -0
- package/src/backup/__tests__/local-writer.test.ts +218 -0
- package/src/backup/__tests__/offsite-writer.test.ts +641 -0
- package/src/backup/__tests__/paths.test.ts +300 -0
- package/src/backup/__tests__/restore.test.ts +498 -0
- package/src/backup/__tests__/snapshot-lock.test.ts +352 -0
- package/src/backup/__tests__/stream-crypt.test.ts +228 -0
- package/src/backup/backup-key.ts +137 -0
- package/src/backup/backup-worker.ts +459 -0
- package/src/backup/list-snapshots.ts +147 -0
- package/src/backup/local-writer.ts +133 -0
- package/src/backup/offsite-writer.ts +222 -0
- package/src/backup/paths.ts +226 -0
- package/src/backup/restore.ts +322 -0
- package/src/backup/snapshot-lock.ts +431 -0
- package/src/backup/stream-crypt.ts +263 -0
- package/src/browser/__tests__/operations.test.ts +163 -0
- package/src/browser/identifiers.ts +51 -0
- package/src/browser/operations.ts +660 -0
- package/src/browser/types.ts +81 -0
- package/src/bundler/package-resolver.ts +4 -0
- package/src/calls/audio-store.ts +11 -5
- package/src/calls/call-controller.ts +226 -71
- package/src/calls/call-domain.ts +9 -0
- package/src/calls/call-speech-output.ts +190 -0
- package/src/calls/call-transport.ts +77 -0
- package/src/calls/guardian-question-copy.ts +2 -2
- package/src/calls/media-stream-audio-transcode.ts +173 -0
- package/src/calls/media-stream-output.ts +660 -0
- package/src/calls/media-stream-parser.ts +300 -0
- package/src/calls/media-stream-protocol.ts +166 -0
- package/src/calls/media-stream-server.ts +592 -0
- package/src/calls/media-stream-stt-session.ts +460 -0
- package/src/calls/media-turn-detector.ts +230 -0
- package/src/calls/relay-server.ts +90 -75
- package/src/calls/resolve-call-tts-provider.ts +136 -0
- package/src/calls/telephony-stt-routing.ts +145 -0
- package/src/calls/tts-call-strategy.ts +161 -0
- package/src/calls/tts-text-sanitizer.ts +32 -16
- package/src/calls/twilio-routes.ts +281 -17
- package/src/calls/voice-quality.ts +78 -35
- package/src/calls/voice-session-bridge.ts +9 -1
- package/src/channels/types.ts +16 -0
- package/src/cli/AGENTS.md +1 -1
- package/src/cli/__tests__/run-assistant-command.ts +11 -1
- package/src/cli/commands/__tests__/attachment.test.ts +438 -0
- package/src/cli/commands/__tests__/backup.test.ts +1165 -0
- package/src/cli/commands/__tests__/browser.test.ts +554 -0
- package/src/cli/commands/__tests__/cache.test.ts +623 -0
- package/src/cli/commands/__tests__/domain-register.test.ts +234 -0
- package/src/cli/commands/__tests__/domain-status.test.ts +132 -0
- package/src/cli/commands/__tests__/email-attachment.test.ts +422 -0
- package/src/cli/commands/__tests__/email-download.test.ts +16 -1
- package/src/cli/commands/__tests__/email-list.test.ts +28 -4
- package/src/cli/commands/__tests__/email-register.test.ts +4 -4
- package/src/cli/commands/__tests__/email-send.test.ts +130 -5
- package/src/cli/commands/__tests__/email-status.test.ts +5 -1
- package/src/cli/commands/__tests__/email-unregister.test.ts +34 -5
- package/src/cli/commands/__tests__/image-generation.test.ts +666 -0
- package/src/cli/commands/__tests__/inference-send.test.ts +451 -0
- package/src/cli/commands/__tests__/stt-transcribe.test.ts +454 -0
- package/src/cli/commands/__tests__/task.test.ts +913 -0
- package/src/cli/commands/__tests__/tts-synthesize.test.ts +594 -0
- package/src/cli/commands/__tests__/ui-confirm.test.ts +650 -0
- package/src/cli/commands/__tests__/ui.test.ts +1215 -0
- package/src/cli/commands/__tests__/watchers.test.ts +716 -0
- package/src/cli/commands/attachment.ts +182 -0
- package/src/cli/commands/backup.ts +993 -0
- package/src/cli/commands/browser.ts +350 -0
- package/src/cli/commands/cache.ts +341 -0
- package/src/cli/commands/completions.ts +0 -3
- package/src/cli/commands/config.ts +6 -6
- package/src/cli/commands/conversations-import.ts +347 -0
- package/src/cli/commands/conversations.ts +90 -0
- package/src/cli/commands/credentials.ts +0 -1
- package/src/cli/commands/domain.ts +210 -0
- package/src/cli/commands/email.ts +308 -16
- package/src/cli/commands/image-generation.ts +300 -0
- package/src/cli/commands/inference.ts +200 -0
- package/src/cli/commands/memory.ts +127 -17
- package/src/cli/commands/oauth/__tests__/connect.test.ts +12 -0
- package/src/cli/commands/oauth/__tests__/providers-delete.test.ts +1 -0
- package/src/cli/commands/oauth/__tests__/providers-register.test.ts +1 -0
- package/src/cli/commands/oauth/__tests__/providers-update.test.ts +1 -0
- package/src/cli/commands/oauth/mode.ts +12 -3
- package/src/cli/commands/oauth/providers.ts +15 -0
- package/src/cli/commands/oauth/shared.ts +2 -1
- package/src/cli/commands/platform/__tests__/callback-routes-list.test.ts +4 -10
- package/src/cli/commands/platform/__tests__/connect.test.ts +6 -1
- package/src/cli/commands/platform/__tests__/disconnect.test.ts +7 -2
- package/src/cli/commands/platform/__tests__/status.test.ts +6 -1
- package/src/cli/commands/stt.ts +339 -0
- package/src/cli/commands/task.ts +795 -0
- package/src/cli/commands/trust.ts +50 -19
- package/src/cli/commands/tts.ts +273 -0
- package/src/cli/commands/ui.ts +670 -0
- package/src/cli/commands/watchers.ts +509 -0
- package/src/cli/lib/daemon-credential-client.ts +0 -19
- package/src/cli/program.ts +53 -8
- package/src/cli.ts +0 -37
- package/src/config/__tests__/backup-schema.test.ts +134 -0
- package/src/config/assistant-feature-flags.ts +61 -62
- package/src/config/bundled-skills/app-builder/references/CUSTOM_ROUTES.md +37 -1
- package/src/config/bundled-skills/contacts/SKILL.md +2 -2
- package/src/config/bundled-skills/conversations/tools/rename-conversation.ts +23 -1
- package/src/config/bundled-skills/media-processing/SKILL.md +3 -9
- package/src/config/bundled-skills/media-processing/TOOLS.json +1 -6
- package/src/config/bundled-skills/media-processing/__tests__/audio-transcribe.test.ts +125 -0
- package/src/config/bundled-skills/media-processing/__tests__/extract-keyframes.test.ts +181 -0
- package/src/config/bundled-skills/media-processing/__tests__/preprocess-audio.test.ts +141 -0
- package/src/config/bundled-skills/media-processing/services/audio-transcribe.ts +32 -87
- package/src/config/bundled-skills/media-processing/services/preprocess.ts +8 -4
- package/src/config/bundled-skills/media-processing/services/reduce.ts +1 -1
- package/src/config/bundled-skills/media-processing/tools/extract-keyframes.ts +0 -10
- package/src/config/bundled-skills/messaging/SKILL.md +5 -5
- package/src/config/bundled-skills/messaging/TOOLS.json +4 -0
- package/src/config/bundled-skills/messaging/tools/messaging-archive-by-sender.ts +9 -2
- package/src/config/bundled-skills/messaging/tools/messaging-read.ts +15 -1
- package/src/config/bundled-skills/messaging/tools/messaging-search.ts +21 -1
- package/src/config/bundled-skills/messaging/tools/messaging-send.ts +11 -12
- package/src/config/bundled-skills/phone-calls/SKILL.md +2 -2
- package/src/config/bundled-skills/phone-calls/references/CONFIG.md +28 -18
- package/src/config/bundled-skills/phone-calls/references/TROUBLESHOOTING.md +3 -3
- package/src/config/bundled-skills/settings/TOOLS.json +3 -3
- package/src/config/bundled-skills/settings/tools/voice-config-update.ts +26 -22
- package/src/config/bundled-skills/transcribe/SKILL.md +9 -14
- package/src/config/bundled-skills/transcribe/TOOLS.json +2 -7
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.test.ts +256 -0
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +38 -188
- package/src/config/bundled-tool-registry.ts +0 -167
- package/src/config/env-registry.ts +24 -0
- package/src/config/env.ts +39 -10
- package/src/config/feature-flag-registry.json +63 -15
- package/src/config/llm-resolver.ts +128 -0
- package/src/config/loader.ts +220 -22
- package/src/config/raw-config-utils.ts +30 -2
- package/src/config/sanitize-for-transfer.ts +35 -0
- package/src/config/schema.ts +65 -51
- package/src/config/schemas/__tests__/stt.test.ts +43 -0
- package/src/config/schemas/analysis.ts +32 -0
- package/src/config/schemas/backup.ts +72 -0
- package/src/config/schemas/calls.ts +1 -30
- package/src/config/schemas/elevenlabs.ts +0 -59
- package/src/config/schemas/filing.ts +49 -14
- package/src/config/schemas/heartbeat.ts +27 -10
- package/src/config/schemas/host-browser.ts +47 -1
- package/src/config/schemas/inference.ts +3 -23
- package/src/config/schemas/llm.ts +318 -0
- package/src/config/schemas/memory-lifecycle.ts +14 -2
- package/src/config/schemas/memory-processing.ts +1 -9
- package/src/config/schemas/notifications.ts +4 -11
- package/src/config/schemas/platform.ts +3 -9
- package/src/config/schemas/security.ts +33 -0
- package/src/config/schemas/services.ts +53 -4
- package/src/config/schemas/stt.ts +60 -0
- package/src/config/schemas/tts.ts +283 -0
- package/src/config/schemas/updates.ts +14 -0
- package/src/config/schemas/workspace-git.ts +3 -40
- package/src/config/skills.ts +6 -2
- package/src/config/types.ts +4 -0
- package/src/contacts/contact-store.ts +56 -11
- package/src/contacts/contacts-write.ts +38 -1
- package/src/context/__tests__/compact-prompt.test.ts +45 -0
- package/src/context/__tests__/microcompact.test.ts +805 -0
- package/src/context/estimator-calibration.ts +136 -0
- package/src/context/microcompact.ts +443 -0
- package/src/context/post-turn-tool-result-truncation.ts +3 -2
- package/src/context/prompts/compact.md +12 -0
- package/src/context/token-estimator.ts +61 -3
- package/src/context/tool-result-truncation.ts +2 -1
- package/src/context/window-manager.ts +272 -35
- package/src/credential-execution/approval-bridge.ts +0 -1
- package/src/credential-execution/executable-discovery.ts +23 -2
- package/src/credential-execution/process-manager.test.ts +109 -0
- package/src/credential-execution/process-manager.ts +96 -2
- package/src/credential-health/credential-health-service.ts +366 -0
- package/src/daemon/__tests__/conversation-lifecycle-auto-analyze.test.ts +324 -0
- package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +497 -0
- package/src/daemon/__tests__/conversation-tool-setup.test.ts +17 -8
- package/src/daemon/__tests__/lifecycle-startup-ordering.test.ts +127 -0
- package/src/daemon/approval-generators.ts +29 -4
- package/src/daemon/assistant-attachments.ts +24 -13
- package/src/daemon/classifier.ts +2 -2
- package/src/daemon/config-watcher.ts +99 -6
- package/src/daemon/context-overflow-reducer.ts +4 -1
- package/src/daemon/conversation-agent-loop-handlers.ts +85 -12
- package/src/daemon/conversation-agent-loop.ts +563 -104
- package/src/daemon/conversation-attachments.ts +2 -6
- package/src/daemon/conversation-error.ts +46 -0
- package/src/daemon/conversation-history.ts +40 -6
- package/src/daemon/conversation-launch.ts +220 -0
- package/src/daemon/conversation-lifecycle.ts +85 -11
- package/src/daemon/conversation-messaging.ts +110 -7
- package/src/daemon/conversation-notifiers.ts +5 -0
- package/src/daemon/conversation-process.ts +591 -23
- package/src/daemon/conversation-queue-manager.ts +27 -0
- package/src/daemon/conversation-runtime-assembly.ts +769 -28
- package/src/daemon/conversation-slash.ts +38 -2
- package/src/daemon/conversation-surfaces.ts +483 -5
- package/src/daemon/conversation-tool-setup.ts +35 -5
- package/src/daemon/conversation-usage.ts +8 -5
- package/src/daemon/conversation.ts +193 -47
- package/src/daemon/external-skills-bootstrap.ts +41 -0
- package/src/daemon/guardian-action-generators.ts +34 -14
- package/src/daemon/handlers/config-model.test.ts +86 -0
- package/src/daemon/handlers/config-model.ts +54 -12
- package/src/daemon/handlers/config-slack-channel.ts +269 -94
- package/src/daemon/handlers/conversations.ts +13 -3
- package/src/daemon/handlers/shared.ts +51 -1
- package/src/daemon/handlers/skills.ts +323 -79
- package/src/daemon/handlers/slack-channel-oauth-install.ts +197 -0
- package/src/daemon/host-browser-proxy.ts +2 -1
- package/src/daemon/lifecycle.ts +185 -26
- package/src/daemon/message-protocol.ts +6 -0
- package/src/daemon/message-types/conversations.ts +48 -1
- package/src/daemon/message-types/home.ts +40 -0
- package/src/daemon/message-types/meet.ts +143 -0
- package/src/daemon/message-types/messages.ts +23 -1
- package/src/daemon/message-types/schedules.ts +34 -2
- package/src/daemon/message-types/skills.ts +16 -0
- package/src/daemon/message-types/surfaces.ts +2 -0
- package/src/daemon/message-types/trust.ts +0 -2
- package/src/daemon/parse-actual-tokens-from-error.test.ts +57 -1
- package/src/daemon/parse-actual-tokens-from-error.ts +66 -0
- package/src/daemon/pkb-context-tracker.test.ts +169 -0
- package/src/daemon/pkb-context-tracker.ts +125 -0
- package/src/daemon/pkb-reminder-builder.test.ts +70 -0
- package/src/daemon/pkb-reminder-builder.ts +31 -0
- package/src/daemon/providers-setup.ts +6 -0
- package/src/daemon/server.ts +463 -10
- package/src/daemon/shutdown-handlers.ts +32 -4
- package/src/daemon/shutdown-registry.ts +40 -0
- package/src/daemon/tool-side-effects.ts +9 -9
- package/src/daemon/watch-handler.ts +4 -4
- package/src/daemon/web-search-history.ts +126 -0
- package/src/email/html-renderer.ts +76 -0
- package/src/events/domain-events.ts +0 -1
- package/src/filing/filing-service.ts +9 -10
- package/src/heartbeat/heartbeat-service.ts +156 -22
- package/src/home/__tests__/assistant-feed-authoring.test.ts +156 -0
- package/src/home/__tests__/emit-feed-event.test.ts +169 -0
- package/src/home/__tests__/feed-scheduler.test.ts +222 -0
- package/src/home/__tests__/feed-types.test.ts +275 -0
- package/src/home/__tests__/feed-writer.test.ts +688 -0
- package/src/home/__tests__/phase5-exit-criteria.test.ts +212 -0
- package/src/home/__tests__/platform-gmail-digest.test.ts +222 -0
- package/src/home/__tests__/progress-formula.test.ts +213 -0
- package/src/home/__tests__/relationship-state-writer.test.ts +740 -0
- package/src/home/__tests__/rollup-producer.test.ts +442 -0
- package/src/home/assistant-feed-authoring.ts +128 -0
- package/src/home/emit-feed-event.ts +162 -0
- package/src/home/feed-scheduler.ts +263 -0
- package/src/home/feed-types.ts +235 -0
- package/src/home/feed-writer.ts +469 -0
- package/src/home/platform-gmail-digest.ts +163 -0
- package/src/home/progress-formula.ts +86 -0
- package/src/home/relationship-state-writer.ts +824 -0
- package/src/home/relationship-state.ts +143 -0
- package/src/home/rollup-producer.ts +413 -0
- package/src/home/suggested-prompts.ts +101 -0
- package/src/hooks/runner.ts +7 -0
- package/src/inbound/platform-callback-registration.ts +12 -3
- package/src/inbound/public-ingress-urls.ts +12 -0
- package/src/instrument.ts +1 -1
- package/src/ipc/__tests__/attachment-ipc.test.ts +213 -0
- package/src/ipc/__tests__/browser-ipc.test.ts +339 -0
- package/src/ipc/__tests__/cache-ipc.test.ts +266 -0
- package/src/ipc/__tests__/cli-ipc.test.ts +200 -0
- package/src/ipc/__tests__/socket-path.test.ts +73 -0
- package/src/ipc/__tests__/task-ipc.test.ts +577 -0
- package/src/ipc/__tests__/ui-request-route.test.ts +495 -0
- package/src/ipc/__tests__/watcher-ipc.test.ts +295 -0
- package/src/ipc/cli-client.ts +152 -0
- package/src/ipc/cli-server.ts +252 -0
- package/src/ipc/gateway-client.ts +180 -0
- package/src/ipc/routes/attachment.ts +114 -0
- package/src/ipc/routes/browser-context.ts +61 -0
- package/src/ipc/routes/browser.ts +96 -0
- package/src/ipc/routes/cache.ts +96 -0
- package/src/ipc/routes/index.ts +21 -0
- package/src/ipc/routes/task-queue.ts +226 -0
- package/src/ipc/routes/task.ts +173 -0
- package/src/ipc/routes/ui-request.ts +50 -0
- package/src/ipc/routes/wake-conversation.ts +19 -0
- package/src/ipc/routes/watcher.ts +203 -0
- package/src/ipc/socket-path.ts +100 -0
- package/src/memory/__tests__/auto-analysis-enqueue.test.ts +356 -0
- package/src/memory/__tests__/auto-analysis-guard.test.ts +57 -0
- package/src/memory/__tests__/conversation-analyze-job.test.ts +233 -0
- package/src/memory/__tests__/conversation-group-migration.test.ts +99 -0
- package/src/memory/__tests__/find-analysis-conversation.test.ts +196 -0
- package/src/memory/admin.ts +18 -0
- package/src/memory/app-store.ts +1 -1
- package/src/memory/attachments-store.ts +70 -0
- package/src/memory/auto-analysis-enqueue.ts +127 -0
- package/src/memory/auto-analysis-guard.ts +27 -0
- package/src/memory/cleanup-schedule-state.ts +37 -0
- package/src/memory/conversation-analyze-job.ts +74 -0
- package/src/memory/conversation-attention-store.ts +13 -6
- package/src/memory/conversation-crud.ts +199 -0
- package/src/memory/conversation-disk-view.ts +7 -0
- package/src/memory/conversation-group-migration.ts +65 -1
- package/src/memory/conversation-queries.ts +6 -5
- package/src/memory/conversation-title-service.ts +7 -4
- package/src/memory/db-init.ts +8 -0
- package/src/memory/db-maintenance.ts +108 -0
- package/src/memory/db.ts +1 -0
- package/src/memory/embedding-backend.ts +1 -1
- package/src/memory/graph/compaction.ts +299 -0
- package/src/memory/graph/consolidation.ts +4 -4
- package/src/memory/graph/conversation-graph-memory.ts +104 -29
- package/src/memory/graph/extraction.test.ts +295 -2
- package/src/memory/graph/extraction.ts +181 -51
- package/src/memory/graph/graph-search.test.ts +92 -0
- package/src/memory/graph/graph-search.ts +4 -1
- package/src/memory/graph/narrative.ts +2 -2
- package/src/memory/graph/pattern-scan.ts +2 -2
- package/src/memory/graph/retriever.test.ts +459 -0
- package/src/memory/graph/retriever.ts +257 -66
- package/src/memory/graph/scoring.test.ts +186 -0
- package/src/memory/graph/scoring.ts +31 -1
- package/src/memory/graph/store.ts +41 -0
- package/src/memory/graph/tool-handlers.ts +27 -0
- package/src/memory/graph/tools.ts +6 -1
- package/src/memory/group-crud.ts +6 -1
- package/src/memory/indexer.ts +95 -16
- package/src/memory/job-handlers/cleanup.ts +11 -8
- package/src/memory/job-handlers/conversation-starters.ts +39 -30
- package/src/memory/job-handlers/summarization.ts +2 -2
- package/src/memory/job-utils.ts +7 -1
- package/src/memory/jobs/embed-pkb-file.test.ts +168 -0
- package/src/memory/jobs/embed-pkb-file.ts +54 -0
- package/src/memory/jobs-store.ts +106 -5
- package/src/memory/jobs-worker.ts +26 -9
- package/src/memory/llm-usage-store.ts +92 -56
- package/src/memory/migrations/140-backfill-usage-cache-accounting.ts +1 -1
- package/src/memory/migrations/219-oauth-providers-token-exchange-body-format.ts +15 -0
- package/src/memory/migrations/220-normalize-user-file-by-principal.ts +190 -0
- package/src/memory/migrations/221-conversations-archived-at.ts +16 -0
- package/src/memory/migrations/222-strip-placeholder-sentinels-from-messages.ts +82 -0
- package/src/memory/migrations/index.ts +7 -0
- package/src/memory/migrations/registry.ts +8 -0
- package/src/memory/pkb/pkb-index.test.ts +368 -0
- package/src/memory/pkb/pkb-index.ts +255 -0
- package/src/memory/pkb/pkb-reconcile.test.ts +251 -0
- package/src/memory/pkb/pkb-reconcile.ts +148 -0
- package/src/memory/pkb/pkb-search.test.ts +438 -0
- package/src/memory/pkb/pkb-search.ts +137 -0
- package/src/memory/pkb/types.ts +53 -0
- package/src/memory/qdrant-client.ts +122 -1
- package/src/memory/qdrant-manager.ts +43 -16
- package/src/memory/schema/conversations.ts +2 -0
- package/src/memory/schema/oauth.ts +3 -0
- package/src/memory/slack-thread-store.ts +37 -0
- package/src/memory/usage-buckets.ts +396 -0
- package/src/messaging/providers/gmail/adapter.ts +6 -16
- package/src/messaging/providers/gmail/client.ts +79 -6
- package/src/messaging/providers/gmail/types.ts +7 -0
- package/src/messaging/providers/slack/__tests__/adapter-token-routing.test.ts +282 -0
- package/src/messaging/providers/slack/adapter.ts +155 -38
- package/src/messaging/providers/slack/backfill.test.ts +257 -0
- package/src/messaging/providers/slack/backfill.ts +101 -0
- package/src/messaging/providers/slack/client.ts +16 -0
- package/src/messaging/providers/slack/message-metadata.test.ts +316 -0
- package/src/messaging/providers/slack/message-metadata.ts +123 -0
- package/src/messaging/providers/slack/render-transcript.test.ts +1373 -0
- package/src/messaging/providers/slack/render-transcript.ts +443 -0
- package/src/messaging/providers/slack/types.ts +4 -0
- package/src/messaging/style-analyzer.ts +5 -2
- package/src/notifications/README.md +9 -5
- package/src/notifications/decision-engine.ts +6 -12
- package/src/notifications/preference-extractor.ts +2 -6
- package/src/notifications/signal.ts +5 -0
- package/src/oauth/__tests__/identity-verifier.test.ts +1 -0
- package/src/oauth/byo-connection.test.ts +18 -1
- package/src/oauth/byo-connection.ts +3 -1
- package/src/oauth/connect-orchestrator.ts +2 -0
- package/src/oauth/connection-resolver.ts +6 -2
- package/src/oauth/connection.ts +2 -0
- package/src/oauth/oauth-store.ts +10 -0
- package/src/oauth/platform-connection.test.ts +145 -0
- package/src/oauth/platform-connection.ts +62 -31
- package/src/oauth/seed-providers.ts +10 -1
- package/src/permissions/approval-policy.test.ts +948 -0
- package/src/permissions/approval-policy.ts +257 -0
- package/src/permissions/bash-risk-classifier.test.ts +1208 -0
- package/src/permissions/bash-risk-classifier.ts +707 -0
- package/src/permissions/checker.ts +218 -699
- package/src/permissions/command-registry.test.ts +535 -0
- package/src/permissions/command-registry.ts +825 -0
- package/src/permissions/defaults.ts +71 -75
- package/src/permissions/file-risk-classifier.test.ts +535 -0
- package/src/permissions/file-risk-classifier.ts +274 -0
- package/src/permissions/risk-types.ts +205 -0
- package/src/permissions/secret-prompter.ts +53 -2
- package/src/permissions/skill-risk-classifier.test.ts +311 -0
- package/src/permissions/skill-risk-classifier.ts +214 -0
- package/src/permissions/trust-client.ts +52 -25
- package/src/permissions/trust-store-interface.ts +1 -6
- package/src/permissions/trust-store.ts +164 -65
- package/src/permissions/types.ts +23 -14
- package/src/permissions/web-risk-classifier.test.ts +170 -0
- package/src/permissions/web-risk-classifier.ts +89 -0
- package/src/permissions/workspace-policy.ts +1 -13
- package/src/platform/client.test.ts +10 -0
- package/src/platform/client.ts +19 -1
- package/src/platform/sync-identity.ts +129 -0
- package/src/prompts/persona-resolver.ts +127 -3
- package/src/prompts/system-prompt.ts +78 -38
- package/src/prompts/templates/BOOTSTRAP.md +5 -5
- package/src/prompts/templates/SOUL.md +5 -3
- package/src/prompts/templates/channels/slack.md +20 -0
- package/src/prompts/update-bulletin-job.ts +190 -0
- package/src/prompts/user-reference.ts +20 -17
- package/src/providers/__tests__/context-overflow-error.test.ts +328 -0
- package/src/providers/__tests__/provider-env-vars.test.ts +102 -0
- package/src/providers/__tests__/provider-secret-catalog.test.ts +42 -0
- package/src/providers/__tests__/retry-callsite.test.ts +424 -0
- package/src/providers/anthropic/client.ts +335 -70
- package/src/providers/call-site-routing.ts +71 -0
- package/src/providers/fireworks/client.ts +2 -2
- package/src/providers/gemini/client.ts +74 -3
- package/src/providers/managed-proxy/constants.ts +2 -1
- package/src/providers/model-catalog.ts +502 -28
- package/src/providers/model-intents.ts +8 -8
- package/src/providers/ollama/client.ts +2 -2
- package/src/providers/openai/chat-completions-provider.ts +530 -0
- package/src/providers/openai/client.ts +25 -440
- package/src/providers/openai/responses-provider.ts +579 -0
- package/src/providers/openrouter/client.ts +168 -4
- package/src/providers/provider-env-vars.ts +56 -0
- package/src/providers/provider-secret-catalog.ts +139 -0
- package/src/providers/provider-send-message.ts +22 -5
- package/src/providers/ratelimit.ts +4 -0
- package/src/providers/registry.ts +21 -10
- package/src/providers/retry.ts +185 -39
- package/src/providers/speech-to-text/__tests__/provider-catalog.test.ts +251 -0
- package/src/providers/speech-to-text/__tests__/resolve.test.ts +883 -0
- package/src/providers/speech-to-text/deepgram-realtime.test.ts +980 -0
- package/src/providers/speech-to-text/deepgram-realtime.ts +767 -0
- package/src/providers/speech-to-text/deepgram.test.ts +332 -0
- package/src/providers/speech-to-text/deepgram.ts +115 -0
- package/src/providers/speech-to-text/google-gemini-live-stream.test.ts +743 -0
- package/src/providers/speech-to-text/google-gemini-live-stream.ts +625 -0
- package/src/providers/speech-to-text/google-gemini.test.ts +226 -0
- package/src/providers/speech-to-text/google-gemini.ts +101 -0
- package/src/providers/speech-to-text/openai-whisper-stream.test.ts +564 -0
- package/src/providers/speech-to-text/openai-whisper-stream.ts +381 -0
- package/src/providers/speech-to-text/openai-whisper.test.ts +1 -37
- package/src/providers/speech-to-text/openai-whisper.ts +63 -33
- package/src/providers/speech-to-text/provider-catalog.ts +323 -0
- package/src/providers/speech-to-text/resolve.ts +393 -6
- package/src/providers/speech-to-text/xai-realtime.test.ts +578 -0
- package/src/providers/speech-to-text/xai-realtime.ts +796 -0
- package/src/providers/speech-to-text/xai.test.ts +155 -0
- package/src/providers/speech-to-text/xai.ts +97 -0
- package/src/providers/types.ts +102 -3
- package/src/runtime/AGENTS.md +45 -3
- package/src/runtime/__tests__/agent-wake.test.ts +872 -0
- package/src/runtime/__tests__/interactive-ui.test.ts +673 -0
- package/src/runtime/__tests__/runtime-mode.test.ts +62 -0
- package/src/runtime/__tests__/slack-block-formatting.test.ts +481 -0
- package/src/runtime/agent-wake.ts +553 -0
- package/src/runtime/auth/__tests__/route-policy.test.ts +40 -0
- package/src/runtime/auth/route-policy.ts +34 -5
- package/src/runtime/auth/token-service.ts +56 -1
- package/src/runtime/btw-sidechain.ts +15 -3
- package/src/runtime/capability-tokens.ts +10 -10
- package/src/runtime/channel-invite-transport.ts +1 -1
- package/src/runtime/channel-invite-transports/email.ts +14 -6
- package/src/runtime/channel-readiness-service.ts +12 -22
- package/src/runtime/channel-reply-delivery.ts +106 -2
- package/src/runtime/chrome-extension-registry.ts +38 -2
- package/src/runtime/decision-token.ts +116 -0
- package/src/runtime/gateway-client.ts +2 -2
- package/src/runtime/http-router.ts +32 -0
- package/src/runtime/http-server.ts +447 -11
- package/src/runtime/http-types.ts +29 -3
- package/src/runtime/interactive-ui.ts +362 -0
- package/src/runtime/invite-instruction-generator.ts +2 -2
- package/src/runtime/migrations/__tests__/gcs-signed-url.test.ts +176 -0
- package/src/runtime/migrations/__tests__/vbundle-import-credentials.test.ts +36 -0
- package/src/runtime/migrations/__tests__/vbundle-legacy-user-md.test.ts +360 -0
- package/src/runtime/migrations/__tests__/vbundle-metadata-merge-integration.test.ts +390 -0
- package/src/runtime/migrations/__tests__/vbundle-metadata-merge.test.ts +221 -0
- package/src/runtime/migrations/__tests__/vbundle-streaming-importer.test.ts +1540 -0
- package/src/runtime/migrations/__tests__/vbundle-streaming-validator.test.ts +453 -0
- package/src/runtime/migrations/__tests__/vbundle-tar-stream.test.ts +222 -0
- package/src/runtime/migrations/gcs-signed-url.ts +162 -0
- package/src/runtime/migrations/migration-transport.ts +1 -0
- package/src/runtime/migrations/migration-wizard.ts +1 -0
- package/src/runtime/migrations/vbundle-import-analyzer.ts +77 -1
- package/src/runtime/migrations/vbundle-importer.ts +187 -8
- package/src/runtime/migrations/vbundle-metadata-merge.ts +124 -0
- package/src/runtime/migrations/vbundle-streaming-importer.ts +2522 -0
- package/src/runtime/migrations/vbundle-streaming-validator.ts +244 -0
- package/src/runtime/migrations/vbundle-tar-stream.ts +217 -0
- package/src/runtime/migrations/vbundle-validator.ts +15 -6
- package/src/runtime/pending-interactions.ts +0 -11
- package/src/runtime/routes/__tests__/backup-routes.test.ts +967 -0
- package/src/runtime/routes/__tests__/home-feed-routes.test.ts +618 -0
- package/src/runtime/routes/__tests__/migration-import-credential-filter.test.ts +247 -0
- package/src/runtime/routes/__tests__/migration-vellum-metadata-reconcile.test.ts +246 -0
- package/src/runtime/routes/__tests__/stt-routes.test.ts +406 -0
- package/src/runtime/routes/__tests__/tts-routes.test.ts +474 -0
- package/src/runtime/routes/__tests__/user-route-dispatcher.test.ts +148 -17
- package/src/runtime/routes/app-management-routes.ts +12 -18
- package/src/runtime/routes/approval-prompt-ts-tracker.ts +58 -0
- package/src/runtime/routes/approval-routes.ts +12 -17
- package/src/runtime/routes/approval-strategies/guardian-callback-strategy.ts +9 -0
- package/src/runtime/routes/attachment-routes.test.ts +9 -3
- package/src/runtime/routes/attachment-routes.ts +216 -17
- package/src/runtime/routes/avatar-routes.ts +20 -4
- package/src/runtime/routes/backup-routes.ts +519 -0
- package/src/runtime/routes/browser-extension-pair-routes.ts +82 -23
- package/src/runtime/routes/btw-routes.ts +9 -10
- package/src/runtime/routes/contact-routes.test.ts +298 -0
- package/src/runtime/routes/contact-routes.ts +132 -5
- package/src/runtime/routes/conversation-analysis-routes.ts +22 -142
- package/src/runtime/routes/conversation-management-routes.ts +133 -0
- package/src/runtime/routes/conversation-routes.ts +487 -160
- package/src/runtime/routes/debug-routes.ts +1 -1
- package/src/runtime/routes/diagnostics-routes.ts +6 -4
- package/src/runtime/routes/events-routes.ts +16 -0
- package/src/runtime/routes/filing-routes.ts +93 -0
- package/src/runtime/routes/guardian-approval-interception.ts +33 -3
- package/src/runtime/routes/guardian-approval-prompt.ts +13 -3
- package/src/runtime/routes/home-feed-routes.ts +452 -0
- package/src/runtime/routes/home-state-routes.ts +138 -0
- package/src/runtime/routes/host-browser-routes.ts +3 -14
- package/src/runtime/routes/identity-intro-cache.ts +7 -3
- package/src/runtime/routes/identity-routes.ts +3 -17
- package/src/runtime/routes/inbound-message-handler.ts +912 -2
- package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +113 -2
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +61 -3
- package/src/runtime/routes/inbound-stages/edit-intercept.ts +129 -6
- package/src/runtime/routes/inbound-stages/transcribe-audio.test.ts +46 -39
- package/src/runtime/routes/inbound-stages/transcribe-audio.ts +15 -15
- package/src/runtime/routes/integrations/slack/__tests__/channel.test.ts +137 -0
- package/src/runtime/routes/integrations/slack/__tests__/share.test.ts +179 -0
- package/src/runtime/routes/integrations/slack/channel.ts +36 -6
- package/src/runtime/routes/integrations/slack/share.ts +45 -7
- package/src/runtime/routes/llm-context-normalization.ts +325 -0
- package/src/runtime/routes/memory-item-routes.test.ts +3 -2
- package/src/runtime/routes/migration-routes.ts +722 -91
- package/src/runtime/routes/settings-routes.ts +26 -7
- package/src/runtime/routes/skills-routes.ts +76 -7
- package/src/runtime/routes/stt-routes.ts +233 -0
- package/src/runtime/routes/surface-action-routes.ts +41 -2
- package/src/runtime/routes/trust-rules-routes.ts +30 -14
- package/src/runtime/routes/tts-routes.ts +108 -24
- package/src/runtime/routes/usage-routes.ts +30 -2
- package/src/runtime/routes/user-route-dispatcher.ts +50 -5
- package/src/runtime/routes/user-routes.ts +13 -1
- package/src/runtime/routes/work-items-routes.test.ts +1 -1
- package/src/runtime/routes/work-items-routes.ts +11 -3
- package/src/runtime/runtime-mode.ts +33 -0
- package/src/runtime/services/__tests__/analyze-conversation.test.ts +426 -0
- package/src/runtime/services/__tests__/analyze-deps-singleton.test.ts +67 -0
- package/src/runtime/services/__tests__/auto-analysis-prompt.test.ts +53 -0
- package/src/runtime/services/__tests__/manual-analysis-prompt.test.ts +41 -0
- package/src/runtime/services/analyze-conversation.ts +340 -0
- package/src/runtime/services/analyze-deps-singleton.ts +32 -0
- package/src/runtime/services/auto-analysis-prompt.ts +55 -0
- package/src/runtime/skill-route-registry.ts +71 -0
- package/src/runtime/slack-block-formatting.ts +437 -10
- package/src/schedule/scheduler.ts +58 -0
- package/src/security/__tests__/provider-key-env-fallback.test.ts +119 -0
- package/src/security/__tests__/untrusted-content.test.ts +109 -0
- package/src/security/oauth2.ts +122 -37
- package/src/security/secure-keys.ts +32 -10
- package/src/security/token-manager.ts +35 -13
- package/src/security/untrusted-content.ts +102 -0
- package/src/sequence/engine.ts +23 -0
- package/src/sequence/types.ts +1 -1
- package/src/skills/catalog-cache.ts +26 -7
- package/src/skills/catalog-files.ts +64 -2
- package/src/skills/catalog-install.ts +31 -3
- package/src/skills/category-inference.ts +122 -0
- package/src/skills/clawhub-files.ts +213 -0
- package/src/skills/clawhub.ts +84 -23
- package/src/skills/skill-cache-store.ts +97 -0
- package/src/skills/skill-file-provider.ts +40 -0
- package/src/skills/skillssh-files.ts +395 -0
- package/src/skills/skillssh-registry.ts +4 -4
- package/src/stt/__tests__/daemon-batch-transcriber.test.ts +468 -0
- package/src/stt/__tests__/types.test.ts +89 -0
- package/src/stt/daemon-batch-transcriber.ts +228 -0
- package/src/stt/stt-stream-session.ts +506 -0
- package/src/stt/types.ts +334 -0
- package/src/stt/wav-encoder.test.ts +373 -0
- package/src/stt/wav-encoder.ts +175 -0
- package/src/subagent/manager.ts +79 -27
- package/src/tasks/ephemeral-permissions.ts +9 -4
- package/src/telemetry/usage-telemetry-reporter.ts +27 -5
- package/src/tools/browser/__tests__/browser-mode.test.ts +119 -0
- package/src/tools/browser/__tests__/browser-status.test.ts +166 -0
- package/src/tools/browser/browser-execution.ts +1208 -41
- package/src/tools/browser/browser-manager.ts +45 -0
- package/src/tools/browser/browser-mode-constants.ts +12 -0
- package/src/tools/browser/browser-mode.ts +92 -0
- package/src/tools/browser/browser-status-constants.ts +33 -0
- package/src/tools/browser/cdp-client/__tests__/cdp-inspect-client.test.ts +393 -0
- package/src/tools/browser/cdp-client/__tests__/extension-cdp-client.test.ts +29 -0
- package/src/tools/browser/cdp-client/__tests__/factory.test.ts +1648 -32
- package/src/tools/browser/cdp-client/cdp-inspect/__tests__/discovery.test.ts +264 -0
- package/src/tools/browser/cdp-client/cdp-inspect/discovery.ts +205 -17
- package/src/tools/browser/cdp-client/cdp-inspect-client.ts +254 -21
- package/src/tools/browser/cdp-client/errors.ts +15 -0
- package/src/tools/browser/cdp-client/extension-cdp-client.ts +39 -16
- package/src/tools/browser/cdp-client/factory.ts +797 -87
- package/src/tools/browser/cdp-client/index.ts +16 -2
- package/src/tools/browser/cdp-client/types.ts +68 -0
- package/src/tools/credentials/tool-policy.ts +39 -5
- package/src/tools/credentials/vault.ts +41 -7
- package/src/tools/executor.ts +4 -0
- package/src/tools/filesystem/write.ts +52 -0
- package/src/tools/host-terminal/host-shell.ts +45 -5
- package/src/tools/memory/register.test.ts +185 -0
- package/src/tools/memory/register.ts +3 -1
- package/src/tools/network/web-fetch.ts +25 -12
- package/src/tools/network/web-search.ts +20 -2
- package/src/tools/permission-checker.ts +36 -15
- package/src/tools/policy-context.ts +25 -8
- package/src/tools/registry.ts +55 -3
- package/src/tools/shared/shell-output.ts +3 -1
- package/src/tools/side-effects.ts +0 -9
- package/src/tools/skills/execute.ts +2 -2
- package/src/tools/skills/sandbox-runner.ts +6 -2
- package/src/tools/terminal/backends/native.ts +51 -2
- package/src/tools/terminal/safe-env.ts +11 -2
- package/src/tools/terminal/shell.ts +16 -4
- package/src/tools/tool-manifest.ts +6 -0
- package/src/tools/types.ts +29 -3
- package/src/tools/ui-surface/definitions.ts +6 -1
- package/src/tools/verification-control-plane-policy.ts +1 -1
- package/src/tts/__tests__/provider-adapters.test.ts +1061 -0
- package/src/tts/__tests__/provider-catalog-consistency.test.ts +196 -0
- package/src/tts/__tests__/provider-catalog.test.ts +183 -0
- package/src/tts/__tests__/provider-registry.test.ts +90 -0
- package/src/tts/provider-catalog.ts +219 -0
- package/src/tts/provider-registry.ts +73 -0
- package/src/tts/providers/deepgram-provider.ts +219 -0
- package/src/tts/providers/elevenlabs-provider.ts +211 -0
- package/src/tts/providers/fish-audio-provider.ts +183 -0
- package/src/tts/providers/index.ts +44 -0
- package/src/tts/providers/register-builtins.ts +130 -0
- package/src/tts/providers/xai-provider.ts +224 -0
- package/src/tts/synthesize-text.ts +110 -0
- package/src/tts/tts-config-resolver.ts +78 -0
- package/src/tts/types.ts +199 -0
- package/src/types/onboarding-context.ts +7 -0
- package/src/types/tar-stream.d.ts +66 -0
- package/src/util/abort-reasons.ts +58 -0
- package/src/util/device-id.ts +32 -16
- package/src/util/errors.ts +9 -1
- package/src/util/json.ts +17 -0
- package/src/util/platform.ts +56 -12
- package/src/util/pricing.ts +78 -5
- package/src/util/spawn.ts +1 -1
- package/src/util/truncate.ts +4 -2
- package/src/util/unicode.ts +201 -0
- package/src/version.ts +19 -24
- package/src/watcher/engine.ts +24 -1
- package/src/watcher/providers/google-calendar.ts +134 -8
- package/src/watcher/providers/outlook-calendar.ts +42 -2
- package/src/watcher/watcher-store.ts +31 -0
- package/src/workspace/git-service.ts +23 -4
- package/src/workspace/migrations/003-seed-device-id.ts +9 -3
- package/src/workspace/migrations/017-seed-persona-dirs.ts +68 -4
- package/src/workspace/migrations/029-seed-pkb.ts +1 -1
- package/src/workspace/migrations/031-drop-user-md.ts +317 -0
- package/src/workspace/migrations/031-llm-log-retention-zero-to-null.ts +73 -0
- package/src/workspace/migrations/032-tts-provider-unification.ts +227 -0
- package/src/workspace/migrations/033-stt-service-explicit-config.ts +122 -0
- package/src/workspace/migrations/034-remove-calls-voice-transcription-provider.ts +215 -0
- package/src/workspace/migrations/035-seed-slack-channel-persona.ts +50 -0
- package/src/workspace/migrations/036-update-pkb-index-bar.ts +37 -0
- package/src/workspace/migrations/037-create-meets-dir.ts +61 -0
- package/src/workspace/migrations/038-unify-llm-callsite-configs.ts +516 -0
- package/src/workspace/migrations/039-drop-legacy-llm-keys.ts +171 -0
- package/src/workspace/migrations/040-seed-latency-callsite-defaults.ts +154 -0
- package/src/workspace/migrations/041-backfill-google-gmail-settings-scope.ts +57 -0
- package/src/workspace/migrations/042-fix-backfill-google-gmail-settings-scope.ts +70 -0
- package/src/workspace/migrations/043-release-notes-latex-rendering.ts +75 -0
- package/src/workspace/migrations/044-bump-stale-provider-stream-timeout.ts +51 -0
- package/src/workspace/migrations/045-release-notes-meet-avatar.ts +130 -0
- package/src/workspace/migrations/AGENTS.md +1 -1
- package/src/workspace/migrations/registry.ts +32 -0
- package/src/workspace/provider-commit-message-generator.ts +19 -38
- package/src/workspace/top-level-renderer.ts +13 -1
- package/src/workspace/turn-commit.ts +31 -0
- package/src/__tests__/email-cli.test.ts +0 -297
- package/src/__tests__/email-service-config-fallback.test.ts +0 -102
- package/src/__tests__/outlook-attachments.test.ts +0 -301
- package/src/__tests__/outlook-automation-tools.test.ts +0 -425
- package/src/__tests__/outlook-categories.test.ts +0 -212
- package/src/__tests__/outlook-compose-tools.test.ts +0 -325
- package/src/__tests__/outlook-declutter-tools.test.ts +0 -585
- package/src/__tests__/outlook-follow-up.test.ts +0 -196
- package/src/__tests__/outlook-trash.test.ts +0 -77
- package/src/__tests__/outlook-unsubscribe.test.ts +0 -250
- package/src/__tests__/update-bulletin-format.test.ts +0 -122
- package/src/__tests__/update-bulletin-state.test.ts +0 -135
- package/src/__tests__/update-bulletin.test.ts +0 -277
- package/src/__tests__/update-template-contract.test.ts +0 -29
- package/src/cli/commands/browser-relay.ts +0 -466
- package/src/cli/commands/doctor.ts +0 -341
- package/src/config/bundled-skills/browser/SKILL.md +0 -63
- package/src/config/bundled-skills/browser/TOOLS.json +0 -393
- package/src/config/bundled-skills/browser/tools/browser-click.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-close.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-extract.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-fill-credential.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-hover.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-navigate.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-press-key.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-screenshot.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-scroll.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-select-option.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-snapshot.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-type.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-wait-for-download.ts +0 -32
- package/src/config/bundled-skills/browser/tools/browser-wait-for.ts +0 -12
- package/src/config/bundled-skills/chatgpt-import/SKILL.md +0 -27
- package/src/config/bundled-skills/chatgpt-import/TOOLS.json +0 -27
- package/src/config/bundled-skills/chatgpt-import/tools/chatgpt-import.ts +0 -378
- package/src/config/bundled-skills/gmail/SKILL.md +0 -175
- package/src/config/bundled-skills/gmail/TOOLS.json +0 -558
- package/src/config/bundled-skills/gmail/tools/gmail-archive.ts +0 -149
- package/src/config/bundled-skills/gmail/tools/gmail-attachments.ts +0 -112
- package/src/config/bundled-skills/gmail/tools/gmail-draft.ts +0 -44
- package/src/config/bundled-skills/gmail/tools/gmail-filters.ts +0 -81
- package/src/config/bundled-skills/gmail/tools/gmail-follow-up.ts +0 -108
- package/src/config/bundled-skills/gmail/tools/gmail-forward.ts +0 -146
- package/src/config/bundled-skills/gmail/tools/gmail-label.ts +0 -53
- package/src/config/bundled-skills/gmail/tools/gmail-outreach-scan.ts +0 -220
- package/src/config/bundled-skills/gmail/tools/gmail-send-draft.ts +0 -26
- package/src/config/bundled-skills/gmail/tools/gmail-sender-digest.ts +0 -251
- package/src/config/bundled-skills/gmail/tools/gmail-trash.ts +0 -29
- package/src/config/bundled-skills/gmail/tools/gmail-unsubscribe.ts +0 -122
- package/src/config/bundled-skills/gmail/tools/gmail-vacation.ts +0 -67
- package/src/config/bundled-skills/gmail/tools/scan-result-store.ts +0 -100
- package/src/config/bundled-skills/gmail/tools/shared.ts +0 -47
- package/src/config/bundled-skills/google-calendar/SKILL.md +0 -51
- package/src/config/bundled-skills/google-calendar/TOOLS.json +0 -226
- package/src/config/bundled-skills/google-calendar/calendar-client.ts +0 -223
- package/src/config/bundled-skills/google-calendar/tools/calendar-check-availability.ts +0 -27
- package/src/config/bundled-skills/google-calendar/tools/calendar-create-event.ts +0 -48
- package/src/config/bundled-skills/google-calendar/tools/calendar-get-event.ts +0 -19
- package/src/config/bundled-skills/google-calendar/tools/calendar-list-events.ts +0 -36
- package/src/config/bundled-skills/google-calendar/tools/calendar-rsvp.ts +0 -58
- package/src/config/bundled-skills/google-calendar/tools/shared.ts +0 -17
- package/src/config/bundled-skills/google-calendar/types.ts +0 -97
- package/src/config/bundled-skills/outlook/SKILL.md +0 -196
- package/src/config/bundled-skills/outlook/TOOLS.json +0 -530
- package/src/config/bundled-skills/outlook/tools/outlook-attachments.ts +0 -85
- package/src/config/bundled-skills/outlook/tools/outlook-categories.ts +0 -77
- package/src/config/bundled-skills/outlook/tools/outlook-draft.ts +0 -84
- package/src/config/bundled-skills/outlook/tools/outlook-follow-up.ts +0 -94
- package/src/config/bundled-skills/outlook/tools/outlook-forward.ts +0 -49
- package/src/config/bundled-skills/outlook/tools/outlook-outreach-scan.ts +0 -237
- package/src/config/bundled-skills/outlook/tools/outlook-rules.ts +0 -161
- package/src/config/bundled-skills/outlook/tools/outlook-send-draft.ts +0 -32
- package/src/config/bundled-skills/outlook/tools/outlook-sender-digest.ts +0 -272
- package/src/config/bundled-skills/outlook/tools/outlook-trash.ts +0 -29
- package/src/config/bundled-skills/outlook/tools/outlook-unsubscribe.ts +0 -129
- package/src/config/bundled-skills/outlook/tools/outlook-vacation.ts +0 -87
- package/src/config/bundled-skills/outlook/tools/shared.ts +0 -20
- package/src/config/bundled-skills/outlook-calendar/SKILL.md +0 -51
- package/src/config/bundled-skills/outlook-calendar/TOOLS.json +0 -221
- package/src/config/bundled-skills/outlook-calendar/calendar-client.ts +0 -252
- package/src/config/bundled-skills/outlook-calendar/tools/outlook-calendar-check-availability.ts +0 -53
- package/src/config/bundled-skills/outlook-calendar/tools/outlook-calendar-create-event.ts +0 -74
- package/src/config/bundled-skills/outlook-calendar/tools/outlook-calendar-get-event.ts +0 -18
- package/src/config/bundled-skills/outlook-calendar/tools/outlook-calendar-list-events.ts +0 -46
- package/src/config/bundled-skills/outlook-calendar/tools/outlook-calendar-rsvp.ts +0 -36
- package/src/config/bundled-skills/outlook-calendar/tools/shared.ts +0 -17
- package/src/config/bundled-skills/outlook-calendar/types.ts +0 -120
- package/src/config/bundled-skills/slack/SKILL.md +0 -107
- package/src/config/bundled-skills/tasks/SKILL.md +0 -37
- package/src/config/bundled-skills/tasks/TOOLS.json +0 -353
- package/src/config/bundled-skills/tasks/icon.svg +0 -34
- package/src/config/bundled-skills/tasks/tools/task-delete.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-list-add.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-list-remove.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-list-show.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-list-update.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-list.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-queue-run.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-run.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-save.ts +0 -12
- package/src/config/bundled-skills/watcher/SKILL.md +0 -31
- package/src/config/bundled-skills/watcher/TOOLS.json +0 -167
- package/src/config/bundled-skills/watcher/tools/watcher-create.ts +0 -12
- package/src/config/bundled-skills/watcher/tools/watcher-delete.ts +0 -12
- package/src/config/bundled-skills/watcher/tools/watcher-digest.ts +0 -12
- package/src/config/bundled-skills/watcher/tools/watcher-list.ts +0 -12
- package/src/config/bundled-skills/watcher/tools/watcher-update.ts +0 -12
- package/src/email/guardrails.ts +0 -221
- package/src/email/provider.ts +0 -117
- package/src/email/providers/agentmail.ts +0 -361
- package/src/email/providers/index.ts +0 -65
- package/src/email/service.ts +0 -384
- package/src/email/types.ts +0 -126
- package/src/prompts/templates/UPDATES.md +0 -38
- package/src/prompts/templates/USER.md +0 -13
- package/src/prompts/update-bulletin-format.ts +0 -68
- package/src/prompts/update-bulletin-state.ts +0 -58
- package/src/prompts/update-bulletin-template-path.ts +0 -13
- package/src/prompts/update-bulletin.ts +0 -128
- package/src/providers/speech-to-text/types.ts +0 -17
- package/src/runtime/routes/browser-cdp-routes.ts +0 -229
- package/src/shared/provider-env-vars.ts +0 -19
- package/src/tools/watcher/create.ts +0 -86
- package/src/tools/watcher/delete.ts +0 -36
- package/src/tools/watcher/digest.ts +0 -54
- package/src/tools/watcher/list.ts +0 -83
- package/src/tools/watcher/update.ts +0 -71
|
@@ -0,0 +1,431 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cross-process snapshot mutex.
|
|
3
|
+
*
|
|
4
|
+
* The backup worker's in-process `snapshotInProgress` flag only protects one
|
|
5
|
+
* process from racing itself. A CLI `vellum backup create` run against a live
|
|
6
|
+
* daemon has its own independent copy of the flag, so both processes could
|
|
7
|
+
* drive the pipeline concurrently: two WAL checkpoints against the live DB,
|
|
8
|
+
* two renames into the same `backup-YYYYMMDD-HHMMSS.vbundle` path (the second
|
|
9
|
+
* silently clobbering the first), and two retention-pruner passes racing.
|
|
10
|
+
*
|
|
11
|
+
* This module provides a small cross-process lock backed by an atomic
|
|
12
|
+
* `O_CREAT | O_EXCL` file create under `~/.vellum/backups/.snapshot.lock`.
|
|
13
|
+
* The in-process flag is kept as a fast path; this lock is the source of
|
|
14
|
+
* truth whenever two processes could collide.
|
|
15
|
+
*
|
|
16
|
+
* The implementation mirrors the pattern in `daemon/daemon-control.ts`'s
|
|
17
|
+
* startup lock, with two refinements:
|
|
18
|
+
* 1. The lock file contains the holder's PID so we can detect stale locks
|
|
19
|
+
* by probing liveness with `kill(pid, 0)` rather than a fixed timeout.
|
|
20
|
+
* 2. Acquisition returns a release function so callers can wire it into
|
|
21
|
+
* a `try/finally` without plumbing a separate release import.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import {
|
|
25
|
+
closeSync,
|
|
26
|
+
constants as fsConstants,
|
|
27
|
+
mkdirSync,
|
|
28
|
+
openSync,
|
|
29
|
+
readFileSync,
|
|
30
|
+
renameSync,
|
|
31
|
+
unlinkSync,
|
|
32
|
+
writeSync,
|
|
33
|
+
} from "node:fs";
|
|
34
|
+
import { dirname, join } from "node:path";
|
|
35
|
+
import { kill } from "node:process";
|
|
36
|
+
import { setTimeout as sleep } from "node:timers/promises";
|
|
37
|
+
|
|
38
|
+
import { getLogger } from "../util/logger.js";
|
|
39
|
+
import { getLocalBackupsDir } from "./paths.js";
|
|
40
|
+
|
|
41
|
+
const log = getLogger("snapshot-lock");
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Upper bound on acquire-loop iterations. Each stale-takeover attempt that
|
|
45
|
+
* loses a rename race or re-acquire race counts as one iteration. The loop
|
|
46
|
+
* is bounded so that a pathological contention pattern (many processes each
|
|
47
|
+
* racing into a newly freed slot) cannot turn into an unbounded spin.
|
|
48
|
+
*/
|
|
49
|
+
const MAX_ACQUIRE_ITERATIONS = 8;
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Delay between acquire-loop iterations when we lose a race and need to
|
|
53
|
+
* retry. Small enough that legitimate contention resolves quickly and large
|
|
54
|
+
* enough that we don't hammer the filesystem.
|
|
55
|
+
*/
|
|
56
|
+
const ACQUIRE_RETRY_DELAY_MS = 10;
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Per-attempt delay when we see an empty lock file (suggesting another
|
|
60
|
+
* process is mid-write between `openSync(O_EXCL)` succeeding and
|
|
61
|
+
* `writeSync(payload)` completing). 50ms is comfortably longer than the
|
|
62
|
+
* write-and-close of a ~30-byte payload even on a loaded host.
|
|
63
|
+
*/
|
|
64
|
+
const EMPTY_FILE_RETRY_DELAY_MS = 50;
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Maximum number of times we re-read an empty lock file before giving up
|
|
68
|
+
* and treating it as contended. Three retries at 50ms each bound the wait
|
|
69
|
+
* at ~150ms — long enough to ride out every realistic partial-write
|
|
70
|
+
* window, short enough that genuine contention surfaces quickly. We must
|
|
71
|
+
* not unlink an empty file: it could belong to a live holder that has
|
|
72
|
+
* just won `O_EXCL` but not yet flushed its PID, and unlinking it would
|
|
73
|
+
* let a second process re-acquire and run concurrently.
|
|
74
|
+
*/
|
|
75
|
+
const EMPTY_FILE_MAX_RETRIES = 3;
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Returns the canonical path to the snapshot lock file. The lock lives one
|
|
79
|
+
* level above the local backups directory so it stays in place even when the
|
|
80
|
+
* backup pool is wiped or rotated — e.g. at `~/.vellum/backups/.snapshot.lock`.
|
|
81
|
+
*
|
|
82
|
+
* Placing it one level up (rather than inside the `local/` subdir) also
|
|
83
|
+
* guarantees that pruning never touches the lock file and that the lock
|
|
84
|
+
* survives custom `localDirectory` overrides, since we always use the default
|
|
85
|
+
* parent directory for cross-process coordination.
|
|
86
|
+
*/
|
|
87
|
+
export function getSnapshotLockPath(): string {
|
|
88
|
+
return join(dirname(getLocalBackupsDir()), ".snapshot.lock");
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Check whether a PID refers to a live process. Uses `kill(pid, 0)`, which
|
|
93
|
+
* does not send any signal — it just probes for existence and permission.
|
|
94
|
+
*
|
|
95
|
+
* Returns `false` for obviously invalid PIDs (<= 0) and for any error that
|
|
96
|
+
* indicates the process is gone. Returns `true` for ESRCH-negative results
|
|
97
|
+
* (meaning a process exists) and for EPERM (process exists but is owned by
|
|
98
|
+
* another user — still a live process, still should not be taken over).
|
|
99
|
+
*/
|
|
100
|
+
function isProcessAlive(pid: number): boolean {
|
|
101
|
+
if (!Number.isInteger(pid) || pid <= 0) return false;
|
|
102
|
+
try {
|
|
103
|
+
kill(pid, 0);
|
|
104
|
+
return true;
|
|
105
|
+
} catch (err) {
|
|
106
|
+
const code = (err as NodeJS.ErrnoException).code;
|
|
107
|
+
// EPERM means the PID exists but we cannot signal it — treat as alive so
|
|
108
|
+
// we don't accidentally take over another user's lock.
|
|
109
|
+
if (code === "EPERM") return true;
|
|
110
|
+
return false;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Try to atomically create the lock file with mode `0o600` and the current
|
|
116
|
+
* PID as its contents. Returns `true` on success, `false` if the file
|
|
117
|
+
* already exists (EEXIST), and rethrows any other error.
|
|
118
|
+
*
|
|
119
|
+
* The write-then-close sequence is ordered so that the payload is flushed
|
|
120
|
+
* before any other process can read the file. Callers must still verify
|
|
121
|
+
* ownership after a successful create to defend against the (theoretical)
|
|
122
|
+
* case where an atomic rename replaces the file between our close and our
|
|
123
|
+
* next action.
|
|
124
|
+
*/
|
|
125
|
+
function tryAtomicCreateLock(lockPath: string): boolean {
|
|
126
|
+
let fd: number | null = null;
|
|
127
|
+
try {
|
|
128
|
+
fd = openSync(
|
|
129
|
+
lockPath,
|
|
130
|
+
fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_WRONLY,
|
|
131
|
+
0o600,
|
|
132
|
+
);
|
|
133
|
+
// Payload: `<pid> <timestamp>` so future callers can diagnose stale locks
|
|
134
|
+
// and so humans inspecting the file can tell how long it has been held.
|
|
135
|
+
const payload = `${process.pid} ${Date.now()}\n`;
|
|
136
|
+
writeSync(fd, payload);
|
|
137
|
+
return true;
|
|
138
|
+
} catch (err) {
|
|
139
|
+
const code = (err as NodeJS.ErrnoException).code;
|
|
140
|
+
if (code === "EEXIST") return false;
|
|
141
|
+
throw err;
|
|
142
|
+
} finally {
|
|
143
|
+
if (fd != null) {
|
|
144
|
+
try {
|
|
145
|
+
closeSync(fd);
|
|
146
|
+
} catch {
|
|
147
|
+
// best-effort
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* Read back the lock file after a successful `tryAtomicCreateLock` and
|
|
155
|
+
* confirm that the holder PID matches `process.pid`. Returns `true` if we
|
|
156
|
+
* own the lock, `false` otherwise. Used as a defense-in-depth check against
|
|
157
|
+
* the (theoretical) case where another process atomically replaced our
|
|
158
|
+
* lock file between our `writeSync` and the next caller's `readFileSync`.
|
|
159
|
+
*
|
|
160
|
+
* Returns `false` for an empty file as well — if the contents have not yet
|
|
161
|
+
* been flushed (which should not happen since we `writeSync` before
|
|
162
|
+
* returning), we conservatively treat it as "not our lock".
|
|
163
|
+
*/
|
|
164
|
+
function verifyLockOwnership(lockPath: string): boolean {
|
|
165
|
+
const result = readLockHolder(lockPath);
|
|
166
|
+
return result.kind === "pid" && result.pid === process.pid;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* Result of inspecting the lock file. We distinguish three cases because the
|
|
171
|
+
* acquire loop must react to each one differently:
|
|
172
|
+
*
|
|
173
|
+
* - `pid`: the lock has a parseable holder PID — check liveness and, if
|
|
174
|
+
* dead, proceed to stale-takeover.
|
|
175
|
+
* - `empty`: the file exists but has no parseable PID. Either a live holder
|
|
176
|
+
* is mid-write between `O_EXCL` and `writeSync`, or the file is garbage.
|
|
177
|
+
* Wait-and-retry; if still empty after the budget, surface a conflict.
|
|
178
|
+
* Never unlink — unlinking a live holder's in-progress lock is exactly
|
|
179
|
+
* the TOCTOU double-acquire this module exists to prevent.
|
|
180
|
+
* - `missing`: the file vanished between our `EEXIST` from `O_EXCL` and
|
|
181
|
+
* our read. The holder already released it. Retry the outer acquire
|
|
182
|
+
* loop — `O_EXCL` should now succeed on the empty slot.
|
|
183
|
+
*
|
|
184
|
+
* Collapsing `missing` into `empty` (as a prior revision did) causes
|
|
185
|
+
* `createSnapshotNow` to spuriously fail when a concurrent holder releases
|
|
186
|
+
* the lock during this very narrow window.
|
|
187
|
+
*/
|
|
188
|
+
type LockReadResult =
|
|
189
|
+
| { kind: "pid"; pid: number }
|
|
190
|
+
| { kind: "empty" }
|
|
191
|
+
| { kind: "missing" };
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* Parse the lock file and classify its state. `ENOENT` is surfaced as
|
|
195
|
+
* `missing` so the caller can re-attempt `O_EXCL` acquire; all other read
|
|
196
|
+
* failures (including an empty or unparseable payload) collapse to `empty`
|
|
197
|
+
* so the caller waits for a potential live-writer to flush its PID.
|
|
198
|
+
*/
|
|
199
|
+
function readLockHolder(lockPath: string): LockReadResult {
|
|
200
|
+
let raw: string;
|
|
201
|
+
try {
|
|
202
|
+
raw = readFileSync(lockPath, "utf-8");
|
|
203
|
+
} catch (err) {
|
|
204
|
+
const code = (err as NodeJS.ErrnoException).code;
|
|
205
|
+
if (code === "ENOENT") return { kind: "missing" };
|
|
206
|
+
return { kind: "empty" };
|
|
207
|
+
}
|
|
208
|
+
const trimmed = raw.trim();
|
|
209
|
+
if (trimmed.length === 0) return { kind: "empty" };
|
|
210
|
+
// The payload is `<pid> <timestamp>`, but be lenient about formats: any
|
|
211
|
+
// leading positive integer is treated as the PID.
|
|
212
|
+
const match = /^\d+/.exec(trimmed);
|
|
213
|
+
if (!match) return { kind: "empty" };
|
|
214
|
+
const pid = Number.parseInt(match[0], 10);
|
|
215
|
+
if (!Number.isFinite(pid) || pid <= 0) return { kind: "empty" };
|
|
216
|
+
return { kind: "pid", pid };
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* Attempt to acquire the cross-process snapshot lock.
|
|
221
|
+
*
|
|
222
|
+
* On success, returns an idempotent release function that unlinks the lock
|
|
223
|
+
* file. Callers must invoke it in a `finally` block.
|
|
224
|
+
*
|
|
225
|
+
* On conflict with a live holder, throws an error whose message STARTS WITH
|
|
226
|
+
* "snapshot in progress" so existing consumers that match on that prefix
|
|
227
|
+
* (HTTP 409 mapping, CLI error output) continue to work without change.
|
|
228
|
+
*
|
|
229
|
+
* ## Stale-lock takeover (TOCTOU-safe)
|
|
230
|
+
*
|
|
231
|
+
* The naive "detect stale → unlink → re-acquire" pattern has a TOCTOU
|
|
232
|
+
* race: two processes can both observe the same stale lock, both call
|
|
233
|
+
* `unlink`, and the second unlink removes the *fresh* lock the first
|
|
234
|
+
* process just re-acquired. Both then succeed at `O_EXCL` create and
|
|
235
|
+
* believe they own the lock.
|
|
236
|
+
*
|
|
237
|
+
* This implementation uses a **rename-aside-then-verify** pattern instead:
|
|
238
|
+
*
|
|
239
|
+
* 1. Atomically rename the stale lock to a unique sideband path
|
|
240
|
+
* (`<lockPath>.stale.<pid>.<timestamp>`). `rename(2)` is atomic; if two
|
|
241
|
+
* processes race, only one wins. The loser sees `ENOENT` and retries
|
|
242
|
+
* the acquire loop from the top.
|
|
243
|
+
* 2. The winner attempts `O_EXCL` create on the now-empty `lockPath`. On
|
|
244
|
+
* success it unlinks the sideband file and verifies ownership by
|
|
245
|
+
* reading back the PID it wrote.
|
|
246
|
+
* 3. Post-acquire verification runs for *every* successful create (not
|
|
247
|
+
* just the takeover path). If the read-back PID doesn't match
|
|
248
|
+
* `process.pid`, we release our idea-of-a-lock as a race and throw —
|
|
249
|
+
* crucially, without unlinking, so we don't destroy whoever does own
|
|
250
|
+
* it.
|
|
251
|
+
*
|
|
252
|
+
* ## Partial-write handling
|
|
253
|
+
*
|
|
254
|
+
* There is a tiny window between `openSync(O_EXCL)` succeeding and
|
|
255
|
+
* `writeSync(payload)` completing where another reader could observe a
|
|
256
|
+
* zero-byte lock file. An empty file is ambiguous: it might be a dead
|
|
257
|
+
* writer's debris, OR it might belong to a *live* holder that has just
|
|
258
|
+
* won `O_EXCL` and not yet flushed the PID. We re-read up to
|
|
259
|
+
* `EMPTY_FILE_MAX_RETRIES` times with a short delay; only a lock file
|
|
260
|
+
* with a parseable PID whose owner is not running is ever taken over.
|
|
261
|
+
* If the file is still unreadable after the retry budget, we surface a
|
|
262
|
+
* conflict — unlinking it would race the live-holder case and risk
|
|
263
|
+
* letting two snapshots run concurrently (the exact corruption scenario
|
|
264
|
+
* this lock exists to prevent).
|
|
265
|
+
*
|
|
266
|
+
* ## Bounded retry
|
|
267
|
+
*
|
|
268
|
+
* The loop is bounded by `MAX_ACQUIRE_ITERATIONS`. A pathological contention
|
|
269
|
+
* pattern (e.g. many backup processes each taking over each other's leftovers
|
|
270
|
+
* faster than we can verify ownership) cannot turn into an unbounded spin.
|
|
271
|
+
* After the bound is exhausted we surface a conflict so callers retry.
|
|
272
|
+
*
|
|
273
|
+
* The lock directory is created on demand so first-run scenarios (no
|
|
274
|
+
* `~/.vellum/backups` yet) work without a separate bootstrap step.
|
|
275
|
+
*/
|
|
276
|
+
export async function acquireSnapshotLock(
|
|
277
|
+
lockPath: string,
|
|
278
|
+
): Promise<() => Promise<void>> {
|
|
279
|
+
// Ensure the parent directory exists. `mkdirSync({ recursive: true })` is
|
|
280
|
+
// idempotent — it will not fail if the directory already exists.
|
|
281
|
+
mkdirSync(dirname(lockPath), { recursive: true, mode: 0o700 });
|
|
282
|
+
|
|
283
|
+
let lastHolderPid: number | null = null;
|
|
284
|
+
|
|
285
|
+
for (let attempt = 0; attempt < MAX_ACQUIRE_ITERATIONS; attempt += 1) {
|
|
286
|
+
// --- Step 1: try fresh atomic create ---
|
|
287
|
+
if (tryAtomicCreateLock(lockPath)) {
|
|
288
|
+
// Post-acquire verification. If another process managed to atomically
|
|
289
|
+
// replace our lock between our `writeSync` and now (which should not
|
|
290
|
+
// happen under O_EXCL, but we verify as defense in depth), someone
|
|
291
|
+
// else owns it — report conflict without unlinking so we don't
|
|
292
|
+
// destroy their lock.
|
|
293
|
+
if (verifyLockOwnership(lockPath)) {
|
|
294
|
+
return makeRelease(lockPath);
|
|
295
|
+
}
|
|
296
|
+
const winner = readLockHolder(lockPath);
|
|
297
|
+
throw new Error(
|
|
298
|
+
winner.kind === "pid"
|
|
299
|
+
? `snapshot in progress (locked by pid ${winner.pid})`
|
|
300
|
+
: "snapshot in progress (race detected)",
|
|
301
|
+
);
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
// --- Step 2: inspect the existing lock ---
|
|
305
|
+
let holder = readLockHolder(lockPath);
|
|
306
|
+
|
|
307
|
+
// Race between `tryAtomicCreateLock` (saw `EEXIST`) and this read: the
|
|
308
|
+
// holder released the lock in between. The slot is free — retry the
|
|
309
|
+
// outer loop so `O_EXCL` can succeed. Treating this case as contended
|
|
310
|
+
// (as a prior revision did) caused `createSnapshotNow` to spuriously
|
|
311
|
+
// fail and `runBackupTick` to skip a due cycle.
|
|
312
|
+
if (holder.kind === "missing") {
|
|
313
|
+
await sleep(ACQUIRE_RETRY_DELAY_MS);
|
|
314
|
+
continue;
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
// Partial-write window: the file exists but has no parseable PID. This
|
|
318
|
+
// can happen in the tiny window between `O_EXCL` create and the payload
|
|
319
|
+
// write, so we retry a bounded number of times before deciding what to
|
|
320
|
+
// do with it.
|
|
321
|
+
for (
|
|
322
|
+
let retry = 0;
|
|
323
|
+
retry < EMPTY_FILE_MAX_RETRIES && holder.kind === "empty";
|
|
324
|
+
retry += 1
|
|
325
|
+
) {
|
|
326
|
+
await sleep(EMPTY_FILE_RETRY_DELAY_MS);
|
|
327
|
+
holder = readLockHolder(lockPath);
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
// The lock vanished while we were waiting on the partial-write window.
|
|
331
|
+
// Same recovery as above: retry the outer loop and let `O_EXCL` take
|
|
332
|
+
// the now-empty slot.
|
|
333
|
+
if (holder.kind === "missing") {
|
|
334
|
+
await sleep(ACQUIRE_RETRY_DELAY_MS);
|
|
335
|
+
continue;
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
// If the file is still unreadable, a live holder may be mid-write.
|
|
339
|
+
// Unlinking would let a second acquirer succeed at `O_EXCL` while the
|
|
340
|
+
// first holder still believes it owns the lock — the TOCTOU double-
|
|
341
|
+
// acquire this module exists to prevent. Surface as a conflict and
|
|
342
|
+
// let the caller retry; we only ever take over a lock with a parseable
|
|
343
|
+
// PID that points at a non-running process.
|
|
344
|
+
if (holder.kind === "empty") {
|
|
345
|
+
throw new Error(
|
|
346
|
+
"snapshot in progress (lock holder unidentified; possible partial write)",
|
|
347
|
+
);
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
if (isProcessAlive(holder.pid)) {
|
|
351
|
+
throw new Error(
|
|
352
|
+
`snapshot in progress (locked by pid ${holder.pid})`,
|
|
353
|
+
);
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
// --- Step 3: stale takeover via rename-aside ---
|
|
357
|
+
//
|
|
358
|
+
// Atomically rename the stale lock to a unique sideband path. If two
|
|
359
|
+
// processes race, only one wins the rename — the loser sees ENOENT and
|
|
360
|
+
// retries the acquire loop from the top. The winner's next `tryAcquire`
|
|
361
|
+
// can then succeed on the empty slot.
|
|
362
|
+
const holderPid = holder.pid;
|
|
363
|
+
lastHolderPid = holderPid;
|
|
364
|
+
const sidebandPath = `${lockPath}.stale.${process.pid}.${Date.now()}.${attempt}`;
|
|
365
|
+
log.info(
|
|
366
|
+
{ lockPath, holderPid, sidebandPath },
|
|
367
|
+
"Taking over stale snapshot lock via rename-aside",
|
|
368
|
+
);
|
|
369
|
+
try {
|
|
370
|
+
renameSync(lockPath, sidebandPath);
|
|
371
|
+
} catch (err) {
|
|
372
|
+
const code = (err as NodeJS.ErrnoException).code;
|
|
373
|
+
if (code === "ENOENT") {
|
|
374
|
+
// Another process already renamed the stale lock away. Retry the
|
|
375
|
+
// whole loop — the slot may be free or may have a new legitimate
|
|
376
|
+
// holder.
|
|
377
|
+
await sleep(ACQUIRE_RETRY_DELAY_MS);
|
|
378
|
+
continue;
|
|
379
|
+
}
|
|
380
|
+
throw err;
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
// The sideband file is ours to clean up. Best-effort unlink; if it
|
|
384
|
+
// fails, the file will sit around as orphaned debris, but it does not
|
|
385
|
+
// affect correctness (the name includes our pid and timestamp so it
|
|
386
|
+
// won't collide with a future run).
|
|
387
|
+
try {
|
|
388
|
+
unlinkSync(sidebandPath);
|
|
389
|
+
} catch {
|
|
390
|
+
// best-effort
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
// Loop back to attempt the acquire on the now-empty slot. Do not break
|
|
394
|
+
// out here — control flow falls through to the next iteration which
|
|
395
|
+
// calls `tryAtomicCreateLock` again.
|
|
396
|
+
await sleep(ACQUIRE_RETRY_DELAY_MS);
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
// Ran out of attempts. This should be vanishingly rare — it would require
|
|
400
|
+
// sustained multi-way contention with every attempt losing a rename or
|
|
401
|
+
// acquire race. Surface as a conflict so the caller can retry.
|
|
402
|
+
throw new Error(
|
|
403
|
+
lastHolderPid != null
|
|
404
|
+
? `snapshot in progress (contended, last seen pid ${lastHolderPid})`
|
|
405
|
+
: "snapshot in progress (lock contended)",
|
|
406
|
+
);
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
/**
|
|
410
|
+
* Build an idempotent release function for an acquired lock file. Calling
|
|
411
|
+
* the returned function twice is safe — the second unlink catches ENOENT
|
|
412
|
+
* and returns without error.
|
|
413
|
+
*/
|
|
414
|
+
function makeRelease(lockPath: string): () => Promise<void> {
|
|
415
|
+
let released = false;
|
|
416
|
+
return async () => {
|
|
417
|
+
if (released) return;
|
|
418
|
+
released = true;
|
|
419
|
+
try {
|
|
420
|
+
unlinkSync(lockPath);
|
|
421
|
+
} catch (err) {
|
|
422
|
+
const code = (err as NodeJS.ErrnoException).code;
|
|
423
|
+
if (code !== "ENOENT") {
|
|
424
|
+
log.warn(
|
|
425
|
+
{ err, lockPath },
|
|
426
|
+
"Failed to release snapshot lock (best-effort)",
|
|
427
|
+
);
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
};
|
|
431
|
+
}
|
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Streaming AES-256-GCM file encryption/decryption for backup bundles.
|
|
3
|
+
*
|
|
4
|
+
* The on-disk format is:
|
|
5
|
+
*
|
|
6
|
+
* [12-byte IV][ciphertext...][16-byte GCM auth tag]
|
|
7
|
+
*
|
|
8
|
+
* Both encrypt and decrypt use Node streams so peak memory stays bounded
|
|
9
|
+
* regardless of input size. This is important for backup archives which may
|
|
10
|
+
* run to many gigabytes on larger workspaces.
|
|
11
|
+
*
|
|
12
|
+
* The key must be exactly 32 bytes (AES-256). The IV is randomly generated
|
|
13
|
+
* per call, which is required for GCM semantic security — never reuse an
|
|
14
|
+
* IV with the same key.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import {
|
|
18
|
+
createCipheriv,
|
|
19
|
+
createDecipheriv,
|
|
20
|
+
randomBytes,
|
|
21
|
+
} from "node:crypto";
|
|
22
|
+
import {
|
|
23
|
+
createReadStream,
|
|
24
|
+
createWriteStream,
|
|
25
|
+
} from "node:fs";
|
|
26
|
+
import { open, rename, stat, unlink } from "node:fs/promises";
|
|
27
|
+
import { Readable, Writable } from "node:stream";
|
|
28
|
+
import { pipeline } from "node:stream/promises";
|
|
29
|
+
|
|
30
|
+
// ---------------------------------------------------------------------------
|
|
31
|
+
// Constants
|
|
32
|
+
// ---------------------------------------------------------------------------
|
|
33
|
+
|
|
34
|
+
/** Size of the AES-GCM initialization vector prefix, in bytes. */
|
|
35
|
+
export const ENCRYPTED_HEADER_SIZE = 12;
|
|
36
|
+
|
|
37
|
+
/** Size of the AES-GCM authentication tag suffix, in bytes. */
|
|
38
|
+
export const GCM_TAG_SIZE = 16;
|
|
39
|
+
|
|
40
|
+
const ALGORITHM = "aes-256-gcm";
|
|
41
|
+
const KEY_LENGTH = 32;
|
|
42
|
+
|
|
43
|
+
// ---------------------------------------------------------------------------
|
|
44
|
+
// Helpers
|
|
45
|
+
// ---------------------------------------------------------------------------
|
|
46
|
+
|
|
47
|
+
function assertKey(key: Buffer): void {
|
|
48
|
+
if (key.length !== KEY_LENGTH) {
|
|
49
|
+
throw new Error("Backup encryption key must be 32 bytes");
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
async function safeUnlink(path: string): Promise<void> {
|
|
54
|
+
try {
|
|
55
|
+
await unlink(path);
|
|
56
|
+
} catch {
|
|
57
|
+
// best-effort cleanup — swallow ENOENT and other errors
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function tempPath(outputPath: string): string {
|
|
62
|
+
return `${outputPath}.tmp.${process.pid}.${randomBytes(4).toString("hex")}`;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// ---------------------------------------------------------------------------
|
|
66
|
+
// Encrypt
|
|
67
|
+
// ---------------------------------------------------------------------------
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Stream-encrypt `inputPath` to `outputPath` using AES-256-GCM.
|
|
71
|
+
*
|
|
72
|
+
* Produces `[IV (12 bytes)][ciphertext][auth tag (16 bytes)]` in the output.
|
|
73
|
+
* Writes to a temp file and atomically renames on success; unlinks the temp
|
|
74
|
+
* file on any error so failed writes don't leave partial bundles behind.
|
|
75
|
+
*/
|
|
76
|
+
export async function encryptFile(
|
|
77
|
+
inputPath: string,
|
|
78
|
+
outputPath: string,
|
|
79
|
+
key: Buffer,
|
|
80
|
+
): Promise<void> {
|
|
81
|
+
assertKey(key);
|
|
82
|
+
|
|
83
|
+
const iv = randomBytes(ENCRYPTED_HEADER_SIZE);
|
|
84
|
+
const cipher = createCipheriv(ALGORITHM, key, iv);
|
|
85
|
+
|
|
86
|
+
const tmp = tempPath(outputPath);
|
|
87
|
+
const writeStream = createWriteStream(tmp);
|
|
88
|
+
|
|
89
|
+
try {
|
|
90
|
+
// Write IV first so decrypt can read it without knowing the ciphertext size.
|
|
91
|
+
await new Promise<void>((resolve, reject) => {
|
|
92
|
+
writeStream.write(iv, (err) => (err ? reject(err) : resolve()));
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
// Stream plaintext through the cipher into the output.
|
|
96
|
+
const readStream = createReadStream(inputPath);
|
|
97
|
+
await pipeline(readStream, cipher, writeStream, { end: false });
|
|
98
|
+
|
|
99
|
+
// Append the auth tag after the ciphertext body.
|
|
100
|
+
const tag = cipher.getAuthTag();
|
|
101
|
+
await new Promise<void>((resolve, reject) => {
|
|
102
|
+
writeStream.write(tag, (err) => (err ? reject(err) : resolve()));
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
await new Promise<void>((resolve, reject) => {
|
|
106
|
+
writeStream.end((err?: Error | null) => (err ? reject(err) : resolve()));
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
await rename(tmp, outputPath);
|
|
110
|
+
} catch (err) {
|
|
111
|
+
// Make sure the write stream is closed before we try to unlink the temp file.
|
|
112
|
+
writeStream.destroy();
|
|
113
|
+
await safeUnlink(tmp);
|
|
114
|
+
throw err;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// ---------------------------------------------------------------------------
|
|
119
|
+
// Decrypt
|
|
120
|
+
// ---------------------------------------------------------------------------
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Stream-decrypt `inputPath` to `outputPath`. Expects the on-disk format
|
|
124
|
+
* produced by `encryptFile`: `[IV][ciphertext][auth tag]`.
|
|
125
|
+
*
|
|
126
|
+
* Reads the IV and auth tag via positional reads, then streams only the
|
|
127
|
+
* ciphertext body through the decipher. Atomic tmp + rename semantics.
|
|
128
|
+
*/
|
|
129
|
+
export async function decryptFile(
|
|
130
|
+
inputPath: string,
|
|
131
|
+
outputPath: string,
|
|
132
|
+
key: Buffer,
|
|
133
|
+
): Promise<void> {
|
|
134
|
+
assertKey(key);
|
|
135
|
+
|
|
136
|
+
const info = await stat(inputPath);
|
|
137
|
+
const totalSize = info.size;
|
|
138
|
+
const minSize = ENCRYPTED_HEADER_SIZE + GCM_TAG_SIZE;
|
|
139
|
+
if (totalSize < minSize) {
|
|
140
|
+
throw new Error(
|
|
141
|
+
`Encrypted file is too small: ${totalSize} bytes (need at least ${minSize})`,
|
|
142
|
+
);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// Read IV (first 12 bytes) and auth tag (last 16 bytes) via positional reads.
|
|
146
|
+
const iv = Buffer.alloc(ENCRYPTED_HEADER_SIZE);
|
|
147
|
+
const tag = Buffer.alloc(GCM_TAG_SIZE);
|
|
148
|
+
const fh = await open(inputPath, "r");
|
|
149
|
+
try {
|
|
150
|
+
await fh.read(iv, 0, ENCRYPTED_HEADER_SIZE, 0);
|
|
151
|
+
await fh.read(tag, 0, GCM_TAG_SIZE, totalSize - GCM_TAG_SIZE);
|
|
152
|
+
} finally {
|
|
153
|
+
await fh.close();
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
const decipher = createDecipheriv(ALGORITHM, key, iv);
|
|
157
|
+
decipher.setAuthTag(tag);
|
|
158
|
+
|
|
159
|
+
const ciphertextStart = ENCRYPTED_HEADER_SIZE;
|
|
160
|
+
const ciphertextEnd = totalSize - GCM_TAG_SIZE - 1; // createReadStream end is inclusive
|
|
161
|
+
const hasCiphertext = ciphertextEnd >= ciphertextStart;
|
|
162
|
+
|
|
163
|
+
const tmp = tempPath(outputPath);
|
|
164
|
+
const writeStream = createWriteStream(tmp);
|
|
165
|
+
|
|
166
|
+
try {
|
|
167
|
+
const ciphertextStream = hasCiphertext
|
|
168
|
+
? createReadStream(inputPath, {
|
|
169
|
+
start: ciphertextStart,
|
|
170
|
+
end: ciphertextEnd,
|
|
171
|
+
})
|
|
172
|
+
: Readable.from([]);
|
|
173
|
+
|
|
174
|
+
// pipeline consumes the ciphertext, pushes it through the decipher, and
|
|
175
|
+
// calls decipher.final() at the end — which is where auth tag verification
|
|
176
|
+
// happens. A bad tag surfaces here as a thrown error.
|
|
177
|
+
await pipeline(ciphertextStream, decipher, writeStream);
|
|
178
|
+
|
|
179
|
+
await rename(tmp, outputPath);
|
|
180
|
+
} catch (err) {
|
|
181
|
+
writeStream.destroy();
|
|
182
|
+
await safeUnlink(tmp);
|
|
183
|
+
throw err;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
// ---------------------------------------------------------------------------
|
|
188
|
+
// Verify
|
|
189
|
+
// ---------------------------------------------------------------------------
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* Verify that `path` is a valid AES-256-GCM encrypted bundle for `key`.
|
|
193
|
+
*
|
|
194
|
+
* Streams the ciphertext through the decipher into a null sink and relies on
|
|
195
|
+
* `decipher.final()` to either succeed (tag matches) or throw (tamper / wrong
|
|
196
|
+
* key). No scratch file is written, so a full or read-only tmpdir cannot
|
|
197
|
+
* cause a healthy backup to be reported as invalid.
|
|
198
|
+
*
|
|
199
|
+
* Returns `true` if the bundle authenticates, `false` on a cryptographic
|
|
200
|
+
* failure (bad auth tag, wrong key, truncated/short input). Filesystem errors
|
|
201
|
+
* on the *source* file (ENOENT, EACCES, EIO, …) are rethrown so callers can
|
|
202
|
+
* distinguish tamper from transient I/O.
|
|
203
|
+
*/
|
|
204
|
+
export async function verifyEncryptedFile(
|
|
205
|
+
path: string,
|
|
206
|
+
key: Buffer,
|
|
207
|
+
): Promise<boolean> {
|
|
208
|
+
assertKey(key);
|
|
209
|
+
|
|
210
|
+
const info = await stat(path);
|
|
211
|
+
const totalSize = info.size;
|
|
212
|
+
const minSize = ENCRYPTED_HEADER_SIZE + GCM_TAG_SIZE;
|
|
213
|
+
if (totalSize < minSize) {
|
|
214
|
+
// Too short to contain an IV + tag — not a valid bundle.
|
|
215
|
+
return false;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
const iv = Buffer.alloc(ENCRYPTED_HEADER_SIZE);
|
|
219
|
+
const tag = Buffer.alloc(GCM_TAG_SIZE);
|
|
220
|
+
const fh = await open(path, "r");
|
|
221
|
+
try {
|
|
222
|
+
await fh.read(iv, 0, ENCRYPTED_HEADER_SIZE, 0);
|
|
223
|
+
await fh.read(tag, 0, GCM_TAG_SIZE, totalSize - GCM_TAG_SIZE);
|
|
224
|
+
} finally {
|
|
225
|
+
await fh.close();
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
const decipher = createDecipheriv(ALGORITHM, key, iv);
|
|
229
|
+
decipher.setAuthTag(tag);
|
|
230
|
+
|
|
231
|
+
const ciphertextStart = ENCRYPTED_HEADER_SIZE;
|
|
232
|
+
const ciphertextEnd = totalSize - GCM_TAG_SIZE - 1;
|
|
233
|
+
const hasCiphertext = ciphertextEnd >= ciphertextStart;
|
|
234
|
+
const ciphertextStream = hasCiphertext
|
|
235
|
+
? createReadStream(path, { start: ciphertextStart, end: ciphertextEnd })
|
|
236
|
+
: Readable.from([]);
|
|
237
|
+
|
|
238
|
+
// Discard-only sink — verification never touches scratch disk.
|
|
239
|
+
const nullSink = new Writable({
|
|
240
|
+
write(_chunk, _encoding, cb) {
|
|
241
|
+
cb();
|
|
242
|
+
},
|
|
243
|
+
});
|
|
244
|
+
|
|
245
|
+
try {
|
|
246
|
+
await pipeline(ciphertextStream, decipher, nullSink);
|
|
247
|
+
return true;
|
|
248
|
+
} catch (err) {
|
|
249
|
+
if (isFilesystemError(err)) {
|
|
250
|
+
throw err;
|
|
251
|
+
}
|
|
252
|
+
return false;
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
// Node errno exceptions surface as uppercase `E`-prefixed codes (ENOENT,
|
|
257
|
+
// EACCES, ENOSPC, EIO, EROFS, …). Crypto errors use `ERR_*` codes or no code
|
|
258
|
+
// at all, so the regex rules them out.
|
|
259
|
+
function isFilesystemError(err: unknown): boolean {
|
|
260
|
+
if (!err || typeof err !== "object") return false;
|
|
261
|
+
const code = (err as { code?: unknown }).code;
|
|
262
|
+
return typeof code === "string" && /^E[A-Z]+$/.test(code);
|
|
263
|
+
}
|