@phuetz/code-buddy 1.6.1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +90 -21
- package/README.md +145 -349
- package/codebuddy-runtime.json +24 -0
- package/dist/advanced/session-replay.js +3 -2
- package/dist/agent/agent-state.d.ts +4 -1
- package/dist/agent/agent-state.js +9 -3
- package/dist/agent/architect-mode.js +1 -1
- package/dist/agent/autonomous/agentic-coding-contract.d.ts +28 -28
- package/dist/agent/autonomous/agentic-coding-runner.d.ts +14 -4
- package/dist/agent/autonomous/agentic-coding-runner.js +176 -27
- package/dist/agent/autonomous/checkpoint-manager.js +3 -12
- package/dist/agent/autonomous/codex-autonomy-directive.d.ts +10 -0
- package/dist/agent/autonomous/codex-autonomy-directive.js +25 -0
- package/dist/agent/autonomous/edit-proposal-producer.js +2 -2
- package/dist/agent/autonomous/fleet-task-types.d.ts +2 -2
- package/dist/agent/autonomous/fleet-task-types.js +2 -2
- package/dist/agent/autonomous/fleet-tick-handler.d.ts +3 -3
- package/dist/agent/autonomous/fleet-tick-handler.js +14 -8
- package/dist/agent/autonomous/recursive-improvement.d.ts +23 -0
- package/dist/agent/autonomous/recursive-improvement.js +114 -0
- package/dist/agent/autonomous/verification-loop.js +4 -2
- package/dist/agent/codebuddy-agent.d.ts +93 -6
- package/dist/agent/codebuddy-agent.js +452 -58
- package/dist/agent/council-lesson-proposer.d.ts +80 -13
- package/dist/agent/council-lesson-proposer.js +201 -43
- package/dist/agent/deep-research-ckg.d.ts +176 -0
- package/dist/agent/deep-research-ckg.js +250 -0
- package/dist/agent/deep-research-storm.d.ts +222 -0
- package/dist/agent/deep-research-storm.js +689 -0
- package/dist/agent/deep-research.d.ts +329 -0
- package/dist/agent/deep-research.js +994 -0
- package/dist/agent/delegation/thread-delegation.d.ts +80 -0
- package/dist/agent/delegation/thread-delegation.js +472 -0
- package/dist/agent/delegation/thread-task-runner.d.ts +37 -0
- package/dist/agent/delegation/thread-task-runner.js +89 -0
- package/dist/agent/dev-loop/dev-loop.d.ts +120 -0
- package/dist/agent/dev-loop/dev-loop.js +404 -0
- package/dist/agent/dev-loop/structural-gate.d.ts +32 -0
- package/dist/agent/dev-loop/structural-gate.js +120 -0
- package/dist/agent/execution/agent-executor.d.ts +52 -31
- package/dist/agent/execution/agent-executor.js +1520 -547
- package/dist/agent/execution/context-pipeline.d.ts +54 -6
- package/dist/agent/execution/context-pipeline.js +285 -88
- package/dist/agent/execution/incremental-token-counter.d.ts +15 -0
- package/dist/agent/execution/incremental-token-counter.js +36 -0
- package/dist/agent/execution/ordered-tool-executor.d.ts +14 -0
- package/dist/agent/execution/ordered-tool-executor.js +54 -0
- package/dist/agent/execution/post-tool-handlers.d.ts +1 -1
- package/dist/agent/execution/post-tool-handlers.js +2 -2
- package/dist/agent/execution/query-classifier.d.ts +2 -0
- package/dist/agent/execution/query-classifier.js +2 -0
- package/dist/agent/execution/tool-dependency-graph.js +7 -0
- package/dist/agent/execution/tool-selection-strategy.d.ts +20 -0
- package/dist/agent/execution/tool-selection-strategy.js +145 -23
- package/dist/agent/facades/message-history-manager.d.ts +1 -1
- package/dist/agent/facades/message-history-manager.js +1 -1
- package/dist/agent/facades/model-routing-facade.d.ts +31 -1
- package/dist/agent/facades/model-routing-facade.js +86 -2
- package/dist/agent/facades/session-facade.d.ts +10 -2
- package/dist/agent/facades/session-facade.js +59 -1
- package/dist/agent/film/film-producer.d.ts +98 -0
- package/dist/agent/film/film-producer.js +289 -0
- package/dist/agent/film/scene-planner.d.ts +38 -0
- package/dist/agent/film/scene-planner.js +93 -0
- package/dist/agent/film/trailer-planner.d.ts +23 -0
- package/dist/agent/film/trailer-planner.js +242 -0
- package/dist/agent/film/video-studio.d.ts +72 -0
- package/dist/agent/film/video-studio.js +249 -0
- package/dist/agent/flow/planning-flow.d.ts +3 -0
- package/dist/agent/flow/planning-flow.js +70 -8
- package/dist/agent/hermes-claw-migrate.d.ts +62 -0
- package/dist/agent/hermes-claw-migrate.js +189 -15
- package/dist/agent/hermes-memory-providers.js +3 -3
- package/dist/agent/hermes-parity-manifest.js +69 -15
- package/dist/agent/infrastructure/agent-infrastructure.d.ts +1 -0
- package/dist/agent/infrastructure/agent-infrastructure.js +1 -1
- package/dist/agent/isolation/agent-workspace.js +5 -2
- package/dist/agent/learning/skill-background-writes.js +9 -4
- package/dist/agent/learning-agent.js +4 -4
- package/dist/agent/learning-background-writes.js +9 -5
- package/dist/agent/lesson-candidate-queue.d.ts +4 -0
- package/dist/agent/lesson-candidate-queue.js +9 -3
- package/dist/agent/lesson-provenance.js +9 -3
- package/dist/agent/lessons-tracker.d.ts +68 -1
- package/dist/agent/lessons-tracker.js +218 -36
- package/dist/agent/loop-detection-service.d.ts +56 -0
- package/dist/agent/loop-detection-service.js +207 -0
- package/dist/agent/middleware/auto-observation.d.ts +2 -0
- package/dist/agent/middleware/auto-observation.js +4 -2
- package/dist/agent/middleware/auto-repair-middleware.d.ts +2 -0
- package/dist/agent/middleware/auto-repair-middleware.js +27 -4
- package/dist/agent/middleware/changed-files.d.ts +19 -0
- package/dist/agent/middleware/changed-files.js +113 -0
- package/dist/agent/middleware/cost-limit.d.ts +0 -2
- package/dist/agent/middleware/cost-limit.js +0 -3
- package/dist/agent/middleware/index.d.ts +2 -0
- package/dist/agent/middleware/index.js +1 -0
- package/dist/agent/middleware/pipeline.d.ts +9 -0
- package/dist/agent/middleware/pipeline.js +18 -0
- package/dist/agent/middleware/plan-completion-audit.d.ts +76 -0
- package/dist/agent/middleware/plan-completion-audit.js +157 -0
- package/dist/agent/middleware/quality-gate-middleware.d.ts +43 -1
- package/dist/agent/middleware/quality-gate-middleware.js +198 -40
- package/dist/agent/middleware/types.d.ts +6 -0
- package/dist/agent/middleware/verification-enforcement.d.ts +10 -3
- package/dist/agent/middleware/verification-enforcement.js +43 -13
- package/dist/agent/middleware/workflow-guard.js +7 -0
- package/dist/agent/model-benchmark.js +11 -17
- package/dist/agent/model-tier.d.ts +2 -2
- package/dist/agent/model-tier.js +10 -7
- package/dist/agent/multi-agent/agent-memory-integration.js +3 -2
- package/dist/agent/multi-agent/base-agent.d.ts +12 -1
- package/dist/agent/multi-agent/base-agent.js +58 -10
- package/dist/agent/multi-agent/enhanced-coordination.js +1 -0
- package/dist/agent/multi-agent/metrics-persistence.js +28 -14
- package/dist/agent/multi-agent/multi-agent-system.d.ts +5 -0
- package/dist/agent/multi-agent/multi-agent-system.js +81 -4
- package/dist/agent/multi-agent/session-fleet-bridge.js +2 -1
- package/dist/agent/multi-agent/session-registry.js +9 -3
- package/dist/agent/multi-agent/team-manager.d.ts +11 -0
- package/dist/agent/multi-agent/team-manager.js +94 -5
- package/dist/agent/multi-agent/types.d.ts +10 -1
- package/dist/agent/multi-agent/workflow-multi-persistence.js +16 -8
- package/dist/agent/multi-agent/workflow-persistence.js +9 -6
- package/dist/agent/observer/trigger-registry.js +3 -5
- package/dist/agent/operating-modes.d.ts +3 -0
- package/dist/agent/operating-modes.js +15 -22
- package/dist/agent/pipelines.d.ts +2 -4
- package/dist/agent/pipelines.js +1 -1
- package/dist/agent/plan-mode.js +1 -1
- package/dist/agent/prompt-suggestions.js +2 -1
- package/dist/agent/prompt-tool-observation.d.ts +55 -0
- package/dist/agent/prompt-tool-observation.js +165 -0
- package/dist/agent/reasoning/tree-of-thought.js +1 -1
- package/dist/agent/repair/repair-engine.js +1 -1
- package/dist/agent/repo-profiler.d.ts +12 -1
- package/dist/agent/repo-profiler.js +21 -15
- package/dist/agent/repo-profiling/cache.js +6 -9
- package/dist/agent/research-worker-provider.d.ts +38 -0
- package/dist/agent/research-worker-provider.js +30 -0
- package/dist/agent/science/experiment-decision.d.ts +48 -0
- package/dist/agent/science/experiment-decision.js +65 -0
- package/dist/agent/science/experiment-empirical-gate.d.ts +83 -0
- package/dist/agent/science/experiment-empirical-gate.js +156 -0
- package/dist/agent/science/experiment-fitness.d.ts +112 -0
- package/dist/agent/science/experiment-fitness.js +188 -0
- package/dist/agent/science/experiment-loop.d.ts +196 -0
- package/dist/agent/science/experiment-loop.js +765 -0
- package/dist/agent/science/experiment-orchestrator.d.ts +203 -0
- package/dist/agent/science/experiment-orchestrator.js +454 -0
- package/dist/agent/science/experiment-sandbox-backends.d.ts +56 -0
- package/dist/agent/science/experiment-sandbox-backends.js +201 -0
- package/dist/agent/science/experiment-sandbox.d.ts +105 -0
- package/dist/agent/science/experiment-sandbox.js +196 -0
- package/dist/agent/science/experiment-variant-store.d.ts +108 -0
- package/dist/agent/science/experiment-variant-store.js +156 -0
- package/dist/agent/science/human-gate.d.ts +31 -0
- package/dist/agent/science/human-gate.js +32 -0
- package/dist/agent/self-improvement/authored-artifact-gate.js +28 -3
- package/dist/agent/self-improvement/authored-tool-runtime.d.ts +4 -2
- package/dist/agent/self-improvement/authored-tool-runtime.js +8 -3
- package/dist/agent/self-improvement/authored-tool-store.js +3 -2
- package/dist/agent/self-improvement/capability-benchmark.js +89 -2
- package/dist/agent/self-improvement/continuous-benchmark.d.ts +105 -0
- package/dist/agent/self-improvement/continuous-benchmark.js +355 -0
- package/dist/agent/self-improvement/delegation-facts.d.ts +105 -0
- package/dist/agent/self-improvement/delegation-facts.js +483 -0
- package/dist/agent/self-improvement/digest-sources.d.ts +56 -0
- package/dist/agent/self-improvement/digest-sources.js +294 -0
- package/dist/agent/self-improvement/digest.d.ts +169 -0
- package/dist/agent/self-improvement/digest.js +543 -0
- package/dist/agent/self-improvement/engine.d.ts +11 -6
- package/dist/agent/self-improvement/engine.js +13 -7
- package/dist/agent/self-improvement/evolution/ast-novelty.d.ts +20 -0
- package/dist/agent/self-improvement/evolution/ast-novelty.js +81 -0
- package/dist/agent/self-improvement/evolution/code-variant-store.d.ts +101 -0
- package/dist/agent/self-improvement/evolution/code-variant-store.js +229 -0
- package/dist/agent/self-improvement/evolution/evolution-engine.d.ts +199 -0
- package/dist/agent/self-improvement/evolution/evolution-engine.js +503 -0
- package/dist/agent/self-improvement/evolution/feature-map.d.ts +24 -0
- package/dist/agent/self-improvement/evolution/feature-map.js +88 -0
- package/dist/agent/self-improvement/evolution/model-bandit.d.ts +57 -0
- package/dist/agent/self-improvement/evolution/model-bandit.js +98 -0
- package/dist/agent/self-improvement/evolution/protected-paths.d.ts +31 -0
- package/dist/agent/self-improvement/evolution/protected-paths.js +105 -0
- package/dist/agent/self-improvement/evolution/research-weakness-source.d.ts +56 -0
- package/dist/agent/self-improvement/evolution/research-weakness-source.js +161 -0
- package/dist/agent/self-improvement/evolution/scrub-env.d.ts +22 -0
- package/dist/agent/self-improvement/evolution/scrub-env.js +40 -0
- package/dist/agent/self-improvement/evolution/variant-fitness.d.ts +86 -0
- package/dist/agent/self-improvement/evolution/variant-fitness.js +217 -0
- package/dist/agent/self-improvement/evolution/variant-planner.d.ts +41 -0
- package/dist/agent/self-improvement/evolution/variant-planner.js +156 -0
- package/dist/agent/self-improvement/evolution/weakness-selector.d.ts +38 -0
- package/dist/agent/self-improvement/evolution/weakness-selector.js +108 -0
- package/dist/agent/self-improvement/evolution/worktree-scorer.d.ts +44 -0
- package/dist/agent/self-improvement/evolution/worktree-scorer.js +99 -0
- package/dist/agent/self-improvement/evolutionary-archive.js +3 -2
- package/dist/agent/self-improvement/experience-source.d.ts +60 -7
- package/dist/agent/self-improvement/experience-source.js +78 -8
- package/dist/agent/self-improvement/index.d.ts +1 -0
- package/dist/agent/self-improvement/index.js +6 -3
- package/dist/agent/self-improvement/learning-store.d.ts +16 -0
- package/dist/agent/self-improvement/learning-store.js +83 -3
- package/dist/agent/self-improvement/llm-drafter.d.ts +3 -1
- package/dist/agent/self-improvement/llm-drafter.js +20 -2
- package/dist/agent/self-improvement/llm-tool-proposer.js +1 -1
- package/dist/agent/self-improvement/proposal-store.d.ts +46 -0
- package/dist/agent/self-improvement/proposal-store.js +87 -0
- package/dist/agent/self-improvement/proposer.d.ts +1 -1
- package/dist/agent/self-improvement/proposer.js +94 -2
- package/dist/agent/self-improvement/rule-store.js +5 -4
- package/dist/agent/self-improvement/sandbox-scorer.d.ts +1 -1
- package/dist/agent/self-improvement/sandbox-scorer.js +8 -4
- package/dist/agent/self-improvement/self-knowledge.d.ts +1 -1
- package/dist/agent/self-improvement/self-knowledge.js +8 -8
- package/dist/agent/self-improvement/skill-benchmark.js +27 -0
- package/dist/agent/self-improvement/skill-consolidator.js +16 -0
- package/dist/agent/self-improvement/skill-engine.d.ts +7 -0
- package/dist/agent/self-improvement/skill-engine.js +54 -28
- package/dist/agent/self-improvement/skill-gate.d.ts +4 -4
- package/dist/agent/self-improvement/skill-gate.js +38 -20
- package/dist/agent/self-improvement/skill-mutator.d.ts +20 -1
- package/dist/agent/self-improvement/skill-mutator.js +138 -19
- package/dist/agent/self-improvement/skill-proposer.js +2 -1
- package/dist/agent/self-improvement/skill-types.d.ts +4 -0
- package/dist/agent/self-improvement/strategy-engine.d.ts +54 -0
- package/dist/agent/self-improvement/strategy-engine.js +104 -0
- package/dist/agent/self-improvement/strategy-gate.d.ts +49 -0
- package/dist/agent/self-improvement/strategy-gate.js +243 -0
- package/dist/agent/self-improvement/strategy-live.d.ts +39 -0
- package/dist/agent/self-improvement/strategy-live.js +114 -0
- package/dist/agent/self-improvement/strategy-proposer.d.ts +30 -0
- package/dist/agent/self-improvement/strategy-proposer.js +114 -0
- package/dist/agent/self-improvement/strategy-replay.d.ts +40 -0
- package/dist/agent/self-improvement/strategy-replay.js +150 -0
- package/dist/agent/self-improvement/strategy-runtime.d.ts +43 -0
- package/dist/agent/self-improvement/strategy-runtime.js +56 -0
- package/dist/agent/self-improvement/strategy-store.d.ts +38 -0
- package/dist/agent/self-improvement/strategy-store.js +139 -0
- package/dist/agent/self-improvement/strategy-types.d.ts +229 -0
- package/dist/agent/self-improvement/strategy-types.js +90 -0
- package/dist/agent/self-improvement/tool-benchmark.js +164 -8
- package/dist/agent/self-improvement/tool-engine.d.ts +12 -3
- package/dist/agent/self-improvement/tool-engine.js +59 -31
- package/dist/agent/self-improvement/tool-gate.d.ts +2 -0
- package/dist/agent/self-improvement/tool-gate.js +26 -0
- package/dist/agent/self-improvement/tool-skill-mutator.d.ts +1 -1
- package/dist/agent/self-improvement/tool-skill-mutator.js +20 -2
- package/dist/agent/self-improvement/tool-types.d.ts +5 -3
- package/dist/agent/self-improvement/types.d.ts +7 -3
- package/dist/agent/session-end-flush.js +2 -1
- package/dist/agent/specialized/agent-registry.d.ts +1 -0
- package/dist/agent/specialized/agent-registry.js +118 -0
- package/dist/agent/specialized/excel-agent.js +15 -1
- package/dist/agent/specialized/index.d.ts +1 -0
- package/dist/agent/specialized/index.js +2 -0
- package/dist/agent/specialized/pdf-agent.js +4 -2
- package/dist/agent/specialized/swe-agent-adapter.js +8 -0
- package/dist/agent/specialized/swe-agent.d.ts +16 -1
- package/dist/agent/specialized/swe-agent.js +75 -7
- package/dist/agent/specialized/types.d.ts +31 -1
- package/dist/agent/specialized/types.js +26 -0
- package/dist/agent/specialized/verifier-agent.d.ts +53 -0
- package/dist/agent/specialized/verifier-agent.js +342 -0
- package/dist/agent/streaming/message-reducer.js +73 -19
- package/dist/agent/streaming/streaming-handler.d.ts +24 -0
- package/dist/agent/streaming/streaming-handler.js +136 -1
- package/dist/agent/subagents.d.ts +10 -4
- package/dist/agent/subagents.js +39 -6
- package/dist/agent/thinking/extended-thinking.js +1 -1
- package/dist/agent/todo-tracker.js +5 -4
- package/dist/agent/tool-executor.d.ts +10 -1
- package/dist/agent/tool-executor.js +29 -0
- package/dist/agent/tool-handler.d.ts +49 -2
- package/dist/agent/tool-handler.js +738 -181
- package/dist/agent/types.d.ts +3 -5
- package/dist/agent/wide-research-checkpoint.d.ts +77 -0
- package/dist/agent/wide-research-checkpoint.js +361 -0
- package/dist/agent/wide-research-files.d.ts +11 -0
- package/dist/agent/wide-research-files.js +189 -0
- package/dist/agent/wide-research.d.ts +148 -3
- package/dist/agent/wide-research.js +833 -56
- package/dist/analytics/code-evolution.js +20 -15
- package/dist/analytics/complexity-analyzer.js +2 -2
- package/dist/analytics/cost-report.d.ts +71 -0
- package/dist/analytics/cost-report.js +724 -0
- package/dist/analytics/dashboard.js +8 -7
- package/dist/analytics/index.d.ts +2 -0
- package/dist/analytics/index.js +2 -0
- package/dist/analytics/repo-explainer-collector.d.ts +13 -0
- package/dist/analytics/repo-explainer-collector.js +351 -0
- package/dist/analytics/repo-explainer.d.ts +224 -0
- package/dist/analytics/repo-explainer.js +605 -0
- package/dist/analytics/roi-tracker.js +11 -9
- package/dist/api/webhooks.js +9 -14
- package/dist/app/application-factory.d.ts +1 -1
- package/dist/app/application-factory.js +6 -5
- package/dist/auth/profile-manager.js +8 -4
- package/dist/avatar/avatar-event-bus.d.ts +16 -0
- package/dist/avatar/avatar-event-bus.js +53 -0
- package/dist/avatar/avatar-gateway-bridge.d.ts +26 -0
- package/dist/avatar/avatar-gateway-bridge.js +54 -0
- package/dist/avatar/avatar-protocol.d.ts +133 -0
- package/dist/avatar/avatar-protocol.js +241 -0
- package/dist/avatar/avatar-renderer-registry.d.ts +56 -0
- package/dist/avatar/avatar-renderer-registry.js +191 -0
- package/dist/avatar/avatar-renderer-simulator.d.ts +31 -0
- package/dist/avatar/avatar-renderer-simulator.js +160 -0
- package/dist/browser-automation/browser-manager.d.ts +15 -1
- package/dist/browser-automation/browser-manager.js +101 -0
- package/dist/browser-automation/browser-operator-executor.d.ts +89 -7
- package/dist/browser-automation/browser-operator-executor.js +1280 -58
- package/dist/browser-automation/browser-operator-runtime.d.ts +87 -0
- package/dist/browser-automation/browser-operator-runtime.js +441 -0
- package/dist/browser-automation/browser-operator-session.js +4 -3
- package/dist/browser-automation/browser-tool.d.ts +17 -2
- package/dist/browser-automation/browser-tool.js +101 -39
- package/dist/browser-automation/internet-scout-plan.d.ts +1 -0
- package/dist/browser-automation/internet-scout-plan.js +17 -0
- package/dist/browser-automation/profile-manager.js +5 -3
- package/dist/browser-automation/types.d.ts +25 -0
- package/dist/cache/advanced-lru-cache.js +4 -4
- package/dist/cache/cache-manager.js +9 -10
- package/dist/cache/embedding-cache.js +6 -5
- package/dist/cache/llm-response-cache.js +3 -4
- package/dist/capture/screen-recorder.d.ts +2 -0
- package/dist/capture/screen-recorder.js +6 -4
- package/dist/capture/screen-watcher.d.ts +6 -3
- package/dist/capture/screen-watcher.js +86 -9
- package/dist/channels/channel-cognitive-port.d.ts +46 -0
- package/dist/channels/channel-cognitive-port.js +182 -0
- package/dist/channels/channel-model-override.d.ts +47 -0
- package/dist/channels/channel-model-override.js +62 -0
- package/dist/channels/companion-channel-profile.d.ts +68 -0
- package/dist/channels/companion-channel-profile.js +142 -0
- package/dist/channels/companion-channel-turn.d.ts +30 -0
- package/dist/channels/companion-channel-turn.js +38 -0
- package/dist/channels/core.d.ts +37 -3
- package/dist/channels/core.js +63 -6
- package/dist/channels/discord/client.js +34 -14
- package/dist/channels/dm-pairing.d.ts +18 -4
- package/dist/channels/dm-pairing.js +199 -33
- package/dist/channels/gateway-lifecycle.d.ts +2 -0
- package/dist/channels/gateway-lifecycle.js +1 -0
- package/dist/channels/identity-links.d.ts +19 -1
- package/dist/channels/identity-links.js +68 -12
- package/dist/channels/matrix/index.js +6 -2
- package/dist/channels/pro/pro-features.js +17 -0
- package/dist/channels/pro/run-tracker.js +9 -4
- package/dist/channels/pro/scoped-auth.js +10 -8
- package/dist/channels/provider-failure-speech.d.ts +28 -0
- package/dist/channels/provider-failure-speech.js +128 -0
- package/dist/channels/reconnection-manager.d.ts +2 -0
- package/dist/channels/reconnection-manager.js +7 -2
- package/dist/channels/resolve-channel-secret.d.ts +43 -0
- package/dist/channels/resolve-channel-secret.js +82 -0
- package/dist/channels/slack/block-builder.d.ts +12 -0
- package/dist/channels/slack/block-builder.js +70 -2
- package/dist/channels/slack/client.js +62 -15
- package/dist/channels/slack/types.d.ts +20 -1
- package/dist/channels/telegram/client.d.ts +36 -3
- package/dist/channels/telegram/client.js +438 -84
- package/dist/channels/telegram/types.d.ts +2 -0
- package/dist/channels/webhook-server.js +8 -1
- package/dist/channels/whatsapp/index.js +2 -1
- package/dist/checkpoints/ghost-snapshot.js +15 -9
- package/dist/checkpoints/persistent-checkpoint-manager.js +15 -8
- package/dist/cli/command-routing.d.ts +22 -0
- package/dist/cli/command-routing.js +54 -0
- package/dist/cli/first-run.d.ts +17 -0
- package/dist/cli/first-run.js +40 -0
- package/dist/cli/headless-options.d.ts +22 -0
- package/dist/cli/headless-options.js +112 -0
- package/dist/cli/headless-prompt-progress.d.ts +22 -0
- package/dist/cli/headless-prompt-progress.js +35 -0
- package/dist/cli/headless.js +2 -2
- package/dist/cli/listen-port.d.ts +8 -0
- package/dist/cli/listen-port.js +20 -0
- package/dist/cli/permission-mode-option.d.ts +14 -0
- package/dist/cli/permission-mode-option.js +64 -0
- package/dist/cli/requested-profile.d.ts +15 -0
- package/dist/cli/requested-profile.js +27 -0
- package/dist/cli/unknown-option-hint.d.ts +16 -0
- package/dist/cli/unknown-option-hint.js +116 -0
- package/dist/cli/voice-command.d.ts +32 -0
- package/dist/cli/voice-command.js +96 -0
- package/dist/codebuddy/abort-signal.d.ts +7 -0
- package/dist/codebuddy/abort-signal.js +31 -0
- package/dist/codebuddy/client.d.ts +68 -0
- package/dist/codebuddy/client.js +460 -11
- package/dist/codebuddy/fleet-tool-defs.js +5 -0
- package/dist/codebuddy/llm-retry.d.ts +24 -0
- package/dist/codebuddy/llm-retry.js +124 -0
- package/dist/codebuddy/provider-error-classifier.d.ts +86 -0
- package/dist/codebuddy/provider-error-classifier.js +410 -0
- package/dist/codebuddy/provider-failover-error.d.ts +16 -0
- package/dist/codebuddy/provider-failover-error.js +50 -0
- package/dist/codebuddy/provider-failover-kind.d.ts +15 -0
- package/dist/codebuddy/provider-failover-kind.js +172 -0
- package/dist/codebuddy/provider-handoff.d.ts +35 -0
- package/dist/codebuddy/provider-handoff.js +294 -0
- package/dist/codebuddy/providers/ollama-native-transport.d.ts +130 -0
- package/dist/codebuddy/providers/ollama-native-transport.js +387 -0
- package/dist/codebuddy/providers/provider-agy-cli.d.ts +29 -0
- package/dist/codebuddy/providers/provider-agy-cli.js +264 -0
- package/dist/codebuddy/providers/provider-chatgpt-responses.d.ts +59 -7
- package/dist/codebuddy/providers/provider-chatgpt-responses.js +519 -164
- package/dist/codebuddy/providers/provider-gemini-cli.js +95 -13
- package/dist/codebuddy/providers/provider-gemini-native.d.ts +2 -0
- package/dist/codebuddy/providers/provider-gemini-native.js +199 -86
- package/dist/codebuddy/providers/provider-openai-compat.d.ts +82 -0
- package/dist/codebuddy/providers/provider-openai-compat.js +469 -45
- package/dist/codebuddy/stream-retry.d.ts +21 -8
- package/dist/codebuddy/stream-retry.js +44 -40
- package/dist/codebuddy/tool-definitions/advanced-tools.d.ts +4 -0
- package/dist/codebuddy/tool-definitions/advanced-tools.js +188 -0
- package/dist/codebuddy/tool-definitions/agent-tools.d.ts +5 -0
- package/dist/codebuddy/tool-definitions/agent-tools.js +195 -7
- package/dist/codebuddy/tool-definitions/browser-tools.d.ts +1 -0
- package/dist/codebuddy/tool-definitions/browser-tools.js +65 -1
- package/dist/codebuddy/tool-definitions/code-exec-tools.d.ts +8 -0
- package/dist/codebuddy/tool-definitions/code-exec-tools.js +29 -0
- package/dist/codebuddy/tool-definitions/code-explorer-tools.js +4 -0
- package/dist/codebuddy/tool-definitions/comfy-recipe-tools.d.ts +3 -0
- package/dist/codebuddy/tool-definitions/comfy-recipe-tools.js +37 -0
- package/dist/codebuddy/tool-definitions/core-tools.d.ts +2 -0
- package/dist/codebuddy/tool-definitions/core-tools.js +58 -2
- package/dist/codebuddy/tool-definitions/delegate-agent-tools.d.ts +11 -0
- package/dist/codebuddy/tool-definitions/delegate-agent-tools.js +50 -0
- package/dist/codebuddy/tool-definitions/design-tools.d.ts +6 -0
- package/dist/codebuddy/tool-definitions/design-tools.js +35 -0
- package/dist/codebuddy/tool-definitions/exit-plan-mode-tools.d.ts +1 -0
- package/dist/codebuddy/tool-definitions/exit-plan-mode-tools.js +23 -1
- package/dist/codebuddy/tool-definitions/index.d.ts +16 -7
- package/dist/codebuddy/tool-definitions/index.js +27 -8
- package/dist/codebuddy/tool-definitions/lsp-tools.d.ts +7 -1
- package/dist/codebuddy/tool-definitions/lsp-tools.js +104 -1
- package/dist/codebuddy/tool-definitions/meeting-tools.d.ts +4 -0
- package/dist/codebuddy/tool-definitions/meeting-tools.js +32 -0
- package/dist/codebuddy/tool-definitions/moa-tools.js +7 -2
- package/dist/codebuddy/tool-definitions/multimodal-tools.d.ts +14 -0
- package/dist/codebuddy/tool-definitions/multimodal-tools.js +615 -3
- package/dist/codebuddy/tool-definitions/research-tools.d.ts +20 -0
- package/dist/codebuddy/tool-definitions/research-tools.js +98 -0
- package/dist/codebuddy/tool-definitions/self-describe-tools.d.ts +4 -0
- package/dist/codebuddy/tool-definitions/self-describe-tools.js +28 -0
- package/dist/codebuddy/tool-definitions/self-evolution-tools.d.ts +4 -0
- package/dist/codebuddy/tool-definitions/self-evolution-tools.js +31 -0
- package/dist/codebuddy/tool-definitions/verify-tools.d.ts +14 -0
- package/dist/codebuddy/tool-definitions/verify-tools.js +36 -0
- package/dist/codebuddy/tool-definitions/web-tools.d.ts +4 -0
- package/dist/codebuddy/tool-definitions/web-tools.js +135 -0
- package/dist/codebuddy/tool-definitions/workspace-tools.d.ts +4 -0
- package/dist/codebuddy/tool-definitions/workspace-tools.js +43 -0
- package/dist/codebuddy/tools.d.ts +15 -17
- package/dist/codebuddy/tools.js +235 -55
- package/dist/cognition/budget-reservations.d.ts +27 -0
- package/dist/cognition/budget-reservations.js +70 -0
- package/dist/cognition/cognitive-bus-client.d.ts +94 -0
- package/dist/cognition/cognitive-bus-client.js +724 -0
- package/dist/cognition/cognitive-hub.d.ts +93 -0
- package/dist/cognition/cognitive-hub.js +366 -0
- package/dist/cognition/cognitive-mesh.d.ts +38 -0
- package/dist/cognition/cognitive-mesh.js +263 -0
- package/dist/cognition/cognitive-port.d.ts +35 -0
- package/dist/cognition/cognitive-port.js +80 -0
- package/dist/cognition/cognitive-wire-contract.d.ts +1542 -0
- package/dist/cognition/cognitive-wire-contract.js +164 -0
- package/dist/cognition/context-renderer.d.ts +52 -0
- package/dist/cognition/context-renderer.js +227 -0
- package/dist/cognition/global-workspace.d.ts +39 -0
- package/dist/cognition/global-workspace.js +181 -0
- package/dist/cognition/index.d.ts +9 -0
- package/dist/cognition/index.js +10 -0
- package/dist/cognition/llm-specialist.d.ts +40 -0
- package/dist/cognition/llm-specialist.js +93 -0
- package/dist/cognition/sensory-workspace.d.ts +57 -0
- package/dist/cognition/sensory-workspace.js +346 -0
- package/dist/cognition/types.d.ts +93 -0
- package/dist/cognition/types.js +7 -0
- package/dist/cognition/voice-specialists.d.ts +21 -0
- package/dist/cognition/voice-specialists.js +89 -0
- package/dist/cognition/world-model.d.ts +95 -0
- package/dist/cognition/world-model.js +228 -0
- package/dist/collaboration/ai-colab-manager.js +12 -13
- package/dist/collaboration/team-session.js +25 -11
- package/dist/commands/apply-permission-mode.d.ts +27 -0
- package/dist/commands/apply-permission-mode.js +35 -0
- package/dist/commands/assistant.d.ts +3 -0
- package/dist/commands/assistant.js +1022 -0
- package/dist/commands/campaign.d.ts +6 -0
- package/dist/commands/campaign.js +140 -0
- package/dist/commands/capsule.d.ts +2 -0
- package/dist/commands/capsule.js +104 -0
- package/dist/commands/changelog.d.ts +29 -0
- package/dist/commands/changelog.js +283 -0
- package/dist/commands/cli/acp-command.js +4 -1
- package/dist/commands/cli/backup-command.d.ts +9 -0
- package/dist/commands/cli/backup-command.js +47 -0
- package/dist/commands/cli/evolve-command.d.ts +26 -0
- package/dist/commands/cli/evolve-command.js +259 -0
- package/dist/commands/cli/fleet-commands.js +105 -0
- package/dist/commands/cli/improve-command.d.ts +2 -1
- package/dist/commands/cli/improve-command.js +309 -20
- package/dist/commands/cli/native-engine-commands.js +405 -28
- package/dist/commands/cli/sensory-command.d.ts +8 -0
- package/dist/commands/cli/sensory-command.js +27 -0
- package/dist/commands/cli/speak-command.js +87 -24
- package/dist/commands/cli/tools-commands.js +68 -36
- package/dist/commands/cli/utility-commands.js +34 -9
- package/dist/commands/client-dispatcher.d.ts +1 -0
- package/dist/commands/client-dispatcher.js +40 -21
- package/dist/commands/compress.js +3 -0
- package/dist/commands/cost.d.ts +33 -0
- package/dist/commands/cost.js +342 -0
- package/dist/commands/council-scoreboard.d.ts +8 -0
- package/dist/commands/council-scoreboard.js +68 -0
- package/dist/commands/council.d.ts +16 -24
- package/dist/commands/council.js +247 -255
- package/dist/commands/curator-cli.d.ts +14 -0
- package/dist/commands/curator-cli.js +57 -0
- package/dist/commands/delegate.js +82 -25
- package/dist/commands/dev/golden-path.d.ts +71 -0
- package/dist/commands/dev/golden-path.js +317 -0
- package/dist/commands/dev/index.d.ts +3 -0
- package/dist/commands/dev/index.js +151 -119
- package/dist/commands/dev/issue-pipeline.js +7 -7
- package/dist/commands/dev/workflows.d.ts +2 -0
- package/dist/commands/dev/workflows.js +39 -6
- package/dist/commands/enhanced-command-handler.d.ts +3 -0
- package/dist/commands/enhanced-command-handler.js +29 -12
- package/dist/commands/exchange.d.ts +2 -0
- package/dist/commands/exchange.js +193 -0
- package/dist/commands/explain.d.ts +12 -0
- package/dist/commands/explain.js +135 -0
- package/dist/commands/film.d.ts +18 -0
- package/dist/commands/film.js +239 -0
- package/dist/commands/flow.js +15 -5
- package/dist/commands/forge.d.ts +2 -0
- package/dist/commands/forge.js +102 -0
- package/dist/commands/goal-cli.d.ts +3 -0
- package/dist/commands/goal-cli.js +9 -4
- package/dist/commands/gpu-worker.d.ts +14 -0
- package/dist/commands/gpu-worker.js +135 -0
- package/dist/commands/handlers/agents-handler.d.ts +11 -2
- package/dist/commands/handlers/agents-handler.js +79 -13
- package/dist/commands/handlers/auth-handlers.js +12 -5
- package/dist/commands/handlers/backup-handlers.d.ts +40 -0
- package/dist/commands/handlers/backup-handlers.js +784 -57
- package/dist/commands/handlers/batch-handlers.d.ts +72 -2
- package/dist/commands/handlers/batch-handlers.js +443 -37
- package/dist/commands/handlers/branch-handlers.d.ts +1 -0
- package/dist/commands/handlers/channel-handlers.d.ts +7 -2
- package/dist/commands/handlers/channel-handlers.js +1665 -111
- package/dist/commands/handlers/companion-handler.js +130 -9
- package/dist/commands/handlers/daily-reset-handler.d.ts +1 -1
- package/dist/commands/handlers/daily-reset-handler.js +1 -1
- package/dist/commands/handlers/deepthink-handler.d.ts +2 -0
- package/dist/commands/handlers/deepthink-handler.js +49 -0
- package/dist/commands/handlers/extra-handlers.js +23 -24
- package/dist/commands/handlers/fleet-handler.d.ts +3 -4
- package/dist/commands/handlers/fleet-handler.js +82 -35
- package/dist/commands/handlers/goal-handler.d.ts +13 -0
- package/dist/commands/handlers/goal-handler.js +26 -7
- package/dist/commands/handlers/grill-me-handler.d.ts +2 -0
- package/dist/commands/handlers/grill-me-handler.js +41 -0
- package/dist/commands/handlers/heartbeat-handler.d.ts +2 -2
- package/dist/commands/handlers/heartbeat-handler.js +2 -2
- package/dist/commands/handlers/index.d.ts +3 -2
- package/dist/commands/handlers/index.js +4 -2
- package/dist/commands/handlers/memory-handlers.js +40 -0
- package/dist/commands/handlers/permissions-handlers.js +3 -2
- package/dist/commands/handlers/pr-handlers.js +15 -17
- package/dist/commands/handlers/security-handlers.d.ts +1 -1
- package/dist/commands/handlers/security-handlers.js +3 -3
- package/dist/commands/handlers/slash-session-handlers.d.ts +9 -0
- package/dist/commands/handlers/slash-session-handlers.js +134 -0
- package/dist/commands/handlers/swarm-handler.js +41 -2
- package/dist/commands/handlers/team-handlers.d.ts +28 -1
- package/dist/commands/handlers/team-handlers.js +243 -2
- package/dist/commands/handlers/think-handlers.js +2 -1
- package/dist/commands/handlers/vibe-handlers.js +8 -1
- package/dist/commands/handlers/worktree-handlers.js +23 -12
- package/dist/commands/headless-slash.d.ts +9 -0
- package/dist/commands/headless-slash.js +41 -3
- package/dist/commands/import.d.ts +34 -0
- package/dist/commands/import.js +478 -0
- package/dist/commands/index.d.ts +0 -1
- package/dist/commands/index.js +0 -1
- package/dist/commands/influencer.d.ts +2 -0
- package/dist/commands/influencer.js +87 -0
- package/dist/commands/intent.d.ts +6 -0
- package/dist/commands/intent.js +131 -0
- package/dist/commands/intents.d.ts +10 -0
- package/dist/commands/intents.js +163 -0
- package/dist/commands/lessons.js +91 -0
- package/dist/commands/llm-provider-resolution.d.ts +20 -0
- package/dist/commands/llm-provider-resolution.js +74 -13
- package/dist/commands/login-prerequisites.d.ts +11 -0
- package/dist/commands/login-prerequisites.js +23 -0
- package/dist/commands/loop-cli.d.ts +31 -0
- package/dist/commands/loop-cli.js +189 -0
- package/dist/commands/lora.d.ts +6 -0
- package/dist/commands/lora.js +445 -0
- package/dist/commands/maison-food.d.ts +10 -0
- package/dist/commands/maison-food.js +485 -0
- package/dist/commands/maison.d.ts +17 -0
- package/dist/commands/maison.js +236 -0
- package/dist/commands/mcp.d.ts +6 -0
- package/dist/commands/mcp.js +236 -7
- package/dist/commands/meeting.d.ts +8 -0
- package/dist/commands/meeting.js +31 -0
- package/dist/commands/pairing.js +7 -8
- package/dist/commands/papers/ask.d.ts +47 -0
- package/dist/commands/papers/ask.js +105 -0
- package/dist/commands/papers/index.d.ts +19 -0
- package/dist/commands/papers/index.js +54 -0
- package/dist/commands/provider.d.ts +9 -0
- package/dist/commands/provider.js +178 -10
- package/dist/commands/replay.d.ts +11 -0
- package/dist/commands/replay.js +131 -0
- package/dist/commands/research/deep.d.ts +66 -0
- package/dist/commands/research/deep.js +236 -0
- package/dist/commands/research/discover-searxng.d.ts +25 -0
- package/dist/commands/research/discover-searxng.js +62 -0
- package/dist/commands/research/index.d.ts +24 -2
- package/dist/commands/research/index.js +456 -79
- package/dist/commands/research/knowledge-ingest.d.ts +170 -0
- package/dist/commands/research/knowledge-ingest.js +449 -0
- package/dist/commands/research/wire-research-worker.d.ts +9 -0
- package/dist/commands/research/wire-research-worker.js +18 -0
- package/dist/commands/run-cli/index.d.ts +3 -1
- package/dist/commands/run-cli/index.js +53 -2
- package/dist/commands/science/deps.d.ts +95 -0
- package/dist/commands/science/deps.js +343 -0
- package/dist/commands/science/index.d.ts +31 -0
- package/dist/commands/science/index.js +285 -0
- package/dist/commands/science/loop-option.d.ts +40 -0
- package/dist/commands/science/loop-option.js +91 -0
- package/dist/commands/science/sandbox-option.d.ts +44 -0
- package/dist/commands/science/sandbox-option.js +42 -0
- package/dist/commands/scrape.d.ts +18 -0
- package/dist/commands/scrape.js +159 -0
- package/dist/commands/self.d.ts +9 -0
- package/dist/commands/self.js +56 -0
- package/dist/commands/shadow.d.ts +3 -0
- package/dist/commands/shadow.js +72 -0
- package/dist/commands/share.d.ts +13 -0
- package/dist/commands/share.js +71 -0
- package/dist/commands/shell-prefix.d.ts +4 -22
- package/dist/commands/shell-prefix.js +11 -73
- package/dist/commands/skills-cli/index.d.ts +2 -0
- package/dist/commands/skills-cli/index.js +190 -15
- package/dist/commands/slash/builtin-commands.js +41 -21
- package/dist/commands/slash-commands.js +4 -1
- package/dist/commands/token.d.ts +80 -0
- package/dist/commands/token.js +319 -0
- package/dist/commands/try.d.ts +72 -0
- package/dist/commands/try.js +357 -0
- package/dist/commands/update.d.ts +14 -1
- package/dist/commands/update.js +97 -15
- package/dist/commands/vision-train.d.ts +15 -0
- package/dist/commands/vision-train.js +214 -0
- package/dist/commands/whoami-status.d.ts +17 -0
- package/dist/commands/whoami-status.js +34 -0
- package/dist/commands/widgets.d.ts +2 -0
- package/dist/commands/widgets.js +179 -0
- package/dist/commands/ws.d.ts +7 -0
- package/dist/commands/ws.js +100 -0
- package/dist/companion/assistant-config.d.ts +79 -0
- package/dist/companion/assistant-config.js +1148 -0
- package/dist/companion/attached-image-grounding.d.ts +29 -0
- package/dist/companion/attached-image-grounding.js +285 -0
- package/dist/companion/away-mode.d.ts +80 -0
- package/dist/companion/away-mode.js +269 -0
- package/dist/companion/camera-share.d.ts +65 -0
- package/dist/companion/camera-share.js +321 -0
- package/dist/companion/camera.d.ts +8 -0
- package/dist/companion/camera.js +57 -31
- package/dist/companion/cards.js +7 -10
- package/dist/companion/check-in.js +23 -8
- package/dist/companion/companion-doctor.d.ts +58 -0
- package/dist/companion/companion-doctor.js +194 -0
- package/dist/companion/companion-history.d.ts +23 -0
- package/dist/companion/companion-history.js +22 -0
- package/dist/companion/companion-mode.d.ts +77 -1
- package/dist/companion/companion-mode.js +916 -6
- package/dist/companion/companion-photo-intake.d.ts +40 -0
- package/dist/companion/companion-photo-intake.js +112 -0
- package/dist/companion/companion-photo.d.ts +144 -0
- package/dist/companion/companion-photo.js +318 -0
- package/dist/companion/companion-turn.d.ts +88 -0
- package/dist/companion/companion-turn.js +223 -0
- package/dist/companion/companion-voice-character.d.ts +62 -0
- package/dist/companion/companion-voice-character.js +148 -0
- package/dist/companion/competitive-radar.js +14 -13
- package/dist/companion/continuity.d.ts +62 -0
- package/dist/companion/continuity.js +284 -0
- package/dist/companion/conversation-improvement-loop.d.ts +61 -0
- package/dist/companion/conversation-improvement-loop.js +317 -0
- package/dist/companion/conversation-quality-insights.d.ts +56 -0
- package/dist/companion/conversation-quality-insights.js +161 -0
- package/dist/companion/cooking-timer-runner.d.ts +20 -0
- package/dist/companion/cooking-timer-runner.js +98 -0
- package/dist/companion/core-adapter.d.ts +60 -0
- package/dist/companion/core-adapter.js +111 -0
- package/dist/companion/crisis-safety.d.ts +52 -0
- package/dist/companion/crisis-safety.js +150 -0
- package/dist/companion/daily-interaction-budget.d.ts +34 -0
- package/dist/companion/daily-interaction-budget.js +260 -0
- package/dist/companion/dialogue-percepts.d.ts +16 -0
- package/dist/companion/dialogue-percepts.js +50 -0
- package/dist/companion/event-followups.d.ts +58 -0
- package/dist/companion/event-followups.js +213 -0
- package/dist/companion/fashion-scene-catalog.d.ts +29 -0
- package/dist/companion/fashion-scene-catalog.js +141 -0
- package/dist/companion/gateway-inbox.js +25 -13
- package/dist/companion/gateway.d.ts +1 -0
- package/dist/companion/gateway.js +37 -17
- package/dist/companion/home-interaction-policy.d.ts +31 -0
- package/dist/companion/home-interaction-policy.js +89 -0
- package/dist/companion/household-time.d.ts +9 -0
- package/dist/companion/household-time.js +15 -0
- package/dist/companion/idle-loop.d.ts +53 -0
- package/dist/companion/idle-loop.js +254 -0
- package/dist/companion/impulses.js +148 -23
- package/dist/companion/inner-life.d.ts +60 -0
- package/dist/companion/inner-life.js +112 -0
- package/dist/companion/jokes.d.ts +21 -0
- package/dist/companion/jokes.js +175 -0
- package/dist/companion/lisa-selfie-cache.d.ts +36 -0
- package/dist/companion/lisa-selfie-cache.js +123 -0
- package/dist/companion/lisa-selfie-ingest.d.ts +36 -0
- package/dist/companion/lisa-selfie-ingest.js +205 -0
- package/dist/companion/lisa-selfie-refill.d.ts +32 -0
- package/dist/companion/lisa-selfie-refill.js +121 -0
- package/dist/companion/lisa-selfie-router.d.ts +40 -0
- package/dist/companion/lisa-selfie-router.js +156 -0
- package/dist/companion/lisa-selfie.d.ts +97 -0
- package/dist/companion/lisa-selfie.js +542 -0
- package/dist/companion/maison-voice-actions.d.ts +34 -0
- package/dist/companion/maison-voice-actions.js +192 -0
- package/dist/companion/migration.d.ts +63 -0
- package/dist/companion/migration.js +270 -0
- package/dist/companion/mission-board.js +7 -10
- package/dist/companion/mission-runner.js +2 -3
- package/dist/companion/mobile-conversation-log.d.ts +19 -0
- package/dist/companion/mobile-conversation-log.js +157 -0
- package/dist/companion/mobile-history.d.ts +34 -0
- package/dist/companion/mobile-history.js +123 -0
- package/dist/companion/mysoulmate-image-prompts.d.ts +26 -0
- package/dist/companion/mysoulmate-image-prompts.js +278 -0
- package/dist/companion/orchestrator.d.ts +32 -0
- package/dist/companion/orchestrator.js +49 -0
- package/dist/companion/percepts.d.ts +70 -0
- package/dist/companion/percepts.js +251 -18
- package/dist/companion/personas/copine.d.ts +8 -0
- package/dist/companion/personas/copine.js +195 -0
- package/dist/companion/personas/index.d.ts +19 -0
- package/dist/companion/personas/index.js +33 -0
- package/dist/companion/personas/types.d.ts +34 -0
- package/dist/companion/personas/types.js +8 -0
- package/dist/companion/photo-memory-fr.d.ts +36 -0
- package/dist/companion/photo-memory-fr.js +291 -0
- package/dist/companion/prefetch-config.d.ts +38 -0
- package/dist/companion/prefetch-config.js +119 -0
- package/dist/companion/prefetch-engine.d.ts +87 -0
- package/dist/companion/prefetch-engine.js +513 -0
- package/dist/companion/presence-loop.d.ts +85 -0
- package/dist/companion/presence-loop.js +297 -0
- package/dist/companion/privacy.js +3 -2
- package/dist/companion/proactive-engine.d.ts +97 -0
- package/dist/companion/proactive-engine.js +447 -0
- package/dist/companion/recent-said.d.ts +30 -0
- package/dist/companion/recent-said.js +98 -0
- package/dist/companion/relational-benchmark-scenarios.d.ts +191 -0
- package/dist/companion/relational-benchmark-scenarios.js +227 -0
- package/dist/companion/relational-context.d.ts +71 -0
- package/dist/companion/relational-context.js +302 -0
- package/dist/companion/relational-episode-evaluator.d.ts +117 -0
- package/dist/companion/relational-episode-evaluator.js +453 -0
- package/dist/companion/relationship-evolution.d.ts +2 -0
- package/dist/companion/relationship-evolution.js +30 -0
- package/dist/companion/relationship-state.d.ts +106 -0
- package/dist/companion/relationship-state.js +293 -0
- package/dist/companion/reminder-runner.d.ts +33 -0
- package/dist/companion/reminder-runner.js +183 -0
- package/dist/companion/reminders.d.ts +228 -0
- package/dist/companion/reminders.js +906 -0
- package/dist/companion/reply-augment.d.ts +87 -0
- package/dist/companion/reply-augment.js +464 -0
- package/dist/companion/safety-ledger.js +19 -12
- package/dist/companion/self-evaluation.js +22 -23
- package/dist/companion/shared-photo-memory.d.ts +65 -0
- package/dist/companion/shared-photo-memory.js +145 -0
- package/dist/companion/shared-photos.d.ts +88 -0
- package/dist/companion/shared-photos.js +321 -0
- package/dist/companion/signature-locations.d.ts +16 -0
- package/dist/companion/signature-locations.js +108 -0
- package/dist/companion/skill-curator.d.ts +9 -0
- package/dist/companion/skill-curator.js +147 -58
- package/dist/companion/untrusted-text.d.ts +22 -0
- package/dist/companion/untrusted-text.js +47 -0
- package/dist/companion/user-name.d.ts +18 -0
- package/dist/companion/user-name.js +22 -0
- package/dist/companion/visual-consent.d.ts +40 -0
- package/dist/companion/visual-consent.js +98 -0
- package/dist/companion/visual-grounding.d.ts +83 -0
- package/dist/companion/visual-grounding.js +474 -0
- package/dist/companion/voice-callbacks.d.ts +9 -0
- package/dist/companion/voice-callbacks.js +84 -0
- package/dist/companion/voice-guidance.d.ts +23 -0
- package/dist/companion/voice-guidance.js +85 -0
- package/dist/companion/voice-improvement-loop.d.ts +53 -0
- package/dist/companion/voice-improvement-loop.js +279 -0
- package/dist/companion/voice-incident-repair.d.ts +41 -0
- package/dist/companion/voice-incident-repair.js +214 -0
- package/dist/config/constants.d.ts +41 -0
- package/dist/config/constants.js +14 -0
- package/dist/config/env-schema.d.ts +2 -0
- package/dist/config/env-schema.js +714 -5
- package/dist/config/feature-flags.js +8 -12
- package/dist/config/feature-surface.d.ts +12 -0
- package/dist/config/feature-surface.js +86 -0
- package/dist/config/headless-local-prompt.d.ts +9 -0
- package/dist/config/headless-local-prompt.js +39 -0
- package/dist/config/local-runtime-context.d.ts +41 -0
- package/dist/config/local-runtime-context.js +430 -0
- package/dist/config/model-registry.js +4 -0
- package/dist/config/model-strengths.d.ts +14 -0
- package/dist/config/model-strengths.js +15 -0
- package/dist/config/model-tools.d.ts +56 -0
- package/dist/config/model-tools.js +819 -21
- package/dist/config/secret-ref.d.ts +22 -3
- package/dist/config/secret-ref.js +103 -41
- package/dist/config/toml-config.d.ts +20 -4
- package/dist/config/toml-config.js +40 -2
- package/dist/context/auto-compact-threshold.d.ts +1 -1
- package/dist/context/auto-compact-threshold.js +1 -1
- package/dist/context/codebase-map.js +9 -4
- package/dist/context/codebase-rag/codebase-rag.js +7 -6
- package/dist/context/codebase-rag/hnsw-store.js +9 -4
- package/dist/context/codebase-rag/vector-store.js +4 -4
- package/dist/context/compaction/parallel-summarizer.js +1 -0
- package/dist/context/compression.js +25 -29
- package/dist/context/context-manager-v2.d.ts +95 -7
- package/dist/context/context-manager-v2.js +442 -89
- package/dist/context/context-manager-v3.d.ts +4 -0
- package/dist/context/context-manager-v3.js +68 -9
- package/dist/context/cross-encoder-reranker.js +8 -3
- package/dist/context/default-context-engine.js +1 -1
- package/dist/context/enhanced-compression.d.ts +26 -2
- package/dist/context/enhanced-compression.js +81 -30
- package/dist/context/file-mentions.d.ts +46 -0
- package/dist/context/file-mentions.js +245 -0
- package/dist/context/importance-scorer.js +4 -3
- package/dist/context/jit-context.d.ts +7 -0
- package/dist/context/jit-context.js +22 -4
- package/dist/context/lm-resizer-compressor.d.ts +102 -0
- package/dist/context/lm-resizer-compressor.js +593 -0
- package/dist/context/precompaction-flush.js +1 -1
- package/dist/context/restorable-compression.d.ts +59 -14
- package/dist/context/restorable-compression.js +310 -60
- package/dist/context/segment-archive.d.ts +36 -0
- package/dist/context/segment-archive.js +220 -0
- package/dist/context/smart-compaction.d.ts +7 -0
- package/dist/context/smart-compaction.js +84 -15
- package/dist/context/token-counter.d.ts +6 -0
- package/dist/context/token-counter.js +41 -0
- package/dist/context/token-juice.d.ts +72 -0
- package/dist/context/token-juice.js +220 -0
- package/dist/context/tool-observation-optimizer.d.ts +90 -0
- package/dist/context/tool-observation-optimizer.js +231 -0
- package/dist/context/tool-pair-preserver.d.ts +1 -1
- package/dist/context/tool-pair-preserver.js +1 -1
- package/dist/context/transcript-repair.d.ts +16 -3
- package/dist/context/transcript-repair.js +72 -35
- package/dist/context/types.d.ts +2 -0
- package/dist/conversation/argument-obligations.d.ts +65 -0
- package/dist/conversation/argument-obligations.js +157 -0
- package/dist/conversation/companion-model-routing.d.ts +121 -0
- package/dist/conversation/companion-model-routing.js +810 -0
- package/dist/conversation/conversation-benchmark.d.ts +143 -0
- package/dist/conversation/conversation-benchmark.js +766 -0
- package/dist/conversation/conversation-blind-comparison.d.ts +135 -0
- package/dist/conversation/conversation-blind-comparison.js +451 -0
- package/dist/conversation/conversation-evaluator.d.ts +56 -0
- package/dist/conversation/conversation-evaluator.js +335 -0
- package/dist/conversation/conversation-orchestrator.d.ts +32 -0
- package/dist/conversation/conversation-orchestrator.js +59 -0
- package/dist/conversation/conversation-pilot-corpus.d.ts +33 -0
- package/dist/conversation/conversation-pilot-corpus.js +313 -0
- package/dist/conversation/conversation-quality.d.ts +31 -0
- package/dist/conversation/conversation-quality.js +280 -0
- package/dist/conversation/conversation-state.d.ts +23 -0
- package/dist/conversation/conversation-state.js +117 -0
- package/dist/conversation/cross-channel-bridge.d.ts +127 -0
- package/dist/conversation/cross-channel-bridge.js +785 -0
- package/dist/conversation/deliberation-thread.d.ts +18 -0
- package/dist/conversation/deliberation-thread.js +303 -0
- package/dist/conversation/dialogue-act.d.ts +11 -0
- package/dist/conversation/dialogue-act.js +132 -0
- package/dist/conversation/discourse-planner.d.ts +3 -0
- package/dist/conversation/discourse-planner.js +141 -0
- package/dist/conversation/fresh-context.d.ts +54 -0
- package/dist/conversation/fresh-context.js +258 -0
- package/dist/conversation/prefetched-turn-context.d.ts +45 -0
- package/dist/conversation/prefetched-turn-context.js +209 -0
- package/dist/conversation/relationship-safety.d.ts +37 -0
- package/dist/conversation/relationship-safety.js +407 -0
- package/dist/conversation/semantic-response-gate.d.ts +262 -0
- package/dist/conversation/semantic-response-gate.js +656 -0
- package/dist/conversation/semantic-response-runtime.d.ts +79 -0
- package/dist/conversation/semantic-response-runtime.js +419 -0
- package/dist/conversation/shared-relationship-state.d.ts +60 -0
- package/dist/conversation/shared-relationship-state.js +262 -0
- package/dist/conversation/types.d.ts +77 -0
- package/dist/conversation/types.js +3 -0
- package/dist/conversation/voice-continuity.d.ts +13 -0
- package/dist/conversation/voice-continuity.js +48 -0
- package/dist/council/conductor.d.ts +26 -0
- package/dist/council/conductor.js +292 -0
- package/dist/council/council-engine.d.ts +29 -0
- package/dist/council/council-engine.js +458 -0
- package/dist/council/deliberation-health.d.ts +58 -0
- package/dist/council/deliberation-health.js +108 -0
- package/dist/council/judge.d.ts +69 -0
- package/dist/council/judge.js +183 -0
- package/dist/council/peers.d.ts +19 -0
- package/dist/council/peers.js +47 -0
- package/dist/council/signals.d.ts +26 -0
- package/dist/council/signals.js +156 -0
- package/dist/council/task-types.d.ts +16 -0
- package/dist/council/task-types.js +58 -0
- package/dist/council/triage.d.ts +48 -0
- package/dist/council/triage.js +228 -0
- package/dist/council/types.d.ts +292 -0
- package/dist/council/types.js +20 -0
- package/dist/council/with-timeout.d.ts +10 -0
- package/dist/council/with-timeout.js +29 -0
- package/dist/curator/curator.d.ts +79 -0
- package/dist/curator/curator.js +317 -0
- package/dist/daemon/agent-task-executor.js +51 -0
- package/dist/daemon/autonomous-daemon.d.ts +2 -0
- package/dist/daemon/autonomous-daemon.js +4 -1
- package/dist/daemon/autonomous-loop.d.ts +9 -0
- package/dist/daemon/autonomous-loop.js +57 -1
- package/dist/daemon/autonomy-bench-candidates.d.ts +33 -0
- package/dist/daemon/autonomy-bench-candidates.js +68 -0
- package/dist/daemon/autonomy-briefing.d.ts +112 -0
- package/dist/daemon/autonomy-briefing.js +359 -0
- package/dist/daemon/autonomy-run-journal.d.ts +20 -0
- package/dist/daemon/autonomy-run-journal.js +41 -0
- package/dist/daemon/daily-reset.d.ts +2 -2
- package/dist/daemon/daily-reset.js +16 -15
- package/dist/daemon/ollama-task-executor.js +9 -1
- package/dist/database/database-manager.d.ts +1 -1
- package/dist/database/database-manager.js +6 -4
- package/dist/database/optional-sqlite.d.ts +20 -0
- package/dist/database/optional-sqlite.js +101 -0
- package/dist/design/design-system-registry.d.ts +21 -0
- package/dist/design/design-system-registry.js +106 -0
- package/dist/desktop/codebuddy-engine-adapter.d.ts +34 -0
- package/dist/desktop/codebuddy-engine-adapter.js +306 -21
- package/dist/desktop/electron-launch-args.d.ts +7 -0
- package/dist/desktop/electron-launch-args.js +13 -0
- package/dist/desktop/engine-adapter.d.ts +6 -0
- package/dist/desktop/launcher.js +2 -1
- package/dist/desktop/permission-bridge.d.ts +12 -1
- package/dist/desktop/permission-bridge.js +12 -5
- package/dist/desktop-automation/omniparser-runner.d.ts +1 -0
- package/dist/desktop-automation/omniparser-runner.js +12 -1
- package/dist/desktop-automation/smart-snapshot.js +7 -0
- package/dist/desktop-automation/windows-native-provider.js +1 -1
- package/dist/doctor/assistant-runtime.d.ts +90 -0
- package/dist/doctor/assistant-runtime.js +472 -0
- package/dist/doctor/env-sanity.d.ts +17 -0
- package/dist/doctor/env-sanity.js +74 -0
- package/dist/doctor/index.d.ts +37 -0
- package/dist/doctor/index.js +389 -37
- package/dist/doctor/llm-key-check.d.ts +17 -0
- package/dist/doctor/llm-key-check.js +100 -0
- package/dist/doctor/ollama-model-selection.d.ts +16 -0
- package/dist/doctor/ollama-model-selection.js +55 -0
- package/dist/embeddings/embedding-provider.d.ts +1 -0
- package/dist/embeddings/embedding-provider.js +21 -9
- package/dist/errors/crash-recovery.js +6 -4
- package/dist/events/index.d.ts +1 -1
- package/dist/events/types.d.ts +44 -0
- package/dist/export/repo-explanation.d.ts +10 -0
- package/dist/export/repo-explanation.js +334 -0
- package/dist/export/session-share.d.ts +34 -0
- package/dist/export/session-share.js +687 -0
- package/dist/fleet/autonomous-tick-broadcaster.d.ts +1 -1
- package/dist/fleet/capability-registry.d.ts +9 -0
- package/dist/fleet/capability-registry.js +268 -56
- package/dist/fleet/colab-store.d.ts +3 -4
- package/dist/fleet/colab-store.js +6 -26
- package/dist/fleet/compatible-model.d.ts +39 -0
- package/dist/fleet/compatible-model.js +148 -0
- package/dist/fleet/consolidation/collecte.d.ts +31 -0
- package/dist/fleet/consolidation/collecte.js +86 -0
- package/dist/fleet/consolidation/digest-texte.d.ts +11 -0
- package/dist/fleet/consolidation/digest-texte.js +52 -0
- package/dist/fleet/consolidation/thalamus.d.ts +35 -0
- package/dist/fleet/consolidation/thalamus.js +151 -0
- package/dist/fleet/consolidation/types.d.ts +66 -0
- package/dist/fleet/consolidation/types.js +22 -0
- package/dist/fleet/cost-tracker.d.ts +11 -0
- package/dist/fleet/cost-tracker.js +64 -17
- package/dist/fleet/fleet-cost-cap.d.ts +36 -0
- package/dist/fleet/fleet-cost-cap.js +175 -0
- package/dist/fleet/fleet-http-surface.d.ts +24 -0
- package/dist/fleet/fleet-http-surface.js +72 -0
- package/dist/fleet/fleet-listener.d.ts +1 -1
- package/dist/fleet/fleet-listener.js +4 -0
- package/dist/fleet/model-capability-heuristics.d.ts +31 -0
- package/dist/fleet/model-capability-heuristics.js +49 -0
- package/dist/fleet/model-inventory.d.ts +46 -0
- package/dist/fleet/model-inventory.js +242 -0
- package/dist/fleet/model-scoreboard.d.ts +118 -13
- package/dist/fleet/model-scoreboard.js +339 -27
- package/dist/fleet/model-selector.d.ts +83 -0
- package/dist/fleet/model-selector.js +218 -0
- package/dist/fleet/peer-chat-bridge.d.ts +9 -2
- package/dist/fleet/peer-chat-bridge.js +133 -19
- package/dist/fleet/peer-chat-client-factory.d.ts +28 -2
- package/dist/fleet/peer-chat-client-factory.js +195 -22
- package/dist/fleet/peer-ckg-bridge.d.ts +67 -0
- package/dist/fleet/peer-ckg-bridge.js +467 -0
- package/dist/fleet/peer-mission-exchange-bridge.d.ts +8 -0
- package/dist/fleet/peer-mission-exchange-bridge.js +127 -0
- package/dist/fleet/peer-session-bridge.d.ts +7 -1
- package/dist/fleet/peer-session-bridge.js +98 -17
- package/dist/fleet/peer-session-store.d.ts +2 -0
- package/dist/fleet/peer-session-store.js +6 -5
- package/dist/fleet/peer-text-sanitizer.d.ts +59 -0
- package/dist/fleet/peer-text-sanitizer.js +99 -0
- package/dist/fleet/peer-tool-bridge.js +38 -10
- package/dist/fleet/privacy-lint.js +80 -8
- package/dist/fleet/saga-store.d.ts +25 -1
- package/dist/fleet/saga-store.js +34 -6
- package/dist/fleet/task-router.d.ts +17 -1
- package/dist/fleet/task-router.js +65 -20
- package/dist/fleet/types.d.ts +9 -5
- package/dist/gateway/device-pairing.js +5 -11
- package/dist/gateway/ws-transport.d.ts +1 -5
- package/dist/gateway/ws-transport.js +2 -22
- package/dist/git/changelog.d.ts +36 -0
- package/dist/git/changelog.js +109 -0
- package/dist/git/worktree-sessions.js +1 -1
- package/dist/goals/counterfactual-forge.d.ts +70 -0
- package/dist/goals/counterfactual-forge.js +184 -0
- package/dist/goals/criterion-progress.d.ts +24 -0
- package/dist/goals/criterion-progress.js +59 -0
- package/dist/goals/evidence-artifact.d.ts +18 -0
- package/dist/goals/evidence-artifact.js +76 -0
- package/dist/goals/goal-decomposer.d.ts +15 -0
- package/dist/goals/goal-decomposer.js +89 -4
- package/dist/goals/goal-loop.d.ts +29 -0
- package/dist/goals/goal-loop.js +92 -6
- package/dist/goals/goal-manager.d.ts +1 -0
- package/dist/goals/goal-manager.js +4 -1
- package/dist/goals/goal-state.d.ts +10 -0
- package/dist/goals/goal-state.js +21 -1
- package/dist/goals/goal-store.js +4 -6
- package/dist/goals/index.d.ts +11 -0
- package/dist/goals/index.js +11 -0
- package/dist/goals/intent-graph.d.ts +40 -0
- package/dist/goals/intent-graph.js +133 -0
- package/dist/goals/mission-constitution.d.ts +59 -0
- package/dist/goals/mission-constitution.js +132 -0
- package/dist/goals/mission-exchange.d.ts +86 -0
- package/dist/goals/mission-exchange.js +216 -0
- package/dist/goals/outcome-capsule.d.ts +80 -0
- package/dist/goals/outcome-capsule.js +186 -0
- package/dist/goals/proof-ledger.d.ts +79 -0
- package/dist/goals/proof-ledger.js +181 -0
- package/dist/goals/proven-outcome-lessons.d.ts +11 -0
- package/dist/goals/proven-outcome-lessons.js +30 -0
- package/dist/goals/proven-outcome-memory.d.ts +55 -0
- package/dist/goals/proven-outcome-memory.js +125 -0
- package/dist/goals/shadow-twin.d.ts +57 -0
- package/dist/goals/shadow-twin.js +110 -0
- package/dist/gpu-worker/gpu-media-worker-server.d.ts +49 -0
- package/dist/gpu-worker/gpu-media-worker-server.js +637 -0
- package/dist/harness/contract.d.ts +84 -84
- package/dist/hooks/advanced-hooks.js +6 -4
- package/dist/hooks/env-persistence.js +5 -6
- package/dist/hooks/hook-manager.js +8 -7
- package/dist/hooks/hook-system.js +6 -4
- package/dist/hooks/lifecycle-hooks.js +6 -3
- package/dist/hooks/moltbot/moltbot-hooks-manager.js +8 -3
- package/dist/hooks/moltbot/session-persistence-manager.js +15 -7
- package/dist/hooks/moltbot/setup-utilities.js +6 -4
- package/dist/hooks/smart-hooks.js +6 -4
- package/dist/hooks/use-input-handler.js +9 -63
- package/dist/hooks/user-hooks.d.ts +10 -1
- package/dist/hooks/user-hooks.js +77 -1
- package/dist/identity/companion-identity.d.ts +4 -1
- package/dist/identity/companion-identity.js +134 -4
- package/dist/identity/identity-manager.d.ts +1 -1
- package/dist/identity/identity-manager.js +7 -6
- package/dist/identity/lisa-introspection.d.ts +36 -0
- package/dist/identity/lisa-introspection.js +411 -0
- package/dist/identity/operational-self-model.d.ts +143 -0
- package/dist/identity/operational-self-model.js +974 -0
- package/dist/index.d.ts +3 -1
- package/dist/index.js +987 -223
- package/dist/input/context-mentions.d.ts +7 -1
- package/dist/input/context-mentions.js +18 -20
- package/dist/input/text-to-speech.d.ts +3 -0
- package/dist/input/text-to-speech.js +70 -27
- package/dist/input/voice-control.d.ts +1 -0
- package/dist/input/voice-control.js +24 -18
- package/dist/input/voice-input-enhanced.d.ts +1 -0
- package/dist/input/voice-input-enhanced.js +31 -16
- package/dist/integrations/ci-autofix-pipeline.js +15 -20
- package/dist/integrations/ide/server.js +3 -2
- package/dist/integrations/json-rpc/server.js +4 -2
- package/dist/integrations/mcp/mcp-server.js +4 -2
- package/dist/integrations/opentelemetry-integration.js +6 -4
- package/dist/integrations/pubcommander-bridge.d.ts +5 -0
- package/dist/integrations/pubcommander-bridge.js +61 -0
- package/dist/integrations/sentry-integration.js +9 -3
- package/dist/intelligence/semantic-search.js +9 -12
- package/dist/intelligence/user-preferences.js +8 -11
- package/dist/intents/intent-checker.d.ts +34 -0
- package/dist/intents/intent-checker.js +175 -0
- package/dist/intents/intent-generator.d.ts +12 -0
- package/dist/intents/intent-generator.js +86 -0
- package/dist/intents/intent-store.d.ts +62 -0
- package/dist/intents/intent-store.js +263 -0
- package/dist/interpreter/computer/skills.js +3 -2
- package/dist/interpreter/interpreter-service.js +18 -9
- package/dist/kanban/kanban-board-registry.js +6 -4
- package/dist/kanban/kanban-store.js +5 -7
- package/dist/knowledge/code-graph-persistence.js +8 -16
- package/dist/knowledge/graph-drift.js +14 -11
- package/dist/knowledge/knowledge-manager.js +2 -1
- package/dist/knowledge/scanners/index.d.ts +3 -0
- package/dist/knowledge/scanners/index.js +57 -29
- package/dist/knowledge/workspace-indexer.js +22 -15
- package/dist/life-rhythm/cooking-timer-store.d.ts +50 -0
- package/dist/life-rhythm/cooking-timer-store.js +240 -0
- package/dist/life-rhythm/day-context.d.ts +30 -0
- package/dist/life-rhythm/day-context.js +198 -0
- package/dist/life-rhythm/etalab-holiday-provider.d.ts +40 -0
- package/dist/life-rhythm/etalab-holiday-provider.js +318 -0
- package/dist/life-rhythm/home-mode-store.d.ts +32 -0
- package/dist/life-rhythm/home-mode-store.js +184 -0
- package/dist/life-rhythm/index.d.ts +6 -0
- package/dist/life-rhythm/index.js +7 -0
- package/dist/life-rhythm/types.d.ts +88 -0
- package/dist/life-rhythm/types.js +33 -0
- package/dist/life-rhythm/zoned-minute.d.ts +24 -0
- package/dist/life-rhythm/zoned-minute.js +113 -0
- package/dist/location/index.d.ts +2 -12
- package/dist/location/index.js +129 -56
- package/dist/logging/interaction-logger.js +29 -46
- package/dist/lora/dataset-v3-plan.d.ts +27 -0
- package/dist/lora/dataset-v3-plan.js +121 -0
- package/dist/lora/dataset.d.ts +26 -0
- package/dist/lora/dataset.js +165 -0
- package/dist/lora/fal-krea-trainer.d.ts +54 -0
- package/dist/lora/fal-krea-trainer.js +197 -0
- package/dist/lora/generate-training-set.d.ts +61 -0
- package/dist/lora/generate-training-set.js +190 -0
- package/dist/lora/identity-dataset-gate.d.ts +104 -0
- package/dist/lora/identity-dataset-gate.js +204 -0
- package/dist/lora/identity-dataset-promotion.d.ts +32 -0
- package/dist/lora/identity-dataset-promotion.js +269 -0
- package/dist/lora/index.d.ts +11 -0
- package/dist/lora/index.js +12 -0
- package/dist/lora/install-comfy.d.ts +14 -0
- package/dist/lora/install-comfy.js +63 -0
- package/dist/lora/lisa-avatar-bible.d.ts +85 -0
- package/dist/lora/lisa-avatar-bible.js +275 -0
- package/dist/lora/local-plan.d.ts +10 -0
- package/dist/lora/local-plan.js +205 -0
- package/dist/lora/pack-dataset.d.ts +7 -0
- package/dist/lora/pack-dataset.js +39 -0
- package/dist/lora/quality-gate.d.ts +31 -0
- package/dist/lora/quality-gate.js +90 -0
- package/dist/lora/types.d.ts +66 -0
- package/dist/lora/types.js +5 -0
- package/dist/lsp/lsp-client.d.ts +12 -0
- package/dist/lsp/lsp-client.js +45 -9
- package/dist/lsp/server.js +1 -1
- package/dist/mcp/approval-elicitation.d.ts +1 -0
- package/dist/mcp/approval-elicitation.js +2 -0
- package/dist/mcp/client.d.ts +15 -1
- package/dist/mcp/client.js +110 -24
- package/dist/mcp/config.d.ts +12 -1
- package/dist/mcp/config.js +55 -12
- package/dist/mcp/index.d.ts +1 -1
- package/dist/mcp/mcp-agent-tools.d.ts +1 -1
- package/dist/mcp/mcp-agent-tools.js +76 -73
- package/dist/mcp/mcp-ckg-tools.d.ts +31 -0
- package/dist/mcp/mcp-ckg-tools.js +121 -0
- package/dist/mcp/mcp-client.js +8 -5
- package/dist/mcp/mcp-desktop-tools.d.ts +1 -1
- package/dist/mcp/mcp-desktop-tools.js +105 -99
- package/dist/mcp/mcp-memory-tools.d.ts +1 -1
- package/dist/mcp/mcp-memory-tools.js +63 -61
- package/dist/mcp/mcp-oauth.js +5 -2
- package/dist/mcp/mcp-server.d.ts +48 -79
- package/dist/mcp/mcp-server.js +445 -478
- package/dist/mcp/mcp-session-tools.d.ts +1 -1
- package/dist/mcp/mcp-session-tools.js +95 -92
- package/dist/mcp/profiles.d.ts +18 -0
- package/dist/mcp/profiles.js +65 -0
- package/dist/mcp/prompt-footprint.d.ts +21 -0
- package/dist/mcp/prompt-footprint.js +39 -0
- package/dist/meals/allergens.d.ts +19 -0
- package/dist/meals/allergens.js +122 -0
- package/dist/meals/compatibility.d.ts +6 -0
- package/dist/meals/compatibility.js +109 -0
- package/dist/meals/food-inventory-store.d.ts +51 -0
- package/dist/meals/food-inventory-store.js +257 -0
- package/dist/meals/index.d.ts +10 -0
- package/dist/meals/index.js +11 -0
- package/dist/meals/meal-engine.d.ts +7 -0
- package/dist/meals/meal-engine.js +206 -0
- package/dist/meals/meal-plan-store.d.ts +73 -0
- package/dist/meals/meal-plan-store.js +271 -0
- package/dist/meals/private-json-store.d.ts +9 -0
- package/dist/meals/private-json-store.js +92 -0
- package/dist/meals/profile-store.d.ts +49 -0
- package/dist/meals/profile-store.js +296 -0
- package/dist/meals/profile-validator.d.ts +7 -0
- package/dist/meals/profile-validator.js +124 -0
- package/dist/meals/recipe-normalizer.d.ts +7 -0
- package/dist/meals/recipe-normalizer.js +162 -0
- package/dist/meals/store-validation.d.ts +21 -0
- package/dist/meals/store-validation.js +101 -0
- package/dist/meals/types.d.ts +170 -0
- package/dist/meals/types.js +12 -0
- package/dist/media/comfy-health-supervisor.d.ts +142 -0
- package/dist/media/comfy-health-supervisor.js +608 -0
- package/dist/media/comfyui-recipe-contract.d.ts +696 -0
- package/dist/media/comfyui-recipe-contract.js +580 -0
- package/dist/media/comfyui-recipe-engine.d.ts +3 -0
- package/dist/media/comfyui-recipe-engine.js +4 -0
- package/dist/media/comfyui-recipe-registry.d.ts +35 -0
- package/dist/media/comfyui-recipe-registry.js +385 -0
- package/dist/media/comfyui-recipe-runtime.d.ts +135 -0
- package/dist/media/comfyui-recipe-runtime.js +942 -0
- package/dist/media/content-tier.d.ts +25 -0
- package/dist/media/content-tier.js +29 -0
- package/dist/meeting/analyzer.d.ts +24 -0
- package/dist/meeting/analyzer.js +402 -0
- package/dist/meeting/index.d.ts +13 -0
- package/dist/meeting/index.js +48 -0
- package/dist/meeting/markdown.d.ts +3 -0
- package/dist/meeting/markdown.js +51 -0
- package/dist/meeting/output.d.ts +13 -0
- package/dist/meeting/output.js +155 -0
- package/dist/meeting/transcript.d.ts +25 -0
- package/dist/meeting/transcript.js +292 -0
- package/dist/meeting/types.d.ts +112 -0
- package/dist/meeting/types.js +9 -0
- package/dist/memory/adapters/network-memory-adapters.d.ts +19 -4
- package/dist/memory/adapters/network-memory-adapters.js +4 -4
- package/dist/memory/auto-memory.js +5 -2
- package/dist/memory/background-extractor.d.ts +43 -0
- package/dist/memory/background-extractor.js +368 -0
- package/dist/memory/buddy-memory-client.d.ts +40 -0
- package/dist/memory/buddy-memory-client.js +198 -0
- package/dist/memory/ckg-engine-policy.d.ts +19 -0
- package/dist/memory/ckg-engine-policy.js +45 -0
- package/dist/memory/ckg-fact-reconciliation.d.ts +81 -0
- package/dist/memory/ckg-fact-reconciliation.js +155 -0
- package/dist/memory/ckg-redaction.d.ts +2 -0
- package/dist/memory/ckg-redaction.js +25 -0
- package/dist/memory/coding-style-analyzer.js +2 -2
- package/dist/memory/collective-knowledge-graph.d.ts +380 -0
- package/dist/memory/collective-knowledge-graph.js +1228 -0
- package/dist/memory/enhanced-memory.d.ts +29 -0
- package/dist/memory/enhanced-memory.js +153 -63
- package/dist/memory/facts-memory.d.ts +9 -4
- package/dist/memory/facts-memory.js +21 -4
- package/dist/memory/hybrid-mmr.d.ts +71 -0
- package/dist/memory/hybrid-mmr.js +127 -0
- package/dist/memory/hybrid-search.js +4 -1
- package/dist/memory/knowledge-graph.d.ts +23 -2
- package/dist/memory/knowledge-graph.js +62 -47
- package/dist/memory/memory-candidate-queue.js +9 -4
- package/dist/memory/memory-consolidation.d.ts +8 -0
- package/dist/memory/memory-consolidation.js +66 -17
- package/dist/memory/memory-forgetting.d.ts +57 -0
- package/dist/memory/memory-forgetting.js +87 -0
- package/dist/memory/ocr-memory-pipeline.js +16 -10
- package/dist/memory/persistent-memory.d.ts +98 -0
- package/dist/memory/persistent-memory.js +711 -119
- package/dist/memory/presence-injector.d.ts +1 -1
- package/dist/memory/presence-injector.js +9 -4
- package/dist/memory/semantic-memory-search.js +12 -9
- package/dist/memory/subagent-memory.js +8 -8
- package/dist/memory/user-model.js +9 -4
- package/dist/metrics/metrics-collector.d.ts +1 -0
- package/dist/metrics/metrics-collector.js +11 -2
- package/dist/nodes/device-node.d.ts +5 -2
- package/dist/nodes/device-node.js +65 -19
- package/dist/nodes/index.d.ts +32 -2
- package/dist/nodes/index.js +208 -7
- package/dist/nodes/transports/adb-transport.d.ts +2 -6
- package/dist/nodes/transports/adb-transport.js +29 -13
- package/dist/nodes/transports/base-transport.d.ts +10 -0
- package/dist/observability/run-store.d.ts +6 -1
- package/dist/observability/run-store.js +89 -22
- package/dist/observability/run-trajectory-load.d.ts +16 -0
- package/dist/observability/run-trajectory-load.js +266 -0
- package/dist/observability/run-trajectory.d.ts +168 -0
- package/dist/observability/run-trajectory.js +468 -0
- package/dist/observability/run-viewer.d.ts +5 -1
- package/dist/observability/run-viewer.js +83 -9
- package/dist/observability/turn-metrics.d.ts +86 -0
- package/dist/observability/turn-metrics.js +201 -0
- package/dist/offline/offline-mode.js +10 -9
- package/dist/openclaw/gateway-bridge.js +3 -2
- package/dist/optimization/latency-optimizer.d.ts +2 -1
- package/dist/optimization/latency-optimizer.js +2 -2
- package/dist/optimization/model-routing.d.ts +3 -0
- package/dist/orchestration/orchestrator.js +4 -1
- package/dist/orchestration/types.d.ts +2 -0
- package/dist/performance/lazy-loader.js +2 -1
- package/dist/performance/tool-cache.d.ts +4 -0
- package/dist/performance/tool-cache.js +4 -2
- package/dist/persistence/conversation-branches.js +9 -2
- package/dist/persistence/session-store.d.ts +26 -0
- package/dist/persistence/session-store.js +56 -17
- package/dist/personas/persona-manager.d.ts +39 -1
- package/dist/personas/persona-manager.js +276 -38
- package/dist/plugins/bundled/groq-provider.js +10 -7
- package/dist/plugins/bundled/openrouter-provider.js +2 -1
- package/dist/plugins/code-explorer/CodeExplorerManager.d.ts +52 -4
- package/dist/plugins/code-explorer/CodeExplorerManager.js +201 -31
- package/dist/plugins/code-explorer/code-explorer-client.d.ts +36 -0
- package/dist/plugins/code-explorer/code-explorer-client.js +86 -0
- package/dist/plugins/code-explorer/index.d.ts +1 -3
- package/dist/plugins/code-explorer/index.js +0 -1
- package/dist/plugins/git-pinned-marketplace.js +3 -2
- package/dist/plugins/plugin-manager.js +6 -9
- package/dist/plugins/sandbox-worker.js +1 -1
- package/dist/prompts/system-base.js +27 -3
- package/dist/protocols/a2a/codebuddy-executor.d.ts +9 -0
- package/dist/protocols/a2a/codebuddy-executor.js +25 -6
- package/dist/protocols/a2a/index.d.ts +6 -0
- package/dist/protocols/a2a/index.js +32 -3
- package/dist/protocols/acp/acp-agentic-runner.d.ts +6 -2
- package/dist/protocols/acp/acp-agentic-runner.js +91 -15
- package/dist/protocols/acp/acp-session-store.js +28 -5
- package/dist/protocols/acp/acp-stdio-server.d.ts +13 -0
- package/dist/protocols/acp/acp-stdio-server.js +30 -3
- package/dist/providers/active-llm-model-pool.d.ts +45 -0
- package/dist/providers/active-llm-model-pool.js +139 -0
- package/dist/providers/active-llm-registry.js +48 -8
- package/dist/providers/auxiliary-provider.d.ts +1 -1
- package/dist/providers/auxiliary-provider.js +23 -10
- package/dist/providers/chatgpt-models.d.ts +71 -0
- package/dist/providers/chatgpt-models.js +302 -0
- package/dist/providers/codex-oauth.d.ts +8 -0
- package/dist/providers/codex-oauth.js +47 -39
- package/dist/providers/gemini-oauth.js +7 -6
- package/dist/providers/grok-provider.js +1 -1
- package/dist/providers/index.d.ts +3 -0
- package/dist/providers/index.js +3 -0
- package/dist/providers/local-model-resolver.d.ts +66 -0
- package/dist/providers/local-model-resolver.js +128 -0
- package/dist/providers/model-egress.d.ts +8 -0
- package/dist/providers/model-egress.js +62 -0
- package/dist/providers/model-provider-compat.d.ts +13 -0
- package/dist/providers/model-provider-compat.js +48 -0
- package/dist/providers/provider-catalog.d.ts +4 -1
- package/dist/providers/provider-catalog.js +494 -28
- package/dist/providers/provider-failover-notify.d.ts +15 -0
- package/dist/providers/provider-failover-notify.js +93 -0
- package/dist/providers/provider-failover-policy.d.ts +28 -0
- package/dist/providers/provider-failover-policy.js +188 -0
- package/dist/providers/provider-failover-user-notice.d.ts +13 -0
- package/dist/providers/provider-failover-user-notice.js +49 -0
- package/dist/providers/provider-fallback.d.ts +1 -0
- package/dist/providers/provider-fallback.js +1 -1
- package/dist/providers/provider-health.d.ts +46 -0
- package/dist/providers/provider-health.js +249 -0
- package/dist/providers/turboquant-provider.d.ts +1 -1
- package/dist/providers/xai-oauth.js +7 -34
- package/dist/queue/persistent-queue.js +9 -6
- package/dist/renderers/charts/special-charts.d.ts +7 -1
- package/dist/renderers/charts/special-charts.js +5 -0
- package/dist/renderers/weather-conditions.d.ts +20 -0
- package/dist/renderers/weather-conditions.js +74 -0
- package/dist/renderers/weather-renderer.js +4 -18
- package/dist/rendering/ansi.d.ts +4 -0
- package/dist/rendering/ansi.js +149 -0
- package/dist/rendering/index.d.ts +19 -0
- package/dist/rendering/index.js +18 -0
- package/dist/rendering/plain.d.ts +2 -0
- package/dist/rendering/plain.js +91 -0
- package/dist/rendering/telegram-html.d.ts +7 -0
- package/dist/rendering/telegram-html.js +76 -16
- package/dist/research/auto-ingest.d.ts +45 -0
- package/dist/research/auto-ingest.js +85 -0
- package/dist/research/code-explorer-source.d.ts +29 -0
- package/dist/research/code-explorer-source.js +76 -0
- package/dist/research/connector-source.d.ts +25 -0
- package/dist/research/connector-source.js +314 -0
- package/dist/research/paper-qa/answer.d.ts +72 -0
- package/dist/research/paper-qa/answer.js +227 -0
- package/dist/research/paper-qa/corpus.d.ts +36 -0
- package/dist/research/paper-qa/corpus.js +102 -0
- package/dist/research/paper-qa/disk-embedding-cache.d.ts +28 -0
- package/dist/research/paper-qa/disk-embedding-cache.js +193 -0
- package/dist/research/paper-qa/index.d.ts +31 -0
- package/dist/research/paper-qa/index.js +24 -0
- package/dist/research/paper-qa/paper-qa-pipeline.d.ts +107 -0
- package/dist/research/paper-qa/paper-qa-pipeline.js +163 -0
- package/dist/research/paper-qa/passage-index.d.ts +203 -0
- package/dist/research/paper-qa/passage-index.js +405 -0
- package/dist/research/paper-qa/pdf-structure.d.ts +37 -0
- package/dist/research/paper-qa/pdf-structure.js +377 -0
- package/dist/research/paper-qa/persistent-corpus-index.d.ts +23 -0
- package/dist/research/paper-qa/persistent-corpus-index.js +122 -0
- package/dist/research/paper-qa/prose-chunker.d.ts +19 -0
- package/dist/research/paper-qa/prose-chunker.js +201 -0
- package/dist/research/paper-qa/provenance.d.ts +24 -0
- package/dist/research/paper-qa/provenance.js +40 -0
- package/dist/research/paper-qa/rcs.d.ts +78 -0
- package/dist/research/paper-qa/rcs.js +200 -0
- package/dist/research/paper-qa/types.d.ts +124 -0
- package/dist/research/paper-qa/types.js +13 -0
- package/dist/research/publication-sources.d.ts +34 -0
- package/dist/research/publication-sources.js +131 -0
- package/dist/research/relation-classifier.d.ts +13 -0
- package/dist/research/relation-classifier.js +58 -0
- package/dist/research/research-topics.d.ts +13 -0
- package/dist/research/research-topics.js +81 -0
- package/dist/review/apply-transaction.d.ts +40 -0
- package/dist/review/apply-transaction.js +112 -0
- package/dist/review/diff-model.d.ts +39 -0
- package/dist/review/diff-model.js +124 -0
- package/dist/review/llm-client.d.ts +30 -0
- package/dist/review/llm-client.js +98 -0
- package/dist/review/llm-reviewer.d.ts +16 -0
- package/dist/review/llm-reviewer.js +122 -0
- package/dist/review/review-engine.d.ts +54 -0
- package/dist/review/review-engine.js +0 -0
- package/dist/review/revision-loop.d.ts +67 -0
- package/dist/review/revision-loop.js +160 -0
- package/dist/review/static-gate.d.ts +13 -0
- package/dist/review/static-gate.js +110 -0
- package/dist/review/types.d.ts +122 -0
- package/dist/review/types.js +24 -0
- package/dist/review/write-gate.d.ts +55 -0
- package/dist/review/write-gate.js +121 -0
- package/dist/sandbox/auto-sandbox.d.ts +4 -2
- package/dist/sandbox/auto-sandbox.js +14 -1
- package/dist/sandbox/docker-sandbox.d.ts +39 -0
- package/dist/sandbox/docker-sandbox.js +292 -14
- package/dist/sandbox/execpolicy.d.ts +36 -1
- package/dist/sandbox/execpolicy.js +375 -40
- package/dist/sandbox/os-sandbox.d.ts +53 -0
- package/dist/sandbox/os-sandbox.js +315 -70
- package/dist/sandbox/sandbox-backend.d.ts +2 -0
- package/dist/scheduler/cron-scheduler.d.ts +1 -0
- package/dist/scheduler/cron-scheduler.js +142 -41
- package/dist/scripting/codebuddy-bindings.js +2 -1
- package/dist/search/bm25.js +5 -2
- package/dist/search/usearch-index.js +7 -5
- package/dist/security/audit-logger.js +14 -7
- package/dist/security/bash-allowlist/allowlist-store.d.ts +8 -2
- package/dist/security/bash-allowlist/allowlist-store.js +52 -12
- package/dist/security/bash-allowlist/approval-flow.d.ts +1 -1
- package/dist/security/bash-allowlist/approval-flow.js +10 -8
- package/dist/security/bash-allowlist/deny-guard.d.ts +14 -0
- package/dist/security/bash-allowlist/deny-guard.js +61 -0
- package/dist/security/bash-allowlist/types.d.ts +5 -0
- package/dist/security/bash-parser.d.ts +32 -0
- package/dist/security/bash-parser.js +129 -14
- package/dist/security/dangerous-patterns.js +18 -3
- package/dist/security/declarative-rules.d.ts +1 -1
- package/dist/security/declarative-rules.js +242 -65
- package/dist/security/dependency-vuln-scanner.js +4 -4
- package/dist/security/dev-origins.d.ts +40 -0
- package/dist/security/dev-origins.js +111 -0
- package/dist/security/docker-sandbox/manager.js +1 -1
- package/dist/security/env-blocklist.d.ts +7 -0
- package/dist/security/env-blocklist.js +22 -2
- package/dist/security/guardian-agent.d.ts +0 -5
- package/dist/security/guardian-agent.js +17 -6
- package/dist/security/index.d.ts +3 -0
- package/dist/security/index.js +2 -0
- package/dist/security/native-sandbox.d.ts +80 -0
- package/dist/security/native-sandbox.js +455 -0
- package/dist/security/pack-contents-policy.d.ts +49 -0
- package/dist/security/pack-contents-policy.js +139 -0
- package/dist/security/permission-config.js +2 -5
- package/dist/security/permission-modes.d.ts +11 -0
- package/dist/security/permission-modes.js +71 -7
- package/dist/security/policy-amendments.js +40 -22
- package/dist/security/policy-engine.js +39 -12
- package/dist/security/powershell-parser.d.ts +40 -0
- package/dist/security/powershell-parser.js +198 -0
- package/dist/security/remote-approval.d.ts +0 -1
- package/dist/security/remote-approval.js +3 -2
- package/dist/security/safe-binaries.d.ts +13 -1
- package/dist/security/safe-binaries.js +240 -32
- package/dist/security/safe-fetch.d.ts +7 -0
- package/dist/security/safe-fetch.js +101 -0
- package/dist/security/sandbox.js +2 -2
- package/dist/security/secret-patterns.d.ts +18 -0
- package/dist/security/secret-patterns.js +232 -0
- package/dist/security/secret-scrubber.d.ts +31 -0
- package/dist/security/secret-scrubber.js +170 -0
- package/dist/security/secrets-detector.d.ts +3 -1
- package/dist/security/secrets-detector.js +3 -106
- package/dist/security/security-modes.js +2 -1
- package/dist/security/session-encryption.d.ts +7 -2
- package/dist/security/session-encryption.js +28 -7
- package/dist/security/shell-env-policy.js +11 -2
- package/dist/security/skill-scanner.d.ts +2 -2
- package/dist/security/skill-scanner.js +237 -11
- package/dist/security/ssrf-guard.d.ts +13 -0
- package/dist/security/ssrf-guard.js +27 -0
- package/dist/security/text-deobfuscation.d.ts +41 -0
- package/dist/security/text-deobfuscation.js +256 -0
- package/dist/security/tool-permissions.js +2 -3
- package/dist/security/tool-policy/approval-scope.d.ts +11 -0
- package/dist/security/tool-policy/approval-scope.js +67 -0
- package/dist/security/tool-policy/policy-manager.d.ts +9 -0
- package/dist/security/tool-policy/policy-manager.js +28 -1
- package/dist/security/tool-policy/profiles.js +24 -0
- package/dist/security/tool-policy/tool-groups.js +26 -3
- package/dist/security/tool-policy/types.d.ts +3 -1
- package/dist/security/tool-policy/types.js +2 -0
- package/dist/security/trust-folders.js +3 -2
- package/dist/security/write-policy.d.ts +8 -0
- package/dist/security/write-policy.js +63 -1
- package/dist/self-model/evolution-notes.d.ts +51 -0
- package/dist/self-model/evolution-notes.js +324 -0
- package/dist/sensory/agent-reply.d.ts +136 -0
- package/dist/sensory/agent-reply.js +682 -0
- package/dist/sensory/alert.d.ts +18 -0
- package/dist/sensory/alert.js +105 -0
- package/dist/sensory/arrival-opener.d.ts +102 -0
- package/dist/sensory/arrival-opener.js +292 -0
- package/dist/sensory/audio-scene.d.ts +24 -0
- package/dist/sensory/audio-scene.js +74 -0
- package/dist/sensory/camera-keyframe-policy.d.ts +11 -0
- package/dist/sensory/camera-keyframe-policy.js +43 -0
- package/dist/sensory/conversation-cues.d.ts +55 -0
- package/dist/sensory/conversation-cues.js +140 -0
- package/dist/sensory/domain-event-bridge.d.ts +7 -0
- package/dist/sensory/domain-event-bridge.js +134 -0
- package/dist/sensory/dreaming.d.ts +17 -0
- package/dist/sensory/dreaming.js +52 -2
- package/dist/sensory/elevenlabs-library.d.ts +91 -0
- package/dist/sensory/elevenlabs-library.js +303 -0
- package/dist/sensory/episodic-journal.d.ts +69 -0
- package/dist/sensory/episodic-journal.js +240 -0
- package/dist/sensory/error-watch-reaction.d.ts +41 -0
- package/dist/sensory/error-watch-reaction.js +312 -0
- package/dist/sensory/heartbeat-fallback.d.ts +46 -0
- package/dist/sensory/heartbeat-fallback.js +171 -0
- package/dist/sensory/heartbeat-scheduler.d.ts +6 -0
- package/dist/sensory/heartbeat-scheduler.js +27 -7
- package/dist/sensory/hybrid-reply.d.ts +132 -0
- package/dist/sensory/hybrid-reply.js +1217 -0
- package/dist/sensory/reactions.d.ts +6 -0
- package/dist/sensory/reactions.js +10 -1
- package/dist/sensory/respond-decider.d.ts +124 -0
- package/dist/sensory/respond-decider.js +608 -0
- package/dist/sensory/rule-templates.d.ts +30 -0
- package/dist/sensory/rule-templates.js +100 -0
- package/dist/sensory/schedule-emitter.d.ts +24 -0
- package/dist/sensory/schedule-emitter.js +57 -0
- package/dist/sensory/screen-reaction.js +15 -1
- package/dist/sensory/semantic-vision-reaction.d.ts +36 -0
- package/dist/sensory/semantic-vision-reaction.js +342 -0
- package/dist/sensory/sensory-action-executor.d.ts +98 -0
- package/dist/sensory/sensory-action-executor.js +473 -0
- package/dist/sensory/sensory-bridge.d.ts +11 -1
- package/dist/sensory/sensory-bridge.js +59 -7
- package/dist/sensory/sensory-rules-engine.d.ts +81 -0
- package/dist/sensory/sensory-rules-engine.js +434 -0
- package/dist/sensory/sensory-status.d.ts +102 -0
- package/dist/sensory/sensory-status.js +291 -0
- package/dist/sensory/speech-engine-config.d.ts +68 -0
- package/dist/sensory/speech-engine-config.js +157 -0
- package/dist/sensory/speech-reaction.d.ts +183 -7
- package/dist/sensory/speech-reaction.js +2141 -35
- package/dist/sensory/speech-sanitizer.d.ts +12 -0
- package/dist/sensory/speech-sanitizer.js +143 -0
- package/dist/sensory/system-vitals-emitter.d.ts +130 -0
- package/dist/sensory/system-vitals-emitter.js +514 -0
- package/dist/sensory/tts-cache.d.ts +76 -0
- package/dist/sensory/tts-cache.js +0 -0
- package/dist/sensory/turn-detector.d.ts +28 -0
- package/dist/sensory/turn-detector.js +57 -0
- package/dist/sensory/vision-description-safety.d.ts +4 -0
- package/dist/sensory/vision-description-safety.js +25 -0
- package/dist/sensory/vision-reaction.d.ts +21 -7
- package/dist/sensory/vision-reaction.js +209 -18
- package/dist/sensory/voice-activity.d.ts +69 -0
- package/dist/sensory/voice-activity.js +271 -0
- package/dist/sensory/voice-clock.d.ts +19 -0
- package/dist/sensory/voice-clock.js +111 -0
- package/dist/sensory/voice-entrainment.d.ts +62 -0
- package/dist/sensory/voice-entrainment.js +176 -0
- package/dist/sensory/voice-interactions.d.ts +13 -0
- package/dist/sensory/voice-interactions.js +258 -0
- package/dist/sensory/voice-loop.d.ts +618 -0
- package/dist/sensory/voice-loop.js +3581 -0
- package/dist/sensory/voice-replay-lab.d.ts +32 -0
- package/dist/sensory/voice-replay-lab.js +109 -0
- package/dist/sensory/voice-stream.d.ts +121 -0
- package/dist/sensory/voice-stream.js +598 -0
- package/dist/sensory/voice-turn-coordinator.d.ts +88 -0
- package/dist/sensory/voice-turn-coordinator.js +258 -0
- package/dist/sensory/voice-turn-taking.d.ts +26 -0
- package/dist/sensory/voice-turn-taking.js +59 -0
- package/dist/server/agent-adapter.d.ts +49 -3
- package/dist/server/agent-adapter.js +26 -4
- package/dist/server/channel-a2a-bridge.js +1 -1
- package/dist/server/exposure-diagnostic.d.ts +25 -0
- package/dist/server/exposure-diagnostic.js +51 -0
- package/dist/server/heartbeat-monitor.d.ts +13 -0
- package/dist/server/heartbeat-monitor.js +8 -3
- package/dist/server/http-agent-sessions.d.ts +32 -0
- package/dist/server/http-agent-sessions.js +245 -0
- package/dist/server/index.d.ts +3 -0
- package/dist/server/index.js +913 -46
- package/dist/server/mcp/approval-elicitation.d.ts +144 -0
- package/dist/server/mcp/approval-elicitation.js +377 -0
- package/dist/server/middleware/auth.d.ts +11 -0
- package/dist/server/middleware/auth.js +54 -0
- package/dist/server/middleware/error-handler.d.ts +5 -0
- package/dist/server/middleware/error-handler.js +30 -4
- package/dist/server/middleware/index.d.ts +1 -1
- package/dist/server/middleware/index.js +1 -1
- package/dist/server/mobile/album.d.ts +46 -0
- package/dist/server/mobile/album.js +144 -0
- package/dist/server/mobile/assets/app.js +3036 -0
- package/dist/server/mobile/assets/emoji-data.js +508 -0
- package/dist/server/mobile/assets/icon-192.png +0 -0
- package/dist/server/mobile/assets/icon-512.png +0 -0
- package/dist/server/mobile/assets/icon-96.png +0 -0
- package/dist/server/mobile/assets/icon.svg +12 -0
- package/dist/server/mobile/assets/index.html +261 -0
- package/dist/server/mobile/assets/manifest.webmanifest +37 -0
- package/dist/server/mobile/assets/styles.css +831 -0
- package/dist/server/mobile/assets/sw.js +149 -0
- package/dist/server/mobile/chat-extras.d.ts +9 -0
- package/dist/server/mobile/chat-extras.js +24 -0
- package/dist/server/mobile/index.d.ts +22 -0
- package/dist/server/mobile/index.js +287 -0
- package/dist/server/mobile/link-preview.d.ts +24 -0
- package/dist/server/mobile/link-preview.js +139 -0
- package/dist/server/mobile/push.d.ts +38 -0
- package/dist/server/mobile/push.js +190 -0
- package/dist/server/mobile/status.d.ts +23 -0
- package/dist/server/mobile/status.js +107 -0
- package/dist/server/mobile/telegram-forward.d.ts +14 -0
- package/dist/server/mobile/telegram-forward.js +32 -0
- package/dist/server/mobile/voice-note.d.ts +54 -0
- package/dist/server/mobile/voice-note.js +305 -0
- package/dist/server/origin-check.d.ts +4 -5
- package/dist/server/origin-check.js +43 -8
- package/dist/server/routes/canvas.d.ts +7 -1
- package/dist/server/routes/canvas.js +138 -24
- package/dist/server/routes/chat.js +340 -203
- package/dist/server/routes/cognition.d.ts +3 -0
- package/dist/server/routes/cognition.js +75 -0
- package/dist/server/routes/health.js +93 -81
- package/dist/server/routes/index.d.ts +5 -1
- package/dist/server/routes/index.js +5 -1
- package/dist/server/routes/lessons.d.ts +13 -0
- package/dist/server/routes/lessons.js +111 -0
- package/dist/server/routes/memory.d.ts +12 -0
- package/dist/server/routes/memory.js +191 -131
- package/dist/server/routes/runs.d.ts +9 -0
- package/dist/server/routes/runs.js +50 -0
- package/dist/server/routes/sessions.js +8 -1
- package/dist/server/routes/tools.js +93 -74
- package/dist/server/routes/webhooks.js +56 -46
- package/dist/server/tunnel-manager.js +21 -1
- package/dist/server/types.d.ts +25 -1
- package/dist/server/webhook-agent-queue.d.ts +12 -0
- package/dist/server/webhook-agent-queue.js +58 -0
- package/dist/server/websocket/cognition-bridge.d.ts +13 -0
- package/dist/server/websocket/cognition-bridge.js +334 -0
- package/dist/server/websocket/confirmation-bridge.d.ts +22 -0
- package/dist/server/websocket/confirmation-bridge.js +178 -0
- package/dist/server/websocket/desktop-handler.d.ts +17 -0
- package/dist/server/websocket/desktop-handler.js +177 -16
- package/dist/server/websocket/handler.d.ts +123 -1
- package/dist/server/websocket/handler.js +852 -67
- package/dist/server/websocket/index.d.ts +2 -1
- package/dist/server/websocket/index.js +2 -1
- package/dist/server/websocket/peer-rpc.d.ts +1 -1
- package/dist/server/websocket/peer-rpc.js +25 -3
- package/dist/services/prompt-builder.d.ts +47 -2
- package/dist/services/prompt-builder.js +310 -99
- package/dist/sessions/timeline-snapshot.d.ts +35 -0
- package/dist/sessions/timeline-snapshot.js +144 -0
- package/dist/sessions/timeline.d.ts +33 -0
- package/dist/sessions/timeline.js +85 -0
- package/dist/shared/context-optimization-metadata.d.ts +29 -0
- package/dist/shared/context-optimization-metadata.js +73 -0
- package/dist/shared/engine-types.d.ts +26 -0
- package/dist/skills/bash-injection.d.ts +2 -2
- package/dist/skills/bash-injection.js +9 -2
- package/dist/skills/bundled/code-explorer.skill.md +50 -0
- package/dist/skills/bundled/file-edit.skill.md +62 -0
- package/dist/skills/bundled/git-commit.skill.md +66 -0
- package/dist/skills/bundled/pubcommander-control/SKILL.md +33 -0
- package/dist/skills/bundled/pubcommander-control/agents/openai.yaml +4 -0
- package/dist/skills/bundled/typescript-expert.skill.md +21 -0
- package/dist/skills/bundled/weather.skill.md +48 -0
- package/dist/skills/bundled/web-app-testing.skill.md +71 -0
- package/dist/skills/bundled/web-search.skill.md +56 -0
- package/dist/skills/hub.js +52 -30
- package/dist/skills/index.d.ts +6 -3
- package/dist/skills/index.js +19 -4
- package/dist/skills/parser.js +2 -0
- package/dist/skills/registry.d.ts +52 -0
- package/dist/skills/registry.js +291 -14
- package/dist/skills/skill-exchange.d.ts +61 -0
- package/dist/skills/skill-exchange.js +523 -0
- package/dist/skills/skill-importer.d.ts +2 -2
- package/dist/skills/skill-importer.js +76 -17
- package/dist/skills/skill-registry.js +3 -2
- package/dist/skills/skill-signing.d.ts +20 -0
- package/dist/skills/skill-signing.js +106 -0
- package/dist/skills/skill-sources.d.ts +3 -2
- package/dist/skills/skill-sources.js +5 -4
- package/dist/skills/types.d.ts +4 -0
- package/dist/spec/spec-store.js +7 -6
- package/dist/speculative/shadow-workspace.d.ts +79 -0
- package/dist/speculative/shadow-workspace.js +500 -0
- package/dist/talk-mode/index.d.ts +4 -4
- package/dist/talk-mode/index.js +3 -3
- package/dist/talk-mode/providers/elevenlabs-client.d.ts +48 -0
- package/dist/talk-mode/providers/elevenlabs-client.js +117 -0
- package/dist/talk-mode/providers/elevenlabs.js +12 -23
- package/dist/talk-mode/providers/index.d.ts +3 -1
- package/dist/talk-mode/providers/index.js +2 -0
- package/dist/talk-mode/providers/pocket-tts.d.ts +88 -0
- package/dist/talk-mode/providers/pocket-tts.js +334 -0
- package/dist/talk-mode/providers/voicebox-tts.d.ts +24 -0
- package/dist/talk-mode/providers/voicebox-tts.js +120 -0
- package/dist/talk-mode/types.d.ts +39 -1
- package/dist/tasks/background-tasks.js +22 -5
- package/dist/telemetry/otel-tracer.js +5 -2
- package/dist/templates/design-system-apply.d.ts +23 -0
- package/dist/templates/design-system-apply.js +100 -0
- package/dist/templates/project-scaffolding.d.ts +2 -0
- package/dist/templates/project-scaffolding.js +616 -1
- package/dist/themes/theme-manager.js +9 -6
- package/dist/themes/theme-schema.d.ts +46 -46
- package/dist/tools/advanced/multi-file-editor.js +3 -1
- package/dist/tools/advanced/operation-history.js +2 -1
- package/dist/tools/app-server-tool.d.ts +66 -0
- package/dist/tools/app-server-tool.js +423 -0
- package/dist/tools/apply-patch.d.ts +31 -1
- package/dist/tools/apply-patch.js +214 -19
- package/dist/tools/authored-tools-manifest-2.d.ts +17 -0
- package/dist/tools/authored-tools-manifest-2.js +133 -0
- package/dist/tools/authored-tools-manifest.d.ts +15 -0
- package/dist/tools/authored-tools-manifest.js +13 -0
- package/dist/tools/bash/bash-tool.d.ts +36 -3
- package/dist/tools/bash/bash-tool.js +240 -100
- package/dist/tools/bash/command-validator.d.ts +1 -1
- package/dist/tools/bash/command-validator.js +115 -40
- package/dist/tools/bash/execution-policy.d.ts +42 -0
- package/dist/tools/bash/execution-policy.js +278 -0
- package/dist/tools/bash/security-patterns.js +43 -5
- package/dist/tools/bash/streaming-executor.d.ts +1 -1
- package/dist/tools/bash/streaming-executor.js +124 -18
- package/dist/tools/build-project-tool.d.ts +25 -0
- package/dist/tools/build-project-tool.js +30 -0
- package/dist/tools/bundle-analyze-tool.d.ts +28 -0
- package/dist/tools/bundle-analyze-tool.js +39 -0
- package/dist/tools/code-exec-tool.d.ts +55 -23
- package/dist/tools/code-exec-tool.js +598 -111
- package/dist/tools/code-explorer-tool.d.ts +1 -1
- package/dist/tools/code-explorer-tool.js +25 -1
- package/dist/tools/code-review.js +5 -0
- package/dist/tools/code-stats-tool.d.ts +43 -0
- package/dist/tools/code-stats-tool.js +70 -0
- package/dist/tools/codebase-replace-tool.js +2 -2
- package/dist/tools/comfy-recipe-tool.d.ts +37 -0
- package/dist/tools/comfy-recipe-tool.js +693 -0
- package/dist/tools/community-search.d.ts +93 -0
- package/dist/tools/community-search.js +334 -0
- package/dist/tools/computer-control-harness.js +5 -2
- package/dist/tools/computer-control-tool.d.ts +8 -0
- package/dist/tools/computer-control-tool.js +71 -26
- package/dist/tools/context-expand-tool.d.ts +19 -0
- package/dist/tools/context-expand-tool.js +132 -0
- package/dist/tools/csv/csv-parse.d.ts +19 -0
- package/dist/tools/csv/csv-parse.js +132 -0
- package/dist/tools/csv/csv-wiring.d.ts +8 -0
- package/dist/tools/csv/csv-wiring.js +9 -0
- package/dist/tools/csv-analyze-tool.d.ts +11 -0
- package/dist/tools/csv-analyze-tool.js +114 -0
- package/dist/tools/csv-preview-tool.d.ts +34 -0
- package/dist/tools/csv-preview-tool.js +69 -0
- package/dist/tools/db-migration.js +11 -3
- package/dist/tools/deep-research-tool.d.ts +68 -0
- package/dist/tools/deep-research-tool.js +257 -0
- package/dist/tools/deferred-schema-state.d.ts +19 -0
- package/dist/tools/deferred-schema-state.js +33 -0
- package/dist/tools/dep-inspect-tool.d.ts +32 -0
- package/dist/tools/dep-inspect-tool.js +74 -0
- package/dist/tools/design-system-tool.d.ts +10 -0
- package/dist/tools/design-system-tool.js +95 -0
- package/dist/tools/device-tool.d.ts +2 -1
- package/dist/tools/device-tool.js +21 -0
- package/dist/tools/diagram-tool.js +8 -2
- package/dist/tools/diff-files-tool.d.ts +28 -0
- package/dist/tools/diff-files-tool.js +49 -0
- package/dist/tools/enhanced-search.js +32 -16
- package/dist/tools/env-doctor-tool.d.ts +30 -0
- package/dist/tools/env-doctor-tool.js +54 -0
- package/dist/tools/execute-code-rpc-invoker.js +2 -2
- package/dist/tools/execute-code-runner.d.ts +21 -0
- package/dist/tools/execute-code-runner.js +117 -8
- package/dist/tools/extension-forge-tool.d.ts +29 -0
- package/dist/tools/extension-forge-tool.js +288 -0
- package/dist/tools/fetch-tool.js +7 -3
- package/dist/tools/file-search-tool.d.ts +41 -0
- package/dist/tools/file-search-tool.js +99 -0
- package/dist/tools/format-project-tool.d.ts +28 -0
- package/dist/tools/format-project-tool.js +45 -0
- package/dist/tools/git-summary-tool.d.ts +36 -0
- package/dist/tools/git-summary-tool.js +65 -0
- package/dist/tools/git-tool.js +16 -4
- package/dist/tools/gpu-avatar-delivery.d.ts +16 -0
- package/dist/tools/gpu-avatar-delivery.js +67 -0
- package/dist/tools/gpu-media-worker.d.ts +96 -0
- package/dist/tools/gpu-media-worker.js +363 -0
- package/dist/tools/gui-tool.js +92 -13
- package/dist/tools/http-probe-tool.d.ts +25 -0
- package/dist/tools/http-probe-tool.js +40 -0
- package/dist/tools/image-tool.js +29 -16
- package/dist/tools/index.d.ts +6 -1
- package/dist/tools/index.js +5 -0
- package/dist/tools/interactive-bash.d.ts +29 -1
- package/dist/tools/interactive-bash.js +141 -21
- package/dist/tools/json-query-tool.d.ts +30 -0
- package/dist/tools/json-query-tool.js +49 -0
- package/dist/tools/license-check-tool.d.ts +22 -0
- package/dist/tools/license-check-tool.js +43 -0
- package/dist/tools/lint-project-tool.d.ts +52 -0
- package/dist/tools/lint-project-tool.js +143 -0
- package/dist/tools/local-binary-launch.d.ts +14 -0
- package/dist/tools/local-binary-launch.js +63 -0
- package/dist/tools/lsp-navigation-tools.d.ts +87 -0
- package/dist/tools/lsp-navigation-tools.js +470 -0
- package/dist/tools/markdown-convert.d.ts +51 -0
- package/dist/tools/markdown-convert.js +159 -0
- package/dist/tools/media-generation-tool.d.ts +90 -1
- package/dist/tools/media-generation-tool.js +1553 -11
- package/dist/tools/meeting-notes-tool.d.ts +25 -0
- package/dist/tools/meeting-notes-tool.js +240 -0
- package/dist/tools/merge-conflict-tool.js +1 -1
- package/dist/tools/metadata.d.ts +7 -1
- package/dist/tools/metadata.js +823 -13
- package/dist/tools/mixture-of-agents-tool.d.ts +7 -0
- package/dist/tools/mixture-of-agents-tool.js +230 -31
- package/dist/tools/multi-edit.js +42 -18
- package/dist/tools/ocr-tool.d.ts +43 -0
- package/dist/tools/ocr-tool.js +24 -18
- package/dist/tools/omission-placeholder-detector.js +12 -6
- package/dist/tools/paper-qa-tool.d.ts +73 -0
- package/dist/tools/paper-qa-tool.js +313 -0
- package/dist/tools/peer-chain-tool.js +4 -1
- package/dist/tools/peer-delegate-tool.d.ts +2 -0
- package/dist/tools/peer-delegate-tool.js +11 -1
- package/dist/tools/port-check-tool.d.ts +31 -0
- package/dist/tools/port-check-tool.js +25 -0
- package/dist/tools/project-map-tool.d.ts +39 -0
- package/dist/tools/project-map-tool.js +128 -0
- package/dist/tools/register-tool-handler.d.ts +2 -2
- package/dist/tools/register-tool-handler.js +7 -4
- package/dist/tools/registry/advanced-tools.js +1 -1
- package/dist/tools/registry/attention-tools.d.ts +3 -3
- package/dist/tools/registry/attention-tools.js +14 -10
- package/dist/tools/registry/authored-extra-tools.d.ts +20 -0
- package/dist/tools/registry/authored-extra-tools.js +147 -0
- package/dist/tools/registry/bash-tools.d.ts +2 -2
- package/dist/tools/registry/bash-tools.js +7 -3
- package/dist/tools/registry/browser-operator-tools.d.ts +1 -1
- package/dist/tools/registry/browser-operator-tools.js +17 -3
- package/dist/tools/registry/browser-tools.d.ts +1 -1
- package/dist/tools/registry/browser-tools.js +38 -33
- package/dist/tools/registry/code-explorer-tools.js +6 -1
- package/dist/tools/registry/comfy-recipe-tools.d.ts +2 -0
- package/dist/tools/registry/comfy-recipe-tools.js +5 -0
- package/dist/tools/registry/context-expand-tools.d.ts +3 -0
- package/dist/tools/registry/context-expand-tools.js +6 -0
- package/dist/tools/registry/csv-tools.d.ts +12 -0
- package/dist/tools/registry/csv-tools.js +62 -0
- package/dist/tools/registry/delegate-agent-tools.d.ts +64 -0
- package/dist/tools/registry/delegate-agent-tools.js +206 -0
- package/dist/tools/registry/design-tools.d.ts +12 -0
- package/dist/tools/registry/design-tools.js +80 -0
- package/dist/tools/registry/fleet-tools.js +6 -0
- package/dist/tools/registry/index.d.ts +25 -5
- package/dist/tools/registry/index.js +80 -6
- package/dist/tools/registry/interactive-adapters.d.ts +37 -0
- package/dist/tools/registry/interactive-adapters.js +116 -0
- package/dist/tools/registry/lsp-tools.d.ts +3 -3
- package/dist/tools/registry/lsp-tools.js +9 -3
- package/dist/tools/registry/meeting-tools.d.ts +3 -0
- package/dist/tools/registry/meeting-tools.js +6 -0
- package/dist/tools/registry/memory-tools.js +32 -20
- package/dist/tools/registry/misc-tools.d.ts +1 -1
- package/dist/tools/registry/misc-tools.js +7 -4
- package/dist/tools/registry/moa-tools.js +13 -0
- package/dist/tools/registry/multimodal-tools.d.ts +73 -1
- package/dist/tools/registry/multimodal-tools.js +581 -4
- package/dist/tools/registry/process-tools.d.ts +16 -0
- package/dist/tools/registry/process-tools.js +136 -1
- package/dist/tools/registry/remind-tools.d.ts +23 -0
- package/dist/tools/registry/remind-tools.js +120 -0
- package/dist/tools/registry/research-tools.d.ts +17 -0
- package/dist/tools/registry/research-tools.js +21 -0
- package/dist/tools/registry/search-tools.d.ts +6 -6
- package/dist/tools/registry/search-tools.js +30 -17
- package/dist/tools/registry/secrets-tools.d.ts +33 -0
- package/dist/tools/registry/secrets-tools.js +71 -0
- package/dist/tools/registry/self-describe-tools.d.ts +23 -0
- package/dist/tools/registry/self-describe-tools.js +156 -0
- package/dist/tools/registry/self-evolution-tools.d.ts +13 -0
- package/dist/tools/registry/self-evolution-tools.js +97 -0
- package/dist/tools/registry/text-editor-tools.d.ts +26 -4
- package/dist/tools/registry/text-editor-tools.js +102 -6
- package/dist/tools/registry/tool-alias-map.d.ts +8 -0
- package/dist/tools/registry/tool-alias-map.js +47 -0
- package/dist/tools/registry/tool-aliases.d.ts +3 -4
- package/dist/tools/registry/tool-aliases.js +12 -48
- package/dist/tools/registry/types.d.ts +5 -0
- package/dist/tools/registry/verify-tools.d.ts +50 -0
- package/dist/tools/registry/verify-tools.js +136 -0
- package/dist/tools/registry/video-studio-tools.d.ts +13 -0
- package/dist/tools/registry/video-studio-tools.js +21 -0
- package/dist/tools/registry/vision-tools.d.ts +15 -1
- package/dist/tools/registry/vision-tools.js +252 -27
- package/dist/tools/registry/web-test-tool.d.ts +66 -0
- package/dist/tools/registry/web-test-tool.js +321 -0
- package/dist/tools/registry/web-tools.d.ts +43 -0
- package/dist/tools/registry/web-tools.js +281 -1
- package/dist/tools/review-gate-helper.d.ts +44 -0
- package/dist/tools/review-gate-helper.js +110 -0
- package/dist/tools/route-peer-tool.js +62 -11
- package/dist/tools/sbom-generate-tool.d.ts +22 -0
- package/dist/tools/sbom-generate-tool.js +41 -0
- package/dist/tools/scaffold-app-tool.d.ts +48 -0
- package/dist/tools/scaffold-app-tool.js +144 -0
- package/dist/tools/screenshot-tool.js +5 -2
- package/dist/tools/search.d.ts +17 -0
- package/dist/tools/search.js +31 -8
- package/dist/tools/self-describe.d.ts +73 -0
- package/dist/tools/self-describe.js +298 -0
- package/dist/tools/stock-quote.d.ts +67 -0
- package/dist/tools/stock-quote.js +645 -0
- package/dist/tools/submit-plan-tool.js +1 -1
- package/dist/tools/test-runner-tool.d.ts +38 -0
- package/dist/tools/test-runner-tool.js +96 -0
- package/dist/tools/text-editor.js +91 -8
- package/dist/tools/text-to-speech-tool.d.ts +1 -1
- package/dist/tools/text-to-speech-tool.js +48 -2
- package/dist/tools/todo-scan-tool.d.ts +39 -0
- package/dist/tools/todo-scan-tool.js +53 -0
- package/dist/tools/tool-manager.js +8 -0
- package/dist/tools/tool-search.js +11 -3
- package/dist/tools/tool-selector.d.ts +7 -0
- package/dist/tools/tool-selector.js +27 -13
- package/dist/tools/tools-md-generator.js +2 -2
- package/dist/tools/types.d.ts +16 -0
- package/dist/tools/types.js +5 -1
- package/dist/tools/video/approved-media-source.d.ts +8 -0
- package/dist/tools/video/approved-media-source.js +59 -0
- package/dist/tools/video/book-manuscript-source.d.ts +49 -0
- package/dist/tools/video/book-manuscript-source.js +263 -0
- package/dist/tools/video/character-in-location.d.ts +42 -0
- package/dist/tools/video/character-in-location.js +161 -0
- package/dist/tools/video/cinematic-trailer-plan.d.ts +203 -0
- package/dist/tools/video/cinematic-trailer-plan.js +498 -0
- package/dist/tools/video/cloud-understand.d.ts +91 -0
- package/dist/tools/video/cloud-understand.js +214 -0
- package/dist/tools/video/comfy-client.d.ts +38 -0
- package/dist/tools/video/comfy-client.js +227 -0
- package/dist/tools/video/comfy-workflow-template.d.ts +70 -0
- package/dist/tools/video/comfy-workflow-template.js +194 -0
- package/dist/tools/video/describe-frame.d.ts +36 -0
- package/dist/tools/video/describe-frame.js +68 -0
- package/dist/tools/video/film-assemble.d.ts +234 -0
- package/dist/tools/video/film-assemble.js +959 -0
- package/dist/tools/video/film-project.d.ts +152 -0
- package/dist/tools/video/film-project.js +312 -0
- package/dist/tools/video/frame-dedup.d.ts +46 -0
- package/dist/tools/video/frame-dedup.js +110 -0
- package/dist/tools/video/frame-sample.d.ts +72 -0
- package/dist/tools/video/frame-sample.js +238 -0
- package/dist/tools/video/google-flow-driver.d.ts +131 -0
- package/dist/tools/video/google-flow-driver.js +312 -0
- package/dist/tools/video/google-flow-handoff.d.ts +75 -0
- package/dist/tools/video/google-flow-handoff.js +121 -0
- package/dist/tools/video/google-flow-plan-export.d.ts +21 -0
- package/dist/tools/video/google-flow-plan-export.js +123 -0
- package/dist/tools/video/google-flow-result-import.d.ts +92 -0
- package/dist/tools/video/google-flow-result-import.js +297 -0
- package/dist/tools/video/hybrid-video-router.d.ts +34 -0
- package/dist/tools/video/hybrid-video-router.js +113 -0
- package/dist/tools/video/localized-media.d.ts +50 -0
- package/dist/tools/video/localized-media.js +128 -0
- package/dist/tools/video/long-form-plan.d.ts +37 -0
- package/dist/tools/video/long-form-plan.js +83 -0
- package/dist/tools/video/long-form-production.d.ts +49 -0
- package/dist/tools/video/long-form-production.js +168 -0
- package/dist/tools/video/long-transcribe.d.ts +94 -0
- package/dist/tools/video/long-transcribe.js +206 -0
- package/dist/tools/video/media-fetch.d.ts +94 -0
- package/dist/tools/video/media-fetch.js +311 -0
- package/dist/tools/video/mermaid-render.d.ts +34 -0
- package/dist/tools/video/mermaid-render.js +127 -0
- package/dist/tools/video/narration.d.ts +96 -0
- package/dist/tools/video/narration.js +505 -0
- package/dist/tools/video/native-fashion-defects.d.ts +27 -0
- package/dist/tools/video/native-fashion-defects.js +80 -0
- package/dist/tools/video/scene-render.d.ts +108 -0
- package/dist/tools/video/scene-render.js +418 -0
- package/dist/tools/video/subtitles.d.ts +49 -0
- package/dist/tools/video/subtitles.js +116 -0
- package/dist/tools/video/video-ckg.d.ts +123 -0
- package/dist/tools/video/video-ckg.js +188 -0
- package/dist/tools/video/video-research-card.d.ts +58 -0
- package/dist/tools/video/video-research-card.js +468 -0
- package/dist/tools/video/video-understanding.d.ts +159 -0
- package/dist/tools/video/video-understanding.js +534 -0
- package/dist/tools/video/visual-gate-report.d.ts +139 -0
- package/dist/tools/video/visual-gate-report.js +379 -0
- package/dist/tools/video/voice-rights-registry.d.ts +13 -0
- package/dist/tools/video/voice-rights-registry.js +142 -0
- package/dist/tools/video/youtube-captions.d.ts +51 -0
- package/dist/tools/video/youtube-captions.js +102 -0
- package/dist/tools/video/youtube-master-quality.d.ts +163 -0
- package/dist/tools/video/youtube-master-quality.js +379 -0
- package/dist/tools/video/youtube-storyboard.d.ts +43 -0
- package/dist/tools/video/youtube-storyboard.js +194 -0
- package/dist/tools/video-flow-handoff-tool.d.ts +181 -0
- package/dist/tools/video-flow-handoff-tool.js +261 -0
- package/dist/tools/video-long-form-plan-tool.d.ts +31 -0
- package/dist/tools/video-long-form-plan-tool.js +87 -0
- package/dist/tools/video-quality-gate-tool.d.ts +97 -0
- package/dist/tools/video-quality-gate-tool.js +189 -0
- package/dist/tools/video-route-tool.d.ts +128 -0
- package/dist/tools/video-route-tool.js +126 -0
- package/dist/tools/video-studio-tool-helpers.d.ts +22 -0
- package/dist/tools/video-studio-tool-helpers.js +128 -0
- package/dist/tools/video-trailer-plan-tool.d.ts +60 -0
- package/dist/tools/video-trailer-plan-tool.js +103 -0
- package/dist/tools/vision/blender-render.d.ts +76 -0
- package/dist/tools/vision/blender-render.js +121 -0
- package/dist/tools/vision/image-processor.d.ts +1 -0
- package/dist/tools/vision/image-processor.js +8 -1
- package/dist/tools/vision/load-sharp.d.ts +53 -0
- package/dist/tools/vision/load-sharp.js +48 -0
- package/dist/tools/vision/mean-luma.d.ts +4 -0
- package/dist/tools/vision/mean-luma.js +189 -0
- package/dist/tools/vision/object-detection.d.ts +90 -0
- package/dist/tools/vision/object-detection.js +403 -0
- package/dist/tools/vision/vision-analysis.js +2 -1
- package/dist/tools/weather.d.ts +33 -0
- package/dist/tools/weather.js +183 -0
- package/dist/tools/web-scrape-tool.d.ts +63 -0
- package/dist/tools/web-scrape-tool.js +318 -0
- package/dist/tools/web-search.d.ts +67 -6
- package/dist/tools/web-search.js +231 -68
- package/dist/tools/workspace-tools.d.ts +39 -0
- package/dist/tools/workspace-tools.js +347 -0
- package/dist/tracks/track-manager.js +9 -8
- package/dist/triggers/webhook-trigger.js +10 -18
- package/dist/ui/components/ChatHistory.js +12 -0
- package/dist/ui/components/ChatInterface.js +6 -2
- package/dist/ui/components/FileAutocomplete.d.ts +6 -1
- package/dist/ui/components/FileAutocomplete.js +136 -76
- package/dist/ui/components/FuzzyPicker.d.ts +5 -0
- package/dist/ui/components/FuzzyPicker.js +1 -1
- package/dist/ui/http-server/server.js +2 -2
- package/dist/undo/checkpoint-manager.d.ts +3 -0
- package/dist/undo/checkpoint-manager.js +6 -1
- package/dist/utils/approval-pattern-tracker.js +5 -13
- package/dist/utils/ascii-banner.d.ts +1 -1
- package/dist/utils/ascii-banner.js +1 -1
- package/dist/utils/atomic-write.d.ts +77 -0
- package/dist/utils/atomic-write.js +530 -0
- package/dist/utils/audio-player.d.ts +1 -0
- package/dist/utils/audio-player.js +7 -3
- package/dist/utils/autonomy-manager.js +12 -3
- package/dist/utils/command-exists.d.ts +14 -0
- package/dist/utils/command-exists.js +43 -0
- package/dist/utils/config-validation/schema.d.ts +13 -13
- package/dist/utils/config-validation/schema.js +33 -1
- package/dist/utils/confirmation-service.d.ts +60 -1
- package/dist/utils/confirmation-service.js +204 -41
- package/dist/utils/cost-tracker.d.ts +66 -1
- package/dist/utils/cost-tracker.js +100 -17
- package/dist/utils/history-manager.js +6 -7
- package/dist/utils/init-project.js +17 -13
- package/dist/utils/input-validation/index.d.ts +16 -16
- package/dist/utils/installation-id.js +7 -22
- package/dist/utils/interactive-setup.js +7 -12
- package/dist/utils/json-salvage.d.ts +22 -0
- package/dist/utils/json-salvage.js +91 -0
- package/dist/utils/llm-retry.js +3 -0
- package/dist/utils/logger.d.ts +1 -0
- package/dist/utils/logger.js +16 -5
- package/dist/utils/model-router.js +8 -12
- package/dist/utils/model-utils.js +14 -0
- package/dist/utils/output-sanitizer.d.ts +4 -8
- package/dist/utils/output-sanitizer.js +10 -10
- package/dist/utils/output-schema-validator.d.ts +7 -0
- package/dist/utils/output-schema-validator.js +25 -1
- package/dist/utils/response-cache.js +11 -22
- package/dist/utils/ripgrep-path.d.ts +11 -0
- package/dist/utils/ripgrep-path.js +60 -0
- package/dist/utils/semantic-cache.js +4 -7
- package/dist/utils/settings-manager.d.ts +27 -4
- package/dist/utils/settings-manager.js +51 -39
- package/dist/utils/shell-completions.js +25 -34
- package/dist/utils/shell-configuration.d.ts +39 -0
- package/dist/utils/shell-configuration.js +106 -0
- package/dist/utils/stream-stall-guard.d.ts +39 -0
- package/dist/utils/stream-stall-guard.js +123 -0
- package/dist/utils/subprocess-env.d.ts +15 -0
- package/dist/utils/subprocess-env.js +112 -0
- package/dist/utils/telegram-api-base.d.ts +9 -0
- package/dist/utils/telegram-api-base.js +15 -0
- package/dist/utils/telemetry-config.js +16 -13
- package/dist/utils/update-notifier.js +5 -10
- package/dist/utils/validators.js +1 -2
- package/dist/versioning/config-migrator.js +3 -2
- package/dist/versioning/migration-manager.js +5 -4
- package/dist/versioning/version-detector.js +2 -1
- package/dist/vision-train/assets.d.ts +37 -0
- package/dist/vision-train/assets.js +97 -0
- package/dist/vision-train/ckg-publish.d.ts +26 -0
- package/dist/vision-train/ckg-publish.js +36 -0
- package/dist/vision-train/coco-to-labels.d.ts +63 -0
- package/dist/vision-train/coco-to-labels.js +63 -0
- package/dist/vision-train/curriculum.d.ts +29 -0
- package/dist/vision-train/curriculum.js +49 -0
- package/dist/vision-train/engine.d.ts +56 -0
- package/dist/vision-train/engine.js +55 -0
- package/dist/vision-train/report.d.ts +15 -0
- package/dist/vision-train/report.js +72 -0
- package/dist/vision-train/scorer.d.ts +67 -0
- package/dist/vision-train/scorer.js +114 -0
- package/dist/vision-train/yolo-labels.d.ts +13 -0
- package/dist/vision-train/yolo-labels.js +66 -0
- package/dist/voice/elevenlabs-voice.d.ts +53 -0
- package/dist/voice/elevenlabs-voice.js +412 -0
- package/dist/voice/kyutai-local-voice.d.ts +33 -0
- package/dist/voice/kyutai-local-voice.js +198 -0
- package/dist/voice/local-tts.d.ts +106 -3
- package/dist/voice/local-tts.js +672 -35
- package/dist/voice/local-whisper.d.ts +11 -0
- package/dist/voice/local-whisper.js +18 -2
- package/dist/voice/pcm-edges.d.ts +43 -0
- package/dist/voice/pcm-edges.js +210 -0
- package/dist/voice/perceived-latency-benchmark.d.ts +56 -0
- package/dist/voice/perceived-latency-benchmark.js +218 -0
- package/dist/voice/tts-bank.d.ts +72 -0
- package/dist/voice/tts-bank.js +172 -0
- package/dist/voice/tts-latency-benchmark.d.ts +36 -0
- package/dist/voice/tts-latency-benchmark.js +76 -0
- package/dist/voice/tts-volume.d.ts +86 -0
- package/dist/voice/tts-volume.js +378 -0
- package/dist/voice/two-speed-voice.d.ts +19 -0
- package/dist/voice/two-speed-voice.js +40 -0
- package/dist/voice/voice-to-code.js +1 -1
- package/dist/voice/voicebox-tts.d.ts +150 -0
- package/dist/voice/voicebox-tts.js +669 -0
- package/dist/voice/wake-word.d.ts +2 -0
- package/dist/voice/wake-word.js +30 -10
- package/dist/webhooks/webhook-manager.js +10 -18
- package/dist/widgets/auto-widget.d.ts +27 -0
- package/dist/widgets/auto-widget.js +111 -0
- package/dist/widgets/canvas-publish.d.ts +15 -0
- package/dist/widgets/canvas-publish.js +39 -0
- package/dist/widgets/curated/news.d.ts +2 -0
- package/dist/widgets/curated/news.js +46 -0
- package/dist/widgets/curated/stock.d.ts +1 -0
- package/dist/widgets/curated/stock.js +87 -0
- package/dist/widgets/curated/weather.d.ts +2 -0
- package/dist/widgets/curated/weather.js +93 -0
- package/dist/widgets/template-engine.d.ts +19 -0
- package/dist/widgets/template-engine.js +134 -0
- package/dist/widgets/widget-engine.d.ts +24 -0
- package/dist/widgets/widget-engine.js +153 -0
- package/dist/widgets/widget-gate.d.ts +19 -0
- package/dist/widgets/widget-gate.js +144 -0
- package/dist/widgets/widget-image-renderer.d.ts +4 -0
- package/dist/widgets/widget-image-renderer.js +164 -0
- package/dist/widgets/widget-matcher.d.ts +48 -0
- package/dist/widgets/widget-matcher.js +109 -0
- package/dist/widgets/widget-proposer.d.ts +17 -0
- package/dist/widgets/widget-proposer.js +75 -0
- package/dist/widgets/widget-registry.d.ts +36 -0
- package/dist/widgets/widget-registry.js +231 -0
- package/dist/widgets/widget-types.d.ts +100 -0
- package/dist/widgets/widget-types.js +17 -0
- package/dist/wizard/environment-detection.d.ts +67 -0
- package/dist/wizard/environment-detection.js +210 -0
- package/dist/wizard/onboarding.d.ts +21 -1
- package/dist/wizard/onboarding.js +202 -27
- package/dist/workflows/state-manager.js +8 -2
- package/dist/workspace/workspace-config.d.ts +37 -0
- package/dist/workspace/workspace-config.js +199 -0
- package/dist/workspace/workspace-isolation.d.ts +19 -0
- package/dist/workspace/workspace-isolation.js +124 -39
- package/dist/workspace/workspace-manager.js +19 -14
- package/examples/claude_desktop_config.json +8 -0
- package/package.json +75 -31
- package/.codebuddy/README.md +0 -65
- package/dist/agent/middleware/learning-first-middleware.d.ts +0 -79
- package/dist/agent/middleware/learning-first-middleware.js +0 -271
- package/dist/agent/middleware/tool-filter-middleware.d.ts +0 -45
- package/dist/agent/middleware/tool-filter-middleware.js +0 -122
- package/dist/codebuddy/tool-definitions/batch-tools.d.ts +0 -10
- package/dist/codebuddy/tool-definitions/batch-tools.js +0 -46
- package/dist/codebuddy/tool-definitions/graph-tools.d.ts +0 -13
- package/dist/codebuddy/tool-definitions/graph-tools.js +0 -78
- package/dist/plugins/code-explorer/CodeExplorerMCPClient.d.ts +0 -129
- package/dist/plugins/code-explorer/CodeExplorerMCPClient.js +0 -143
- package/dist/tools/registry/batch-tools.d.ts +0 -36
- package/dist/tools/registry/batch-tools.js +0 -135
- package/dist/tools/registry/graph-tools.d.ts +0 -45
- package/dist/tools/registry/graph-tools.js +0 -283
|
@@ -1,82 +1,2188 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Speech reaction — closes the perception→cognition loop.
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
2
|
+
* Speech reaction — closes the perception→cognition loop. Two input paths feed the
|
|
3
|
+
* SAME cognition (respond gate → `hearing` percept → optional `onHeard` action):
|
|
4
|
+
* - batch: a `speech_end` event carrying the source WAV (the daemon tags it) →
|
|
5
|
+
* transcribe the utterance here (STT);
|
|
6
|
+
* - live: an `audio/transcript_final` event from buddy-sense's `live-audio` sense
|
|
7
|
+
* whose payload ALREADY carries the decoded text → no WAV, no STT on this side.
|
|
8
|
+
* DEBOUNCED (one transcription per utterance — the energy VAD over-segments), opt-in
|
|
9
|
+
* (`CODEBUDDY_SENSORY_SPEECH=true`), injectable transcriber, never-throws.
|
|
10
|
+
* Processed fallback WAVs are ephemeral and removed after the job settles. Set
|
|
11
|
+
* `CODEBUDDY_SENSORY_KEEP_WAV=true` to retain them for audio/STT debugging.
|
|
8
12
|
*
|
|
9
13
|
* @module sensory/speech-reaction
|
|
10
14
|
*/
|
|
15
|
+
import { existsSync, readFileSync } from 'fs';
|
|
16
|
+
import { homedir } from 'os';
|
|
17
|
+
import { fileURLToPath } from 'url';
|
|
18
|
+
import { basename, delimiter, dirname, join, resolve } from 'path';
|
|
11
19
|
import { getGlobalEventBus } from '../events/event-bus.js';
|
|
12
20
|
import { logger } from '../utils/logger.js';
|
|
21
|
+
import { classifyRecentVoiceEcho, hasRecentSpokenReference, isRecentVoiceFragmentEcho, isSensoryAecTrusted, isSpeaking, measureVoiceResumeTiming, } from './voice-activity.js';
|
|
13
22
|
import { perceptionOf } from './reactions.js';
|
|
14
|
-
|
|
15
|
-
|
|
23
|
+
import { resolveSpeechRecognitionEngine, resolveSpeechLanguage, resolveSpeechTranscriptionPlan, resolveParakeetModelDir, isFrenchParakeetModelAvailable, resolveSpeechSttThreads, expandSpeechPath, } from './speech-engine-config.js';
|
|
24
|
+
import { resolveUserName } from '../companion/user-name.js';
|
|
25
|
+
import { isLikelyIncompleteVoiceTurn, joinVoiceTurnFragments, resolveConversationalTurnEndSilenceMs, resolveIncompleteTurnHoldMs, } from './voice-turn-taking.js';
|
|
26
|
+
import { getVoiceTurnCoordinator } from './voice-turn-coordinator.js';
|
|
27
|
+
import { assessAudioScene } from './audio-scene.js';
|
|
28
|
+
import { createConversationCueController, shouldRepairTranscript, } from './conversation-cues.js';
|
|
29
|
+
import { resolveTurnDetectorDecision, } from './turn-detector.js';
|
|
30
|
+
// Re-exported for back-compat: callers + tests import these from speech-reaction.
|
|
31
|
+
export { resolveSpeechRecognitionEngine };
|
|
32
|
+
/**
|
|
33
|
+
* Safe half-duplex barge-in gate. Requiring the assistant name or an explicit
|
|
34
|
+
* stop phrase avoids treating its own loudspeaker echo as a human interruption.
|
|
35
|
+
*/
|
|
36
|
+
export function isBargeInTranscript(text, robotName = process.env.CODEBUDDY_ROBOT_NAME || 'Lisa') {
|
|
37
|
+
const normalized = text
|
|
38
|
+
.toLowerCase()
|
|
39
|
+
.normalize('NFD')
|
|
40
|
+
.replace(/\p{M}+/gu, '')
|
|
41
|
+
.replace(/[^\p{L}\p{N}]+/gu, ' ')
|
|
42
|
+
.replace(/\s+/g, ' ')
|
|
43
|
+
.trim();
|
|
44
|
+
if (!normalized)
|
|
45
|
+
return false;
|
|
46
|
+
const name = robotName
|
|
47
|
+
.toLowerCase()
|
|
48
|
+
.normalize('NFD')
|
|
49
|
+
.replace(/\p{M}+/gu, '')
|
|
50
|
+
.replace(/[^\p{L}\p{N}]+/gu, ' ')
|
|
51
|
+
.trim();
|
|
52
|
+
if (name && new RegExp(`(^|\\s)${name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(\\s|$)`).test(normalized)) {
|
|
53
|
+
return true;
|
|
54
|
+
}
|
|
55
|
+
return /^(stop|arrete|tais toi|attends|une seconde|laisse moi parler)(\s|$)/.test(normalized);
|
|
56
|
+
}
|
|
57
|
+
export const DEFAULT_VOICE_BARGEIN_MIN_MS = 500;
|
|
58
|
+
export const DEFAULT_VOICE_BARGEIN_MIN_SPEECH_MS = 250;
|
|
59
|
+
export const DEFAULT_VOICE_BARGEIN_MARGIN_DB = 6;
|
|
60
|
+
export const VOICE_BARGEIN_LEAKAGE_REFERENCE_MS = 300;
|
|
61
|
+
export function voiceBargeInEnabled(env = process.env) {
|
|
62
|
+
return env.CODEBUDDY_SENSORY_BARGE_IN?.trim().toLowerCase() === 'true';
|
|
63
|
+
}
|
|
64
|
+
function assessAcousticBargeInEligibility(payload, env) {
|
|
65
|
+
return {
|
|
66
|
+
trustedAec: isSensoryAecTrusted(payload.aecActive === true, env),
|
|
67
|
+
sustainedSpeech: (capturedSpeechMs(payload) ?? 0) >= DEFAULT_VOICE_BARGEIN_MIN_SPEECH_MS,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Speech-start has no transcript yet. It may cut only when active capture-side
|
|
72
|
+
* AEC accompanies both sustained speech and a calibrated energy margin.
|
|
73
|
+
*/
|
|
74
|
+
export function shouldTriggerVoiceBargeInOnSpeechStart(payload, env = process.env) {
|
|
75
|
+
const eligibility = assessAcousticBargeInEligibility(payload, env);
|
|
76
|
+
if (!eligibility.trustedAec || !eligibility.sustainedSpeech) {
|
|
77
|
+
return false;
|
|
78
|
+
}
|
|
79
|
+
const rms = finiteTimestamp(payload.rms);
|
|
80
|
+
const leakageRms = finiteTimestamp(payload.noiseFloorRms);
|
|
81
|
+
return rms !== undefined
|
|
82
|
+
&& leakageRms !== undefined
|
|
83
|
+
&& exceedsVoiceLeakageMargin(rms, leakageRms, resolveVoiceBargeInMarginDb(env));
|
|
84
|
+
}
|
|
85
|
+
export function resolveVoiceBargeInMarginDb(env = process.env) {
|
|
86
|
+
const configured = Number(env.CODEBUDDY_SENSORY_BARGE_IN_MARGIN_DB);
|
|
87
|
+
if (!Number.isFinite(configured) || configured < 0)
|
|
88
|
+
return DEFAULT_VOICE_BARGEIN_MARGIN_DB;
|
|
89
|
+
return Math.min(60, configured);
|
|
90
|
+
}
|
|
91
|
+
/** True when the current microphone energy clears the adaptive leakage margin. */
|
|
92
|
+
export function exceedsVoiceLeakageMargin(rms, leakageRms, marginDb = DEFAULT_VOICE_BARGEIN_MARGIN_DB) {
|
|
93
|
+
if (!Number.isFinite(rms) || rms <= 0 || !Number.isFinite(leakageRms) || leakageRms <= 0) {
|
|
94
|
+
return false;
|
|
95
|
+
}
|
|
96
|
+
return 20 * Math.log10(rms / leakageRms) >= marginDb;
|
|
97
|
+
}
|
|
98
|
+
export function resolveVoiceBargeInMinMs(env = process.env) {
|
|
99
|
+
const configured = Number(env.CODEBUDDY_VOICE_BARGEIN_MIN_MS);
|
|
100
|
+
if (!Number.isFinite(configured) || configured < 0)
|
|
101
|
+
return DEFAULT_VOICE_BARGEIN_MIN_MS;
|
|
102
|
+
return Math.min(10_000, Math.floor(configured));
|
|
103
|
+
}
|
|
104
|
+
function capturedSpeechMs(payload) {
|
|
105
|
+
const durations = [
|
|
106
|
+
finiteTimestamp(payload.audioMs),
|
|
107
|
+
finiteTimestamp(payload.ms),
|
|
108
|
+
finiteTimestamp(payload.durationMs),
|
|
109
|
+
].filter((value) => value !== undefined && value >= 0);
|
|
110
|
+
const startedAtMs = finiteTimestamp(payload.startedAtMs);
|
|
111
|
+
const endedAtMs = finiteTimestamp(payload.endedAtMs);
|
|
112
|
+
if (startedAtMs !== undefined && endedAtMs !== undefined && endedAtMs >= startedAtMs) {
|
|
113
|
+
durations.push(endedAtMs - startedAtMs);
|
|
114
|
+
}
|
|
115
|
+
return durations.length > 0 ? Math.max(...durations) : undefined;
|
|
116
|
+
}
|
|
117
|
+
const MIN_STT_RECOVERY_AUDIO_MS = 200;
|
|
118
|
+
const DEFAULT_SPEECH_NOISE_RMS = 0.02;
|
|
119
|
+
const STT_FAILURE_REPLY = "Pardon, je n'ai pas compris.";
|
|
120
|
+
/** Read only the small PCM signal facts needed to avoid speaking on a fake/empty WAV. */
|
|
121
|
+
function readPcm16WavSignal(wav) {
|
|
122
|
+
try {
|
|
123
|
+
const source = readFileSync(expandSpeechPath(wav));
|
|
124
|
+
if (source.length < 12
|
|
125
|
+
|| source.toString('ascii', 0, 4) !== 'RIFF'
|
|
126
|
+
|| source.toString('ascii', 8, 12) !== 'WAVE')
|
|
127
|
+
return undefined;
|
|
128
|
+
let channels = 0;
|
|
129
|
+
let sampleRate = 0;
|
|
130
|
+
let bitsPerSample = 0;
|
|
131
|
+
let dataOffset = -1;
|
|
132
|
+
let dataSize = 0;
|
|
133
|
+
for (let offset = 12; offset + 8 <= source.length;) {
|
|
134
|
+
const chunkSize = source.readUInt32LE(offset + 4);
|
|
135
|
+
const chunkStart = offset + 8;
|
|
136
|
+
const chunkEnd = Math.min(source.length, chunkStart + chunkSize);
|
|
137
|
+
const chunk = source.toString('ascii', offset, offset + 4);
|
|
138
|
+
if (chunk === 'fmt ' && chunkEnd - chunkStart >= 16) {
|
|
139
|
+
const format = source.readUInt16LE(chunkStart);
|
|
140
|
+
channels = source.readUInt16LE(chunkStart + 2);
|
|
141
|
+
sampleRate = source.readUInt32LE(chunkStart + 4);
|
|
142
|
+
bitsPerSample = source.readUInt16LE(chunkStart + 14);
|
|
143
|
+
if (format !== 1)
|
|
144
|
+
return undefined;
|
|
145
|
+
}
|
|
146
|
+
else if (chunk === 'data') {
|
|
147
|
+
dataOffset = chunkStart;
|
|
148
|
+
dataSize = chunkEnd - chunkStart;
|
|
149
|
+
}
|
|
150
|
+
const nextOffset = chunkStart + chunkSize + (chunkSize % 2);
|
|
151
|
+
if (nextOffset <= offset)
|
|
152
|
+
break;
|
|
153
|
+
offset = nextOffset;
|
|
154
|
+
}
|
|
155
|
+
if (channels < 1 || sampleRate < 1 || bitsPerSample !== 16 || dataOffset < 0 || dataSize < 2) {
|
|
156
|
+
return undefined;
|
|
157
|
+
}
|
|
158
|
+
const sampleCount = Math.floor(dataSize / 2);
|
|
159
|
+
let squareSum = 0;
|
|
160
|
+
for (let index = 0; index < sampleCount; index += 1) {
|
|
161
|
+
const sample = source.readInt16LE(dataOffset + index * 2) / 32_768;
|
|
162
|
+
squareSum += sample * sample;
|
|
163
|
+
}
|
|
164
|
+
return {
|
|
165
|
+
durationMs: dataSize / (sampleRate * channels * 2) * 1_000,
|
|
166
|
+
rms: Math.sqrt(squareSum / sampleCount),
|
|
167
|
+
channels,
|
|
168
|
+
sampleRate,
|
|
169
|
+
bitsPerSample,
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
catch {
|
|
173
|
+
return undefined;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
function describeSherpaRustAttempt(wav, startedAtMs, worker, reason) {
|
|
177
|
+
const signal = readPcm16WavSignal(wav);
|
|
178
|
+
const exitCode = worker?.proc.exitCode;
|
|
179
|
+
const stderr = worker?.stderrTail?.trim() || '<empty>';
|
|
180
|
+
return [
|
|
181
|
+
`reason=${reason}`,
|
|
182
|
+
`exit_code=${exitCode === null || exitCode === undefined ? 'running' : exitCode}`,
|
|
183
|
+
`stderr=${JSON.stringify(stderr.slice(-300))}`,
|
|
184
|
+
`request_ms=${Math.max(0, Date.now() - startedAtMs)}`,
|
|
185
|
+
`audio_ms=${signal ? Math.round(signal.durationMs) : 'unknown'}`,
|
|
186
|
+
`format=${signal ? `pcm_s16le/${signal.sampleRate}Hz/${signal.channels}ch` : 'unknown'}`,
|
|
187
|
+
`rms=${signal ? signal.rms.toFixed(6) : 'unknown'}`,
|
|
188
|
+
].join(' ');
|
|
189
|
+
}
|
|
190
|
+
function resolveSherpaEmptyThreshold() {
|
|
191
|
+
return Math.max(1, Math.round(numericEnv('CODEBUDDY_SHERPA_EMPTY_THRESHOLD', DEFAULT_SHERPA_EMPTY_THRESHOLD)));
|
|
192
|
+
}
|
|
193
|
+
function recordSherpaRustSuccess() {
|
|
194
|
+
if (sherpaRustInactiveAnnounced) {
|
|
195
|
+
logger.info(`[speech] sherpa-rs actif à nouveau après ${sherpaRustEmptyStreak} transcript(s) vide(s)`);
|
|
196
|
+
}
|
|
197
|
+
sherpaRustEmptyStreak = 0;
|
|
198
|
+
sherpaRustInactiveAnnounced = false;
|
|
199
|
+
}
|
|
200
|
+
function recordSherpaRustEmpty(details, fallbackEnabled) {
|
|
201
|
+
sherpaRustEmptyStreak += 1;
|
|
202
|
+
const threshold = resolveSherpaEmptyThreshold();
|
|
203
|
+
if (sherpaRustEmptyStreak === 1) {
|
|
204
|
+
logger.warn(`[speech] sherpa-rs empty transcript; ${details} fallback=${fallbackEnabled ? 'faster-whisper' : 'disabled'}`);
|
|
205
|
+
}
|
|
206
|
+
if (sherpaRustEmptyStreak >= threshold && !sherpaRustInactiveAnnounced) {
|
|
207
|
+
sherpaRustInactiveAnnounced = true;
|
|
208
|
+
logger.warn(`[speech] sherpa-rs inactif : ${details} consecutive_empty=${sherpaRustEmptyStreak}`);
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
function realSpeechCapture(wav, payload) {
|
|
212
|
+
if (!existsSync(expandSpeechPath(wav)))
|
|
213
|
+
return undefined;
|
|
214
|
+
const measured = readPcm16WavSignal(wav);
|
|
215
|
+
if (!measured)
|
|
216
|
+
return undefined;
|
|
217
|
+
const durationMs = measured.durationMs;
|
|
218
|
+
const rms = measured.rms;
|
|
219
|
+
const threshold = finiteTimestamp(payload.rmsOn) ?? DEFAULT_SPEECH_NOISE_RMS;
|
|
220
|
+
if (durationMs < MIN_STT_RECOVERY_AUDIO_MS || rms < threshold)
|
|
221
|
+
return undefined;
|
|
222
|
+
return { durationMs, rms };
|
|
223
|
+
}
|
|
224
|
+
/** Explicit wake/stop always works; explicitly trusted AEC permits sustained natural speech. */
|
|
225
|
+
export function shouldTriggerVoiceBargeIn(text, payload = {}, env = process.env) {
|
|
226
|
+
if (isBargeInTranscript(text))
|
|
227
|
+
return true;
|
|
228
|
+
if (!isSensoryAecTrusted(payload.aecActive === true, env))
|
|
229
|
+
return false;
|
|
230
|
+
return (capturedSpeechMs(payload) ?? 0) >= resolveVoiceBargeInMinMs(env);
|
|
231
|
+
}
|
|
232
|
+
/**
|
|
233
|
+
* Deduplicate repeated `speech_end` events without imposing a multi-second pause between
|
|
234
|
+
* human turns. Playback echo is already covered independently by `voice-activity`'s
|
|
235
|
+
* speaking guard + echo tail, so this only needs to absorb duplicate capture events.
|
|
236
|
+
*/
|
|
237
|
+
export const DEFAULT_SPEECH_DEBOUNCE_MS = 800;
|
|
238
|
+
/**
|
|
239
|
+
* Decide whether a transcript captured around loudspeaker playback is safe to
|
|
240
|
+
* treat as a human turn. During playback we deliberately fail closed: without
|
|
241
|
+
* acoustic echo cancellation, only an explicit barge-in (Lisa/stop/attends…)
|
|
242
|
+
* may pass. With AEC, sustained speech is trusted as human input even while
|
|
243
|
+
* playback is active. In the short acoustic tail, the existing similarity
|
|
244
|
+
* classifier remains authoritative.
|
|
245
|
+
*/
|
|
246
|
+
export function shouldSuppressPlaybackCapture(kind, classification, explicitBargeIn, aecActive = false) {
|
|
247
|
+
if (kind === 'during_playback') {
|
|
248
|
+
if (aecActive && explicitBargeIn)
|
|
249
|
+
return false;
|
|
250
|
+
return classification === 'echo' || !explicitBargeIn;
|
|
251
|
+
}
|
|
252
|
+
return classification !== 'distinct';
|
|
253
|
+
}
|
|
254
|
+
/** Debug kill switch: keep fallback utterance WAVs instead of deleting them. */
|
|
255
|
+
export const SPEECH_KEEP_WAV_ENV = 'CODEBUDDY_SENSORY_KEEP_WAV';
|
|
256
|
+
export function resolveSpeechDebounceMs(env = process.env) {
|
|
257
|
+
const raw = env.CODEBUDDY_SPEECH_DEBOUNCE_MS?.trim();
|
|
258
|
+
if (!raw)
|
|
259
|
+
return DEFAULT_SPEECH_DEBOUNCE_MS;
|
|
260
|
+
const value = Number(raw);
|
|
261
|
+
return Number.isFinite(value) && value >= 0 ? value : DEFAULT_SPEECH_DEBOUNCE_MS;
|
|
262
|
+
}
|
|
263
|
+
function companionAudioDirectories(env = process.env) {
|
|
264
|
+
const configured = [
|
|
265
|
+
env.BUDDY_EAR_WAV_DIR?.trim(),
|
|
266
|
+
env.CODEBUDDY_COMPANION_AUDIO_DIR?.trim(),
|
|
267
|
+
].filter((value) => Boolean(value));
|
|
268
|
+
return [...new Set([
|
|
269
|
+
resolve(join(homedir(), '.codebuddy', 'companion')),
|
|
270
|
+
...configured.map(value => resolve(expandSpeechPath(value))),
|
|
271
|
+
])];
|
|
272
|
+
}
|
|
273
|
+
/**
|
|
274
|
+
* Remove only producer-owned fallback audio. An event payload is untrusted: a
|
|
275
|
+
* path outside the configured companion directory, a symlink, or any filename
|
|
276
|
+
* other than `utt-<digits>.wav` is never removed.
|
|
277
|
+
*/
|
|
278
|
+
async function removeProcessedCompanionWav(wav) {
|
|
279
|
+
if (truthyEnv(SPEECH_KEEP_WAV_ENV, false))
|
|
280
|
+
return;
|
|
281
|
+
if (!/^utt-\d+\.wav$/.test(basename(wav)))
|
|
282
|
+
return;
|
|
283
|
+
const candidate = resolve(expandSpeechPath(wav));
|
|
284
|
+
const candidateParent = dirname(candidate);
|
|
285
|
+
const allowedDirectory = companionAudioDirectories().find(dir => dir === candidateParent);
|
|
286
|
+
if (!allowedDirectory)
|
|
287
|
+
return;
|
|
288
|
+
try {
|
|
289
|
+
const { lstat, realpath, unlink } = await import('node:fs/promises');
|
|
290
|
+
const info = await lstat(candidate);
|
|
291
|
+
if (!info.isFile() || info.isSymbolicLink())
|
|
292
|
+
return;
|
|
293
|
+
const [realParent, realAllowedDirectory] = await Promise.all([
|
|
294
|
+
realpath(candidateParent),
|
|
295
|
+
realpath(allowedDirectory),
|
|
296
|
+
]);
|
|
297
|
+
if (realParent !== realAllowedDirectory)
|
|
298
|
+
return;
|
|
299
|
+
await unlink(candidate);
|
|
300
|
+
}
|
|
301
|
+
catch (error) {
|
|
302
|
+
if (error.code !== 'ENOENT') {
|
|
303
|
+
logger.debug('[speech] fallback WAV cleanup skipped', {
|
|
304
|
+
error: error instanceof Error ? error.message : String(error),
|
|
305
|
+
});
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
function resolveSpeechPython() {
|
|
310
|
+
const configured = process.env.CODEBUDDY_SPEECH_PYTHON ||
|
|
311
|
+
process.env.CODEBUDDY_VOICE_PYTHON ||
|
|
312
|
+
process.env.COWORK_VOICE_PYTHON ||
|
|
313
|
+
process.env.CODEBUDDY_PYTHON_BIN;
|
|
314
|
+
if (configured?.trim())
|
|
315
|
+
return configured.trim();
|
|
316
|
+
const candidates = [
|
|
317
|
+
join(homedir(), '.codebuddy/voice/.venv/bin/python'),
|
|
318
|
+
join(homedir(), 'DEV/ai-stack/voice/.venv/bin/python'),
|
|
319
|
+
join(homedir(), 'ai-stack/voice/.venv/bin/python'),
|
|
320
|
+
join(homedir(), 'vision_tests/venv/bin/python'),
|
|
321
|
+
];
|
|
322
|
+
return candidates.find((candidate) => existsSync(candidate)) || 'python3';
|
|
323
|
+
}
|
|
324
|
+
function truthyEnv(name, defaultValue) {
|
|
325
|
+
const value = process.env[name]?.trim().toLowerCase();
|
|
326
|
+
if (!value)
|
|
327
|
+
return defaultValue;
|
|
328
|
+
return ['1', 'true', 'yes', 'on'].includes(value);
|
|
329
|
+
}
|
|
330
|
+
function numericEnv(name, defaultValue) {
|
|
331
|
+
const value = Number(process.env[name]);
|
|
332
|
+
return Number.isFinite(value) && value > 0 ? value : defaultValue;
|
|
333
|
+
}
|
|
334
|
+
let fasterWhisperWorker = null;
|
|
335
|
+
let fasterWhisperWorkerSeq = 0;
|
|
336
|
+
let parakeetWorker = null;
|
|
337
|
+
let parakeetWorkerSeq = 0;
|
|
338
|
+
// In-process Rust STT (`buddy-sense stt`) — same persistent-worker protocol as the
|
|
339
|
+
// python workers above, but the recognizer runs in-process (sherpa-onnx) so there is
|
|
340
|
+
// no python on the hot path. The python whisper/parakeet workers stay as fallback.
|
|
341
|
+
let sherpaRustWorker = null;
|
|
342
|
+
let sherpaRustWorkerSeq = 0;
|
|
343
|
+
export const DEFAULT_SHERPA_EMPTY_THRESHOLD = 3;
|
|
344
|
+
let sherpaRustEmptyStreak = 0;
|
|
345
|
+
let sherpaRustInactiveAnnounced = false;
|
|
346
|
+
class SpeechWorkerRequestError extends Error {
|
|
347
|
+
constructor(message) {
|
|
348
|
+
super(message);
|
|
349
|
+
this.name = 'SpeechWorkerRequestError';
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
function defaultSpeechInitialPrompt() {
|
|
353
|
+
return 'Transcription en français. Ne complète pas les silences.';
|
|
354
|
+
}
|
|
355
|
+
function splitSpeechPhrases(value) {
|
|
356
|
+
return value
|
|
357
|
+
.split(/[\n,;]/)
|
|
358
|
+
.map((item) => item.trim())
|
|
359
|
+
.filter(Boolean);
|
|
360
|
+
}
|
|
361
|
+
function parakeetFallbackEnabled() {
|
|
362
|
+
return process.env.CODEBUDDY_SPEECH_FALLBACK?.trim().toLowerCase() !== 'false';
|
|
363
|
+
}
|
|
364
|
+
/**
|
|
365
|
+
* Locate the `buddy-sense` binary built with `--features stt`. Explicit override via
|
|
366
|
+
* CODEBUDDY_SPEECH_STT_BIN, else the conventional cargo output under the repo's
|
|
367
|
+
* `buddy-sense/target/{release,debug}/` (resolved from both cwd and this module's
|
|
368
|
+
* location, so it works under tsx-src and dist alike). Returns the first existing
|
|
369
|
+
* candidate, or '' when none is found (caller falls back to python STT).
|
|
370
|
+
*/
|
|
371
|
+
function resolveSherpaRustBin() {
|
|
372
|
+
const explicit = process.env.CODEBUDDY_SPEECH_STT_BIN?.trim();
|
|
373
|
+
if (explicit)
|
|
374
|
+
return expandSpeechPath(explicit);
|
|
375
|
+
const roots = new Set();
|
|
376
|
+
roots.add(process.cwd());
|
|
377
|
+
try {
|
|
378
|
+
// …/src/sensory/ or …/dist/sensory/ → repo root is two levels up.
|
|
379
|
+
roots.add(join(dirname(fileURLToPath(import.meta.url)), '..', '..'));
|
|
380
|
+
}
|
|
381
|
+
catch {
|
|
382
|
+
// import.meta.url unavailable (some test transforms) — cwd candidate suffices.
|
|
383
|
+
}
|
|
384
|
+
for (const root of roots) {
|
|
385
|
+
for (const profile of ['release', 'debug']) {
|
|
386
|
+
const candidate = join(root, 'buddy-sense', 'target', profile, 'buddy-sense');
|
|
387
|
+
if (existsSync(candidate))
|
|
388
|
+
return candidate;
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
return '';
|
|
392
|
+
}
|
|
393
|
+
function readSpeechHotwordsFile(filePath) {
|
|
394
|
+
try {
|
|
395
|
+
return splitSpeechPhrases(readFileSync(expandSpeechPath(filePath), 'utf8'));
|
|
396
|
+
}
|
|
397
|
+
catch (err) {
|
|
398
|
+
logger.warn(`[speech] could not read CODEBUDDY_SPEECH_HOTWORDS_FILE '${filePath}': ${err instanceof Error ? err.message : String(err)}`);
|
|
399
|
+
return [];
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
function defaultSpeechHotwords() {
|
|
403
|
+
const phrases = [
|
|
404
|
+
process.env.CODEBUDDY_ROBOT_NAME?.trim(),
|
|
405
|
+
'Lisa',
|
|
406
|
+
'Buddy',
|
|
407
|
+
'Code Buddy',
|
|
408
|
+
resolveUserName(),
|
|
409
|
+
...splitSpeechPhrases(process.env.CODEBUDDY_SPEECH_HOTWORDS ?? ''),
|
|
410
|
+
...(process.env.CODEBUDDY_SPEECH_HOTWORDS_FILE?.trim()
|
|
411
|
+
? readSpeechHotwordsFile(process.env.CODEBUDDY_SPEECH_HOTWORDS_FILE.trim())
|
|
412
|
+
: []),
|
|
413
|
+
].filter(Boolean);
|
|
414
|
+
return [...new Set(phrases)].slice(0, 80).join(', ');
|
|
415
|
+
}
|
|
416
|
+
export function resolveFasterWhisperOptions() {
|
|
417
|
+
const language = resolveSpeechLanguage();
|
|
418
|
+
const initialPrompt = process.env.CODEBUDDY_SPEECH_INITIAL_PROMPT?.trim() || defaultSpeechInitialPrompt();
|
|
419
|
+
const hotwords = defaultSpeechHotwords();
|
|
420
|
+
return {
|
|
421
|
+
language,
|
|
422
|
+
beamSize: numericEnv('CODEBUDDY_SPEECH_BEAM_SIZE', 1),
|
|
423
|
+
vadFilter: truthyEnv('CODEBUDDY_SPEECH_VAD_FILTER', true),
|
|
424
|
+
conditionOnPreviousText: truthyEnv('CODEBUDDY_SPEECH_CONDITION_PREVIOUS_TEXT', false),
|
|
425
|
+
initialPrompt,
|
|
426
|
+
...(hotwords ? { hotwords } : {}),
|
|
427
|
+
};
|
|
428
|
+
}
|
|
429
|
+
const SUBTITLE_HALLUCINATION_PATTERNS = [
|
|
430
|
+
/\bsous[-\s]?titres?\b.*\b(amara|communaut[eé]|r[ée]alis[ée]s?)\b/i,
|
|
431
|
+
/\bsous[-\s]?titrage\b.*\b(soci[ée]t[ée]\s+radio[-\s]?canada|radio[-\s]?canada)\b/i,
|
|
432
|
+
/\bamara\.org\b/i,
|
|
433
|
+
/\bmerci d['’]avoir regard[ée]\b/i,
|
|
434
|
+
/\bn['’]h[ée]sitez pas [àa] vous abonner\b/i,
|
|
435
|
+
/\bthank you for watching\b/i,
|
|
436
|
+
/\bsubtitles? by\b/i,
|
|
437
|
+
/\bcaptions? by\b/i,
|
|
438
|
+
/\btranscribed by\b/i,
|
|
439
|
+
];
|
|
440
|
+
const PROMPT_LEAKAGE_PATTERNS = [
|
|
441
|
+
/\b(transcription|conversation|conservation|fascination)\s+en\s+fran[çc]ais\b/i,
|
|
442
|
+
/\bconserve\s+les\s+noms\s+propres\b/i,
|
|
443
|
+
];
|
|
444
|
+
function looksLikeRepetitiveNoise(text) {
|
|
445
|
+
const compact = text
|
|
446
|
+
.toLowerCase()
|
|
447
|
+
.normalize('NFKC')
|
|
448
|
+
.replace(/[^\p{L}\p{N}]/gu, '');
|
|
449
|
+
if (compact.length < 6)
|
|
450
|
+
return false;
|
|
451
|
+
const unique = new Set([...compact]);
|
|
452
|
+
if (unique.size === 1)
|
|
453
|
+
return true;
|
|
454
|
+
return /^(.{1,3})\1{3,}$/.test(compact);
|
|
455
|
+
}
|
|
456
|
+
function looksLikeFillerNoise(text) {
|
|
457
|
+
const compact = text
|
|
458
|
+
.toLowerCase()
|
|
459
|
+
.normalize('NFKC')
|
|
460
|
+
.replace(/[^\p{L}]/gu, '');
|
|
461
|
+
return /^(m+|hm+|hmm+|mmh+|euh+|heu+|hum+)$/.test(compact);
|
|
462
|
+
}
|
|
463
|
+
export function normalizeSpeechTranscript(raw) {
|
|
464
|
+
const text = raw.normalize('NFKC').replace(/\s+/g, ' ').trim();
|
|
465
|
+
if (!text)
|
|
466
|
+
return { text: '' };
|
|
467
|
+
if (!/[\p{L}\p{N}]/u.test(text)) {
|
|
468
|
+
return { text: '', filteredReason: 'non_speech' };
|
|
469
|
+
}
|
|
470
|
+
if (SUBTITLE_HALLUCINATION_PATTERNS.some((pattern) => pattern.test(text))) {
|
|
471
|
+
return { text: '', filteredReason: 'subtitle_hallucination' };
|
|
472
|
+
}
|
|
473
|
+
if (PROMPT_LEAKAGE_PATTERNS.some((pattern) => pattern.test(text))) {
|
|
474
|
+
return { text: '', filteredReason: 'prompt_leakage' };
|
|
475
|
+
}
|
|
476
|
+
if (looksLikeRepetitiveNoise(text)) {
|
|
477
|
+
return { text: '', filteredReason: 'repetitive_noise' };
|
|
478
|
+
}
|
|
479
|
+
if (looksLikeFillerNoise(text)) {
|
|
480
|
+
return { text: '', filteredReason: 'filler_noise' };
|
|
481
|
+
}
|
|
482
|
+
return { text };
|
|
483
|
+
}
|
|
484
|
+
function elapsedSince(startMs, now) {
|
|
485
|
+
return Math.max(0, now() - startMs);
|
|
486
|
+
}
|
|
487
|
+
function finiteTimestamp(value) {
|
|
488
|
+
return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : undefined;
|
|
489
|
+
}
|
|
490
|
+
function speechWorkerEnabled() {
|
|
491
|
+
if (process.env.CODEBUDDY_SPEECH_WORKER?.trim()) {
|
|
492
|
+
return truthyEnv('CODEBUDDY_SPEECH_WORKER', true);
|
|
493
|
+
}
|
|
494
|
+
return truthyEnv('CODEBUDDY_SENSORY_SPEECH', false);
|
|
495
|
+
}
|
|
496
|
+
function buildPythonTranscribeKwargs(options) {
|
|
497
|
+
return [
|
|
498
|
+
options.language ? `"language": ${JSON.stringify(options.language)}` : '',
|
|
499
|
+
`"beam_size": ${options.beamSize}`,
|
|
500
|
+
`"vad_filter": ${options.vadFilter ? 'True' : 'False'}`,
|
|
501
|
+
`"condition_on_previous_text": ${options.conditionOnPreviousText ? 'True' : 'False'}`,
|
|
502
|
+
options.initialPrompt ? `"initial_prompt": ${JSON.stringify(options.initialPrompt)}` : '',
|
|
503
|
+
options.hotwords ? `"hotwords": ${JSON.stringify(options.hotwords)}` : '',
|
|
504
|
+
]
|
|
505
|
+
.filter(Boolean)
|
|
506
|
+
.join(', ');
|
|
507
|
+
}
|
|
508
|
+
function buildFasterWhisperWorkerScript(model, options) {
|
|
509
|
+
const transcribeKwargs = buildPythonTranscribeKwargs(options);
|
|
510
|
+
return [
|
|
511
|
+
'import inspect, json, sys, traceback',
|
|
512
|
+
'from faster_whisper import WhisperModel',
|
|
513
|
+
`m = WhisperModel(${JSON.stringify(model)}, device='cpu', compute_type='int8')`,
|
|
514
|
+
`kwargs = {${transcribeKwargs}}`,
|
|
515
|
+
'supported_kwargs = set(inspect.signature(m.transcribe).parameters.keys())',
|
|
516
|
+
'kwargs = {k: v for k, v in kwargs.items() if k in supported_kwargs}',
|
|
517
|
+
"print(json.dumps({'ready': True}), flush=True)",
|
|
518
|
+
'for line in sys.stdin:',
|
|
519
|
+
' line = line.strip()',
|
|
520
|
+
' if not line:',
|
|
521
|
+
' continue',
|
|
522
|
+
' try:',
|
|
523
|
+
' req = json.loads(line)',
|
|
524
|
+
" req_id = req.get('id')",
|
|
525
|
+
" wav = req.get('wav')",
|
|
526
|
+
' segs, _ = m.transcribe(wav, **kwargs)',
|
|
527
|
+
" text = ' '.join(s.text for s in segs).strip()",
|
|
528
|
+
" print(json.dumps({'id': req_id, 'text': text}), flush=True)",
|
|
529
|
+
' except Exception as exc:',
|
|
530
|
+
" print(json.dumps({'id': locals().get('req_id'), 'error': str(exc)}), flush=True)",
|
|
531
|
+
' traceback.print_exc(file=sys.stderr)',
|
|
532
|
+
].join('\n');
|
|
533
|
+
}
|
|
534
|
+
function buildParakeetWorkerScript(modelDir, numThreads) {
|
|
535
|
+
return [
|
|
536
|
+
'import json, sys, traceback, wave',
|
|
537
|
+
'import numpy as np',
|
|
538
|
+
'import sherpa_onnx',
|
|
539
|
+
`model_dir = ${JSON.stringify(modelDir)}`,
|
|
540
|
+
'rec = sherpa_onnx.OfflineRecognizer.from_transducer(',
|
|
541
|
+
" encoder=f'{model_dir}/encoder.int8.onnx',",
|
|
542
|
+
" decoder=f'{model_dir}/decoder.int8.onnx',",
|
|
543
|
+
" joiner=f'{model_dir}/joiner.int8.onnx',",
|
|
544
|
+
" tokens=f'{model_dir}/tokens.txt',",
|
|
545
|
+
` num_threads=${numThreads},`,
|
|
546
|
+
" decoding_method='greedy_search',",
|
|
547
|
+
" model_type='nemo_transducer',",
|
|
548
|
+
')',
|
|
549
|
+
'def transcribe(wav):',
|
|
550
|
+
" with wave.open(wav, 'rb') as wf:",
|
|
551
|
+
' sr = wf.getframerate()',
|
|
552
|
+
' channels = wf.getnchannels()',
|
|
553
|
+
' width = wf.getsampwidth()',
|
|
554
|
+
' raw = wf.readframes(wf.getnframes())',
|
|
555
|
+
' if width != 2:',
|
|
556
|
+
" raise RuntimeError(f'expected 16-bit PCM WAV, got sample width {width}')",
|
|
557
|
+
' audio = np.frombuffer(raw, dtype=np.int16).astype(np.float32) / 32768.0',
|
|
558
|
+
' if channels > 1:',
|
|
559
|
+
' audio = audio.reshape(-1, channels)[:, 0]',
|
|
560
|
+
' stream = rec.create_stream()',
|
|
561
|
+
' stream.accept_waveform(sr, audio)',
|
|
562
|
+
' rec.decode_stream(stream)',
|
|
563
|
+
" return getattr(stream.result, 'text', str(stream.result)).strip()",
|
|
564
|
+
"print(json.dumps({'ready': True}), flush=True)",
|
|
565
|
+
'for line in sys.stdin:',
|
|
566
|
+
' line = line.strip()',
|
|
567
|
+
' if not line:',
|
|
568
|
+
' continue',
|
|
569
|
+
' try:',
|
|
570
|
+
' req = json.loads(line)',
|
|
571
|
+
" req_id = req.get('id')",
|
|
572
|
+
" wav = req.get('wav')",
|
|
573
|
+
" print(json.dumps({'id': req_id, 'text': transcribe(wav)}), flush=True)",
|
|
574
|
+
' except Exception as exc:',
|
|
575
|
+
" print(json.dumps({'id': locals().get('req_id'), 'error': str(exc)}), flush=True)",
|
|
576
|
+
' traceback.print_exc(file=sys.stderr)',
|
|
577
|
+
].join('\n');
|
|
578
|
+
}
|
|
579
|
+
function fasterWhisperWorkerKey(python, model, options) {
|
|
580
|
+
return JSON.stringify({ python, model, options });
|
|
581
|
+
}
|
|
582
|
+
function parakeetWorkerKey(python, modelDir, numThreads) {
|
|
583
|
+
return JSON.stringify({ python, modelDir, numThreads });
|
|
584
|
+
}
|
|
585
|
+
function settlePending(worker, error) {
|
|
586
|
+
for (const pending of worker.pending.values()) {
|
|
587
|
+
clearTimeout(pending.timeout);
|
|
588
|
+
pending.reject(pending.describeFailure
|
|
589
|
+
? new SpeechWorkerRequestError(pending.describeFailure(error.message))
|
|
590
|
+
: error);
|
|
591
|
+
}
|
|
592
|
+
worker.pending.clear();
|
|
593
|
+
}
|
|
594
|
+
function disposeFasterWhisperWorker(worker) {
|
|
595
|
+
if (fasterWhisperWorker === worker)
|
|
596
|
+
fasterWhisperWorker = null;
|
|
597
|
+
settlePending(worker, new SpeechWorkerRequestError('faster-whisper worker disposed'));
|
|
598
|
+
worker.rl.close();
|
|
599
|
+
worker.proc.stdin.destroy();
|
|
600
|
+
worker.proc.kill();
|
|
601
|
+
}
|
|
602
|
+
function disposeParakeetWorker(worker) {
|
|
603
|
+
if (parakeetWorker === worker)
|
|
604
|
+
parakeetWorker = null;
|
|
605
|
+
settlePending(worker, new SpeechWorkerRequestError('Parakeet worker disposed'));
|
|
606
|
+
worker.rl.close();
|
|
607
|
+
worker.proc.stdin.destroy();
|
|
608
|
+
worker.proc.kill();
|
|
609
|
+
}
|
|
610
|
+
async function createFasterWhisperWorker(python, model, options) {
|
|
16
611
|
const { spawn } = await import('child_process');
|
|
17
|
-
const
|
|
612
|
+
const { createInterface } = await import('readline');
|
|
613
|
+
const key = fasterWhisperWorkerKey(python, model, options);
|
|
614
|
+
const proc = spawn(python, ['-u', '-c', buildFasterWhisperWorkerScript(model, options)], {
|
|
615
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
616
|
+
});
|
|
617
|
+
let resolveReady = () => { };
|
|
618
|
+
let rejectReady = () => { };
|
|
619
|
+
const ready = new Promise((resolve, reject) => {
|
|
620
|
+
resolveReady = resolve;
|
|
621
|
+
rejectReady = reject;
|
|
622
|
+
});
|
|
623
|
+
const worker = {
|
|
624
|
+
key,
|
|
625
|
+
proc,
|
|
626
|
+
rl: createInterface({ input: proc.stdout }),
|
|
627
|
+
ready,
|
|
628
|
+
readySettled: false,
|
|
629
|
+
pending: new Map(),
|
|
630
|
+
};
|
|
631
|
+
let stderr = '';
|
|
632
|
+
proc.stderr.on('data', (data) => {
|
|
633
|
+
stderr = `${stderr}${String(data)}`.slice(-2_000);
|
|
634
|
+
});
|
|
635
|
+
worker.rl.on('line', (line) => {
|
|
636
|
+
let message;
|
|
637
|
+
try {
|
|
638
|
+
message = JSON.parse(line);
|
|
639
|
+
}
|
|
640
|
+
catch {
|
|
641
|
+
logger.warn(`[speech] STT worker emitted invalid JSON: ${line.slice(0, 160)}`);
|
|
642
|
+
return;
|
|
643
|
+
}
|
|
644
|
+
if (message.ready) {
|
|
645
|
+
worker.readySettled = true;
|
|
646
|
+
resolveReady();
|
|
647
|
+
return;
|
|
648
|
+
}
|
|
649
|
+
const id = message.id;
|
|
650
|
+
if (!id)
|
|
651
|
+
return;
|
|
652
|
+
const pending = worker.pending.get(id);
|
|
653
|
+
if (!pending)
|
|
654
|
+
return;
|
|
655
|
+
worker.pending.delete(id);
|
|
656
|
+
clearTimeout(pending.timeout);
|
|
657
|
+
if (message.error) {
|
|
658
|
+
logger.warn(`[speech] STT worker request failed: ${message.error.slice(0, 300)}`);
|
|
659
|
+
pending.reject(new SpeechWorkerRequestError(message.error.slice(0, 300)));
|
|
660
|
+
return;
|
|
661
|
+
}
|
|
662
|
+
pending.resolve(message.text?.trim() || '');
|
|
663
|
+
});
|
|
664
|
+
proc.on('close', (code) => {
|
|
665
|
+
if (fasterWhisperWorker === worker)
|
|
666
|
+
fasterWhisperWorker = null;
|
|
667
|
+
settlePending(worker, new SpeechWorkerRequestError(`faster-whisper worker closed (code=${code})`));
|
|
668
|
+
if (!worker.readySettled) {
|
|
669
|
+
rejectReady(new Error(`faster-whisper worker exited before ready (code=${code})`));
|
|
670
|
+
}
|
|
671
|
+
if (stderr.trim()) {
|
|
672
|
+
logger.warn(`[speech] STT worker closed (code=${code}): ${stderr.trim().slice(0, 300)}`);
|
|
673
|
+
}
|
|
674
|
+
});
|
|
675
|
+
proc.on('error', (err) => {
|
|
676
|
+
if (fasterWhisperWorker === worker)
|
|
677
|
+
fasterWhisperWorker = null;
|
|
678
|
+
settlePending(worker, worker.readySettled ? new SpeechWorkerRequestError(err.message) : err);
|
|
679
|
+
if (!worker.readySettled)
|
|
680
|
+
rejectReady(err);
|
|
681
|
+
});
|
|
682
|
+
fasterWhisperWorker = worker;
|
|
683
|
+
return worker;
|
|
684
|
+
}
|
|
685
|
+
async function createParakeetWorker(python, modelDir, numThreads) {
|
|
686
|
+
const { spawn } = await import('child_process');
|
|
687
|
+
const { createInterface } = await import('readline');
|
|
688
|
+
const key = parakeetWorkerKey(python, modelDir, numThreads);
|
|
689
|
+
const proc = spawn(python, ['-u', '-c', buildParakeetWorkerScript(modelDir, numThreads)], {
|
|
690
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
691
|
+
});
|
|
692
|
+
let resolveReady = () => { };
|
|
693
|
+
let rejectReady = () => { };
|
|
694
|
+
const ready = new Promise((resolve, reject) => {
|
|
695
|
+
resolveReady = resolve;
|
|
696
|
+
rejectReady = reject;
|
|
697
|
+
});
|
|
698
|
+
const worker = {
|
|
699
|
+
key,
|
|
700
|
+
proc,
|
|
701
|
+
rl: createInterface({ input: proc.stdout }),
|
|
702
|
+
ready,
|
|
703
|
+
readySettled: false,
|
|
704
|
+
pending: new Map(),
|
|
705
|
+
};
|
|
706
|
+
let stderr = '';
|
|
707
|
+
proc.stderr.on('data', (data) => {
|
|
708
|
+
stderr = `${stderr}${String(data)}`.slice(-2_000);
|
|
709
|
+
});
|
|
710
|
+
worker.rl.on('line', (line) => {
|
|
711
|
+
let message;
|
|
712
|
+
try {
|
|
713
|
+
message = JSON.parse(line);
|
|
714
|
+
}
|
|
715
|
+
catch {
|
|
716
|
+
logger.warn(`[speech] Parakeet worker emitted invalid JSON: ${line.slice(0, 160)}`);
|
|
717
|
+
return;
|
|
718
|
+
}
|
|
719
|
+
if (message.ready) {
|
|
720
|
+
worker.readySettled = true;
|
|
721
|
+
resolveReady();
|
|
722
|
+
return;
|
|
723
|
+
}
|
|
724
|
+
const id = message.id;
|
|
725
|
+
if (!id)
|
|
726
|
+
return;
|
|
727
|
+
const pending = worker.pending.get(id);
|
|
728
|
+
if (!pending)
|
|
729
|
+
return;
|
|
730
|
+
worker.pending.delete(id);
|
|
731
|
+
clearTimeout(pending.timeout);
|
|
732
|
+
if (message.error) {
|
|
733
|
+
logger.warn(`[speech] Parakeet request failed: ${message.error.slice(0, 300)}`);
|
|
734
|
+
pending.reject(new SpeechWorkerRequestError(message.error.slice(0, 300)));
|
|
735
|
+
return;
|
|
736
|
+
}
|
|
737
|
+
pending.resolve(message.text?.trim() || '');
|
|
738
|
+
});
|
|
739
|
+
proc.on('close', (code) => {
|
|
740
|
+
if (parakeetWorker === worker)
|
|
741
|
+
parakeetWorker = null;
|
|
742
|
+
settlePending(worker, new SpeechWorkerRequestError(`Parakeet worker closed (code=${code})`));
|
|
743
|
+
if (!worker.readySettled) {
|
|
744
|
+
rejectReady(new Error(`Parakeet worker exited before ready (code=${code})`));
|
|
745
|
+
}
|
|
746
|
+
if (stderr.trim()) {
|
|
747
|
+
logger.warn(`[speech] Parakeet worker closed (code=${code}): ${stderr.trim().slice(0, 300)}`);
|
|
748
|
+
}
|
|
749
|
+
});
|
|
750
|
+
proc.on('error', (err) => {
|
|
751
|
+
if (parakeetWorker === worker)
|
|
752
|
+
parakeetWorker = null;
|
|
753
|
+
settlePending(worker, worker.readySettled ? new SpeechWorkerRequestError(err.message) : err);
|
|
754
|
+
if (!worker.readySettled)
|
|
755
|
+
rejectReady(err);
|
|
756
|
+
});
|
|
757
|
+
parakeetWorker = worker;
|
|
758
|
+
return worker;
|
|
759
|
+
}
|
|
760
|
+
async function getFasterWhisperWorker(python, model, options) {
|
|
761
|
+
const key = fasterWhisperWorkerKey(python, model, options);
|
|
762
|
+
if (fasterWhisperWorker?.key === key)
|
|
763
|
+
return fasterWhisperWorker;
|
|
764
|
+
if (fasterWhisperWorker)
|
|
765
|
+
disposeFasterWhisperWorker(fasterWhisperWorker);
|
|
766
|
+
return createFasterWhisperWorker(python, model, options);
|
|
767
|
+
}
|
|
768
|
+
async function getParakeetWorker(python, modelDir, numThreads) {
|
|
769
|
+
const key = parakeetWorkerKey(python, modelDir, numThreads);
|
|
770
|
+
if (parakeetWorker?.key === key)
|
|
771
|
+
return parakeetWorker;
|
|
772
|
+
if (parakeetWorker)
|
|
773
|
+
disposeParakeetWorker(parakeetWorker);
|
|
774
|
+
return createParakeetWorker(python, modelDir, numThreads);
|
|
775
|
+
}
|
|
776
|
+
async function waitForWorkerReady(worker, timeoutMs) {
|
|
777
|
+
timeoutMs ??= numericEnv('CODEBUDDY_SPEECH_WORKER_READY_TIMEOUT_MS', 30_000);
|
|
778
|
+
let timeout;
|
|
779
|
+
try {
|
|
780
|
+
await Promise.race([
|
|
781
|
+
worker.ready,
|
|
782
|
+
new Promise((_, reject) => {
|
|
783
|
+
timeout = setTimeout(() => reject(new Error(`faster-whisper worker did not become ready within ${timeoutMs}ms`)), timeoutMs);
|
|
784
|
+
}),
|
|
785
|
+
]);
|
|
786
|
+
}
|
|
787
|
+
finally {
|
|
788
|
+
if (timeout)
|
|
789
|
+
clearTimeout(timeout);
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
async function transcribeWavWithWorker(wav, python, model, options) {
|
|
793
|
+
const worker = await getFasterWhisperWorker(python, model, options);
|
|
794
|
+
await waitForWorkerReady(worker);
|
|
795
|
+
const timeoutMs = numericEnv('CODEBUDDY_SPEECH_WORKER_TIMEOUT_MS', 20_000);
|
|
796
|
+
const id = `speech-${Date.now()}-${++fasterWhisperWorkerSeq}`;
|
|
797
|
+
return new Promise((resolve, reject) => {
|
|
798
|
+
const timeout = setTimeout(() => {
|
|
799
|
+
worker.pending.delete(id);
|
|
800
|
+
const error = new SpeechWorkerRequestError(`faster-whisper worker request timed out after ${timeoutMs}ms`);
|
|
801
|
+
logger.warn(`[speech] STT worker request timed out after ${timeoutMs}ms`);
|
|
802
|
+
disposeFasterWhisperWorker(worker);
|
|
803
|
+
reject(error);
|
|
804
|
+
}, timeoutMs);
|
|
805
|
+
worker.pending.set(id, { resolve, reject, timeout });
|
|
806
|
+
try {
|
|
807
|
+
worker.proc.stdin.write(`${JSON.stringify({ id, wav })}\n`);
|
|
808
|
+
}
|
|
809
|
+
catch (err) {
|
|
810
|
+
worker.pending.delete(id);
|
|
811
|
+
clearTimeout(timeout);
|
|
812
|
+
reject(err instanceof Error ? err : new Error(String(err)));
|
|
813
|
+
}
|
|
814
|
+
});
|
|
815
|
+
}
|
|
816
|
+
async function transcribeWavWithParakeetWorker(wav, python, modelDir, numThreads) {
|
|
817
|
+
const worker = await getParakeetWorker(python, modelDir, numThreads);
|
|
818
|
+
await waitForWorkerReady(worker);
|
|
819
|
+
const timeoutMs = numericEnv('CODEBUDDY_SPEECH_WORKER_TIMEOUT_MS', 20_000);
|
|
820
|
+
const id = `parakeet-${Date.now()}-${++parakeetWorkerSeq}`;
|
|
821
|
+
return new Promise((resolve, reject) => {
|
|
822
|
+
const timeout = setTimeout(() => {
|
|
823
|
+
worker.pending.delete(id);
|
|
824
|
+
const error = new SpeechWorkerRequestError(`Parakeet worker request timed out after ${timeoutMs}ms`);
|
|
825
|
+
logger.warn(`[speech] Parakeet worker request timed out after ${timeoutMs}ms`);
|
|
826
|
+
disposeParakeetWorker(worker);
|
|
827
|
+
reject(error);
|
|
828
|
+
}, timeoutMs);
|
|
829
|
+
worker.pending.set(id, { resolve, reject, timeout });
|
|
830
|
+
try {
|
|
831
|
+
worker.proc.stdin.write(`${JSON.stringify({ id, wav })}\n`);
|
|
832
|
+
}
|
|
833
|
+
catch (err) {
|
|
834
|
+
worker.pending.delete(id);
|
|
835
|
+
clearTimeout(timeout);
|
|
836
|
+
reject(err instanceof Error ? err : new Error(String(err)));
|
|
837
|
+
}
|
|
838
|
+
});
|
|
839
|
+
}
|
|
840
|
+
async function transcribeWavOneShot(wav, python, model, options) {
|
|
841
|
+
const { spawn } = await import('child_process');
|
|
842
|
+
const transcribeKwargs = buildPythonTranscribeKwargs(options);
|
|
18
843
|
const py = [
|
|
19
|
-
'import sys',
|
|
844
|
+
'import inspect, sys',
|
|
20
845
|
'from faster_whisper import WhisperModel',
|
|
21
846
|
`m = WhisperModel(${JSON.stringify(model)}, device='cpu', compute_type='int8')`,
|
|
22
|
-
|
|
847
|
+
`kwargs = {${transcribeKwargs}}`,
|
|
848
|
+
'supported_kwargs = set(inspect.signature(m.transcribe).parameters.keys())',
|
|
849
|
+
'kwargs = {k: v for k, v in kwargs.items() if k in supported_kwargs}',
|
|
850
|
+
'segs, _ = m.transcribe(sys.argv[1], **kwargs)',
|
|
23
851
|
"print(' '.join(s.text for s in segs).strip())",
|
|
24
852
|
].join('\n');
|
|
25
|
-
return new Promise((resolve) => {
|
|
26
|
-
|
|
853
|
+
return new Promise((resolve, reject) => {
|
|
854
|
+
// Capture stderr (was ignored) so an STT failure is LOUD in the journal, not silent.
|
|
855
|
+
const proc = spawn(python, ['-c', py, wav], { stdio: ['ignore', 'pipe', 'pipe'] });
|
|
27
856
|
let out = '';
|
|
857
|
+
let err = '';
|
|
28
858
|
proc.stdout.on('data', (d) => (out += String(d)));
|
|
29
|
-
proc.on('
|
|
30
|
-
proc.on('
|
|
859
|
+
proc.stderr.on('data', (d) => (err += String(d)));
|
|
860
|
+
proc.on('close', (code) => {
|
|
861
|
+
if (code !== 0) {
|
|
862
|
+
const cause = err.trim().slice(0, 300) || `process exited with code ${code}`;
|
|
863
|
+
logger.warn(`[speech] STT failed (python='${python}', exit=${code}): ${cause}`);
|
|
864
|
+
reject(new Error(`faster-whisper STT failed: ${cause}`));
|
|
865
|
+
return;
|
|
866
|
+
}
|
|
867
|
+
resolve(out.trim());
|
|
868
|
+
});
|
|
869
|
+
proc.on('error', (e) => {
|
|
870
|
+
logger.warn(`[speech] STT spawn failed (python='${python}'): ${e instanceof Error ? e.message : String(e)}`);
|
|
871
|
+
reject(e instanceof Error ? e : new Error(String(e)));
|
|
872
|
+
});
|
|
873
|
+
});
|
|
874
|
+
}
|
|
875
|
+
async function transcribeWavParakeetOneShot(wav, python, modelDir, numThreads) {
|
|
876
|
+
const { spawn } = await import('child_process');
|
|
877
|
+
const py = [
|
|
878
|
+
'import sys',
|
|
879
|
+
buildParakeetWorkerScript(modelDir, numThreads),
|
|
880
|
+
'print(transcribe(sys.argv[1]))',
|
|
881
|
+
].join('\n');
|
|
882
|
+
return new Promise((resolve, reject) => {
|
|
883
|
+
const proc = spawn(python, ['-c', py, wav], { stdio: ['ignore', 'pipe', 'pipe'] });
|
|
884
|
+
let out = '';
|
|
885
|
+
let err = '';
|
|
886
|
+
proc.stdout.on('data', (d) => (out += String(d)));
|
|
887
|
+
proc.stderr.on('data', (d) => (err += String(d)));
|
|
888
|
+
proc.on('close', (code) => {
|
|
889
|
+
const lines = out
|
|
890
|
+
.split(/\r?\n/)
|
|
891
|
+
.map((line) => line.trim())
|
|
892
|
+
.filter(Boolean);
|
|
893
|
+
const text = lines.filter((line) => !line.startsWith('{')).at(-1) || '';
|
|
894
|
+
if (code !== 0) {
|
|
895
|
+
const cause = err.trim().slice(0, 300) || `process exited with code ${code}`;
|
|
896
|
+
logger.warn(`[speech] Parakeet STT failed (python='${python}', exit=${code}): ${cause}`);
|
|
897
|
+
reject(new Error(`Parakeet STT failed: ${cause}`));
|
|
898
|
+
return;
|
|
899
|
+
}
|
|
900
|
+
resolve(text.trim());
|
|
901
|
+
});
|
|
902
|
+
proc.on('error', (e) => {
|
|
903
|
+
logger.warn(`[speech] Parakeet STT spawn failed (python='${python}'): ${e instanceof Error ? e.message : String(e)}`);
|
|
904
|
+
reject(e instanceof Error ? e : new Error(String(e)));
|
|
905
|
+
});
|
|
906
|
+
});
|
|
907
|
+
}
|
|
908
|
+
async function transcribeWavWithFasterWhisperRaw(wav) {
|
|
909
|
+
const model = process.env.CODEBUDDY_SPEECH_MODEL ?? 'base';
|
|
910
|
+
const options = resolveFasterWhisperOptions();
|
|
911
|
+
// Resolve the Python interpreter from env so STT works when faster-whisper lives
|
|
912
|
+
// OUTSIDE the service PATH's python3 (e.g. a conda/miniforge env). Without this a
|
|
913
|
+
// systemd service whose python3 is /usr/bin/python3 (no faster_whisper) fails STT
|
|
914
|
+
// SILENTLY → no transcription, no spoken reply. Set CODEBUDDY_SPEECH_PYTHON to the
|
|
915
|
+
// interpreter that has faster-whisper.
|
|
916
|
+
const python = resolveSpeechPython();
|
|
917
|
+
if (speechWorkerEnabled()) {
|
|
918
|
+
try {
|
|
919
|
+
return await transcribeWavWithWorker(wav, python, model, options);
|
|
920
|
+
}
|
|
921
|
+
catch (err) {
|
|
922
|
+
if (err instanceof SpeechWorkerRequestError)
|
|
923
|
+
throw err;
|
|
924
|
+
logger.warn(`[speech] STT worker unavailable, falling back to one-shot: ${err instanceof Error ? err.message : String(err)}`);
|
|
925
|
+
if (fasterWhisperWorker)
|
|
926
|
+
disposeFasterWhisperWorker(fasterWhisperWorker);
|
|
927
|
+
}
|
|
928
|
+
}
|
|
929
|
+
return transcribeWavOneShot(wav, python, model, options);
|
|
930
|
+
}
|
|
931
|
+
async function transcribeWavWithParakeetRaw(wav) {
|
|
932
|
+
const python = resolveSpeechPython();
|
|
933
|
+
const modelDir = resolveParakeetModelDir();
|
|
934
|
+
const numThreads = numericEnv('CODEBUDDY_PARAKEET_THREADS', numericEnv('CODEBUDDY_SPEECH_THREADS', 12));
|
|
935
|
+
if (speechWorkerEnabled()) {
|
|
936
|
+
try {
|
|
937
|
+
return await transcribeWavWithParakeetWorker(wav, python, modelDir, numThreads);
|
|
938
|
+
}
|
|
939
|
+
catch (err) {
|
|
940
|
+
if (err instanceof SpeechWorkerRequestError)
|
|
941
|
+
throw err;
|
|
942
|
+
logger.warn(`[speech] Parakeet worker unavailable, falling back to one-shot: ${err instanceof Error ? err.message : String(err)}`);
|
|
943
|
+
if (parakeetWorker)
|
|
944
|
+
disposeParakeetWorker(parakeetWorker);
|
|
945
|
+
}
|
|
946
|
+
}
|
|
947
|
+
return transcribeWavParakeetOneShot(wav, python, modelDir, numThreads);
|
|
948
|
+
}
|
|
949
|
+
function sherpaRustWorkerKey(bin, modelDir, numThreads) {
|
|
950
|
+
return JSON.stringify({ bin, modelDir, numThreads });
|
|
951
|
+
}
|
|
952
|
+
function disposeSherpaRustWorker(worker) {
|
|
953
|
+
if (sherpaRustWorker === worker)
|
|
954
|
+
sherpaRustWorker = null;
|
|
955
|
+
settlePending(worker, new SpeechWorkerRequestError('sherpa-rs worker disposed'));
|
|
956
|
+
worker.rl.close();
|
|
957
|
+
worker.proc.stdin.destroy();
|
|
958
|
+
worker.proc.kill();
|
|
959
|
+
}
|
|
960
|
+
async function createSherpaRustWorker(bin, modelDir, numThreads) {
|
|
961
|
+
const { spawn } = await import('child_process');
|
|
962
|
+
const { createInterface } = await import('readline');
|
|
963
|
+
const key = sherpaRustWorkerKey(bin, modelDir, numThreads);
|
|
964
|
+
// The prebuilt sherpa-onnx + onnxruntime .so live next to the binary (cargo copies
|
|
965
|
+
// them into target/<profile>/); point the loader at that directory.
|
|
966
|
+
const libDir = dirname(bin);
|
|
967
|
+
const proc = spawn(bin, ['stt'], {
|
|
968
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
969
|
+
env: {
|
|
970
|
+
...process.env,
|
|
971
|
+
LD_LIBRARY_PATH: `${libDir}${delimiter}${process.env.LD_LIBRARY_PATH ?? ''}`,
|
|
972
|
+
BUDDY_SENSE_STT_MODEL_DIR: modelDir,
|
|
973
|
+
BUDDY_SENSE_STT_THREADS: String(numThreads),
|
|
974
|
+
},
|
|
975
|
+
});
|
|
976
|
+
let resolveReady = () => { };
|
|
977
|
+
let rejectReady = () => { };
|
|
978
|
+
const ready = new Promise((resolve, reject) => {
|
|
979
|
+
resolveReady = resolve;
|
|
980
|
+
rejectReady = reject;
|
|
981
|
+
});
|
|
982
|
+
const worker = {
|
|
983
|
+
key,
|
|
984
|
+
proc,
|
|
985
|
+
rl: createInterface({ input: proc.stdout }),
|
|
986
|
+
ready,
|
|
987
|
+
readySettled: false,
|
|
988
|
+
pending: new Map(),
|
|
989
|
+
stderrTail: '',
|
|
990
|
+
};
|
|
991
|
+
proc.stderr.on('data', (data) => {
|
|
992
|
+
worker.stderrTail = `${worker.stderrTail ?? ''}${String(data)}`.slice(-2_000);
|
|
993
|
+
});
|
|
994
|
+
worker.rl.on('line', (line) => {
|
|
995
|
+
let message;
|
|
996
|
+
try {
|
|
997
|
+
message = JSON.parse(line);
|
|
998
|
+
}
|
|
999
|
+
catch {
|
|
1000
|
+
logger.warn(`[speech] sherpa-rs worker emitted invalid JSON: ${line.slice(0, 160)}`);
|
|
1001
|
+
return;
|
|
1002
|
+
}
|
|
1003
|
+
if (message.ready) {
|
|
1004
|
+
worker.readySettled = true;
|
|
1005
|
+
resolveReady();
|
|
1006
|
+
return;
|
|
1007
|
+
}
|
|
1008
|
+
const id = message.id;
|
|
1009
|
+
if (!id)
|
|
1010
|
+
return;
|
|
1011
|
+
const pending = worker.pending.get(id);
|
|
1012
|
+
if (!pending)
|
|
1013
|
+
return;
|
|
1014
|
+
worker.pending.delete(id);
|
|
1015
|
+
clearTimeout(pending.timeout);
|
|
1016
|
+
if (message.error) {
|
|
1017
|
+
logger.warn(`[speech] sherpa-rs request failed: ${message.error.slice(0, 300)}`);
|
|
1018
|
+
const reason = `request_error:${message.error.slice(0, 300)}`;
|
|
1019
|
+
pending.reject(new SpeechWorkerRequestError(pending.describeFailure ? pending.describeFailure(reason) : reason));
|
|
1020
|
+
return;
|
|
1021
|
+
}
|
|
1022
|
+
pending.resolve(message.text?.trim() || '');
|
|
1023
|
+
});
|
|
1024
|
+
proc.on('close', (code) => {
|
|
1025
|
+
if (sherpaRustWorker === worker)
|
|
1026
|
+
sherpaRustWorker = null;
|
|
1027
|
+
settlePending(worker, new SpeechWorkerRequestError(`sherpa-rs worker closed (code=${code})`));
|
|
1028
|
+
if (!worker.readySettled) {
|
|
1029
|
+
rejectReady(new Error(`sherpa-rs worker exited before ready (code=${code})`));
|
|
1030
|
+
}
|
|
1031
|
+
if (worker.stderrTail?.trim()) {
|
|
1032
|
+
logger.warn(`[speech] sherpa-rs worker closed (code=${code}): ${worker.stderrTail.trim().slice(0, 300)}`);
|
|
1033
|
+
}
|
|
1034
|
+
});
|
|
1035
|
+
proc.on('error', (err) => {
|
|
1036
|
+
if (sherpaRustWorker === worker)
|
|
1037
|
+
sherpaRustWorker = null;
|
|
1038
|
+
settlePending(worker, worker.readySettled ? new SpeechWorkerRequestError(err.message) : err);
|
|
1039
|
+
if (!worker.readySettled)
|
|
1040
|
+
rejectReady(err);
|
|
1041
|
+
});
|
|
1042
|
+
sherpaRustWorker = worker;
|
|
1043
|
+
return worker;
|
|
1044
|
+
}
|
|
1045
|
+
async function getSherpaRustWorker(bin, modelDir, numThreads) {
|
|
1046
|
+
const key = sherpaRustWorkerKey(bin, modelDir, numThreads);
|
|
1047
|
+
if (sherpaRustWorker?.key === key)
|
|
1048
|
+
return sherpaRustWorker;
|
|
1049
|
+
if (sherpaRustWorker)
|
|
1050
|
+
disposeSherpaRustWorker(sherpaRustWorker);
|
|
1051
|
+
return createSherpaRustWorker(bin, modelDir, numThreads);
|
|
1052
|
+
}
|
|
1053
|
+
async function transcribeWavWithSherpaRustWorker(wav, bin, modelDir, numThreads) {
|
|
1054
|
+
const worker = await getSherpaRustWorker(bin, modelDir, numThreads);
|
|
1055
|
+
// The Rust recognizer loads in ~1.8 s, so a tight ready-timeout fails fast on a
|
|
1056
|
+
// broken/featureless binary instead of stalling 30 s per utterance (a stale binary
|
|
1057
|
+
// built without `--features stt` ignores the `stt` arg and runs the daemon).
|
|
1058
|
+
await waitForWorkerReady(worker, numericEnv('CODEBUDDY_SPEECH_STT_READY_TIMEOUT_MS', 8_000));
|
|
1059
|
+
const timeoutMs = numericEnv('CODEBUDDY_SPEECH_WORKER_TIMEOUT_MS', 20_000);
|
|
1060
|
+
const id = `sherpa-rs-${Date.now()}-${++sherpaRustWorkerSeq}`;
|
|
1061
|
+
const startedAtMs = Date.now();
|
|
1062
|
+
const describeFailure = (reason) => describeSherpaRustAttempt(wav, startedAtMs, worker, reason);
|
|
1063
|
+
return new Promise((resolve, reject) => {
|
|
1064
|
+
const timeout = setTimeout(() => {
|
|
1065
|
+
worker.pending.delete(id);
|
|
1066
|
+
const error = new SpeechWorkerRequestError(`sherpa-rs worker request timed out after ${timeoutMs}ms; ${describeFailure('timeout')}`);
|
|
1067
|
+
logger.warn(`[speech] sherpa-rs worker request timed out; ${error.message}`);
|
|
1068
|
+
disposeSherpaRustWorker(worker);
|
|
1069
|
+
reject(error);
|
|
1070
|
+
}, timeoutMs);
|
|
1071
|
+
worker.pending.set(id, { resolve, reject, timeout, describeFailure });
|
|
1072
|
+
try {
|
|
1073
|
+
worker.proc.stdin.write(`${JSON.stringify({ id, wav })}\n`);
|
|
1074
|
+
}
|
|
1075
|
+
catch (err) {
|
|
1076
|
+
worker.pending.delete(id);
|
|
1077
|
+
clearTimeout(timeout);
|
|
1078
|
+
reject(err instanceof Error ? err : new Error(String(err)));
|
|
1079
|
+
}
|
|
31
1080
|
});
|
|
32
1081
|
}
|
|
1082
|
+
/** In-process Rust STT. Throws when the binary isn't built/resolvable, so callers can
|
|
1083
|
+
* fall back to python STT (mirrors the parakeet fallback). No one-shot path: the
|
|
1084
|
+
* recognizer's value is the loaded-once persistent worker. */
|
|
1085
|
+
async function transcribeWavWithSherpaRustRaw(wav) {
|
|
1086
|
+
const bin = resolveSherpaRustBin();
|
|
1087
|
+
if (!bin)
|
|
1088
|
+
throw new Error('buddy-sense stt binary not found (build with --features stt or set CODEBUDDY_SPEECH_STT_BIN)');
|
|
1089
|
+
return transcribeWavWithSherpaRustWorker(wav, bin, resolveParakeetModelDir(), resolveSpeechSttThreads());
|
|
1090
|
+
}
|
|
1091
|
+
const warnedSpeechFallbacks = new Set();
|
|
1092
|
+
function warnAutoFallbackOnce(reason) {
|
|
1093
|
+
const key = `auto:${reason}`;
|
|
1094
|
+
if (warnedSpeechFallbacks.has(key))
|
|
1095
|
+
return;
|
|
1096
|
+
warnedSpeechFallbacks.add(key);
|
|
1097
|
+
logger.warn(`[speech] auto STT fallback activated: effective=faster-whisper reason=${reason}`);
|
|
1098
|
+
}
|
|
1099
|
+
/** Log one configuration-level STT fallback once for the lifetime of this process. */
|
|
1100
|
+
export function warnSpeechFallbackOnce(plan, engine, hotwordCount) {
|
|
1101
|
+
if (!plan.fallbackReason)
|
|
1102
|
+
return;
|
|
1103
|
+
const key = plan.fallbackReason;
|
|
1104
|
+
if (warnedSpeechFallbacks.has(key))
|
|
1105
|
+
return;
|
|
1106
|
+
warnedSpeechFallbacks.add(key);
|
|
1107
|
+
logger.warn(`[speech] STT fallback activated: requested=${plan.requestedEngine} effective=${engine} `
|
|
1108
|
+
+ `language=${plan.language} reason=${plan.fallbackReason} hotwords=${hotwordCount}`);
|
|
1109
|
+
}
|
|
1110
|
+
/** Transcribe a WAV and report the decoder that produced the returned text. */
|
|
1111
|
+
export async function transcribeWavWithMetadata(wav, engineOverride) {
|
|
1112
|
+
// `engineOverride` lets ONE call path (e.g. long/video transcription) prefer a faster
|
|
1113
|
+
// engine WITHOUT touching the global `CODEBUDDY_SPEECH_ENGINE` default that the
|
|
1114
|
+
// companion/sensory hot paths read. Unset → the env-driven resolution (unchanged).
|
|
1115
|
+
const requestedEngine = engineOverride ?? resolveSpeechRecognitionEngine();
|
|
1116
|
+
const plan = resolveSpeechTranscriptionPlan(requestedEngine);
|
|
1117
|
+
const engine = plan.effectiveEngine;
|
|
1118
|
+
if (plan.blockingReason) {
|
|
1119
|
+
const message = `requested=${plan.requestedEngine} language=${plan.language} reason=${plan.blockingReason}`;
|
|
1120
|
+
logger.error(`[speech] STT unavailable: ${message}`);
|
|
1121
|
+
throw new Error(`STT unavailable: ${message}`);
|
|
1122
|
+
}
|
|
1123
|
+
if (plan.fallbackReason) {
|
|
1124
|
+
const hotwordCount = splitSpeechPhrases(defaultSpeechHotwords()).length;
|
|
1125
|
+
warnSpeechFallbackOnce(plan, engine, hotwordCount);
|
|
1126
|
+
}
|
|
1127
|
+
if (engine === 'faster-whisper') {
|
|
1128
|
+
return { text: await transcribeWavWithFasterWhisperRaw(wav), engine };
|
|
1129
|
+
}
|
|
1130
|
+
if (engine === 'sherpa-rs') {
|
|
1131
|
+
const startedAtMs = Date.now();
|
|
1132
|
+
try {
|
|
1133
|
+
const text = await transcribeWavWithSherpaRustRaw(wav);
|
|
1134
|
+
if (text) {
|
|
1135
|
+
recordSherpaRustSuccess();
|
|
1136
|
+
return { text, engine };
|
|
1137
|
+
}
|
|
1138
|
+
const details = describeSherpaRustAttempt(wav, startedAtMs, sherpaRustWorker, 'no_tokens');
|
|
1139
|
+
recordSherpaRustEmpty(details, plan.fallbackEnabled);
|
|
1140
|
+
if (!plan.fallbackEnabled)
|
|
1141
|
+
return { text, engine };
|
|
1142
|
+
}
|
|
1143
|
+
catch (err) {
|
|
1144
|
+
if (!plan.fallbackEnabled)
|
|
1145
|
+
throw err;
|
|
1146
|
+
const details = err instanceof SpeechWorkerRequestError && err.message.includes('reason=')
|
|
1147
|
+
? err.message
|
|
1148
|
+
: describeSherpaRustAttempt(wav, startedAtMs, sherpaRustWorker, `runtime_error:${err instanceof Error ? err.message : String(err)}`);
|
|
1149
|
+
logger.warn(`[speech] sherpa-rs failed; falling back to faster-whisper: ${details}`);
|
|
1150
|
+
}
|
|
1151
|
+
return { text: await transcribeWavWithFasterWhisperRaw(wav), engine: 'faster-whisper' };
|
|
1152
|
+
}
|
|
1153
|
+
if (engine === 'parakeet') {
|
|
1154
|
+
try {
|
|
1155
|
+
const text = await transcribeWavWithParakeetRaw(wav);
|
|
1156
|
+
if (text || !parakeetFallbackEnabled())
|
|
1157
|
+
return { text, engine };
|
|
1158
|
+
logger.warn('[speech] Parakeet returned an empty transcript; falling back to faster-whisper');
|
|
1159
|
+
}
|
|
1160
|
+
catch (err) {
|
|
1161
|
+
if (!parakeetFallbackEnabled())
|
|
1162
|
+
throw err;
|
|
1163
|
+
logger.warn(`[speech] Parakeet failed; falling back to faster-whisper: ${err instanceof Error ? err.message : String(err)}`);
|
|
1164
|
+
}
|
|
1165
|
+
return { text: await transcribeWavWithFasterWhisperRaw(wav), engine: 'faster-whisper' };
|
|
1166
|
+
}
|
|
1167
|
+
// Auto mode: prefer the in-process Rust engine only when both its binary and a
|
|
1168
|
+
// complete locally evidenced French model are present. A bare directory or a
|
|
1169
|
+
// stale binary must never silently select the sherpa path.
|
|
1170
|
+
const sherpaBin = resolveSherpaRustBin();
|
|
1171
|
+
const modelDir = resolveParakeetModelDir();
|
|
1172
|
+
const binaryAvailable = Boolean(sherpaBin && existsSync(sherpaBin));
|
|
1173
|
+
const frenchModelAvailable = isFrenchParakeetModelAvailable(modelDir);
|
|
1174
|
+
if (binaryAvailable && frenchModelAvailable) {
|
|
1175
|
+
try {
|
|
1176
|
+
const text = await transcribeWavWithSherpaRustRaw(wav);
|
|
1177
|
+
if (text)
|
|
1178
|
+
recordSherpaRustSuccess();
|
|
1179
|
+
return { text, engine: 'sherpa-rs' };
|
|
1180
|
+
}
|
|
1181
|
+
catch (err) {
|
|
1182
|
+
warnAutoFallbackOnce('sherpa-rs-runtime-unavailable');
|
|
1183
|
+
logger.warn(`[speech] auto STT: sherpa-rs unavailable; trying Parakeet/faster-whisper: ${err instanceof Error ? err.message : String(err)}`);
|
|
1184
|
+
}
|
|
1185
|
+
}
|
|
1186
|
+
else {
|
|
1187
|
+
warnAutoFallbackOnce(!binaryAvailable
|
|
1188
|
+
? 'sherpa-rs-binary-missing'
|
|
1189
|
+
: 'french-parakeet-model-missing-or-incomplete');
|
|
1190
|
+
}
|
|
1191
|
+
if (frenchModelAvailable) {
|
|
1192
|
+
try {
|
|
1193
|
+
return { text: await transcribeWavWithParakeetRaw(wav), engine: 'parakeet' };
|
|
1194
|
+
}
|
|
1195
|
+
catch (err) {
|
|
1196
|
+
warnAutoFallbackOnce('parakeet-runtime-unavailable');
|
|
1197
|
+
logger.warn(`[speech] auto STT: Parakeet unavailable; trying faster-whisper: ${err instanceof Error ? err.message : String(err)}`);
|
|
1198
|
+
}
|
|
1199
|
+
}
|
|
1200
|
+
return { text: await transcribeWavWithFasterWhisperRaw(wav), engine: 'faster-whisper' };
|
|
1201
|
+
}
|
|
1202
|
+
async function transcribeWavRaw(wav, engineOverride) {
|
|
1203
|
+
return (await transcribeWavWithMetadata(wav, engineOverride)).text;
|
|
1204
|
+
}
|
|
1205
|
+
/** Default transcriber: local faster-whisper (base), best-effort, $0. Exported so the
|
|
1206
|
+
* push-to-talk CLI path (`buddy voice`) transcribes through the exact same STT as the daemon.
|
|
1207
|
+
* `engineOverride` (additive, back-compat with the `Transcriber` type) lets a specific caller
|
|
1208
|
+
* pin/prefer an engine for its call only — e.g. the long/video path passes `auto` to lean on
|
|
1209
|
+
* the in-process Rust sherpa-rs engine — without changing the global STT default. */
|
|
1210
|
+
export async function transcribeWav(wav, engineOverride) {
|
|
1211
|
+
return normalizeSpeechTranscript(await transcribeWavRaw(wav, engineOverride)).text;
|
|
1212
|
+
}
|
|
33
1213
|
export function wireSpeechReaction(options = {}) {
|
|
34
1214
|
const bus = getGlobalEventBus();
|
|
35
|
-
const
|
|
1215
|
+
const env = options.env ?? process.env;
|
|
1216
|
+
const debounceMs = options.debounceMs ?? resolveSpeechDebounceMs(env);
|
|
1217
|
+
const incompleteTurnHoldMs = options.incompleteTurnHoldMs ?? resolveIncompleteTurnHoldMs(env);
|
|
36
1218
|
const now = options.now ?? (() => Date.now());
|
|
37
|
-
const
|
|
1219
|
+
const customTranscribe = options.transcriber;
|
|
1220
|
+
const turnCoordinator = getVoiceTurnCoordinator();
|
|
1221
|
+
const sensoryBackchannelEnabled = env.CODEBUDDY_SENSORY_BACKCHANNEL === 'true' && Boolean(options.onConversationCue);
|
|
1222
|
+
const conversationCues = createConversationCueController({
|
|
1223
|
+
env,
|
|
1224
|
+
canPlay: () => !isSpeaking(now()),
|
|
1225
|
+
...(options.onConversationCue ? { player: options.onConversationCue } : {}),
|
|
1226
|
+
});
|
|
38
1227
|
let lastAt = Number.NEGATIVE_INFINITY;
|
|
1228
|
+
let lastSttFailureReplyAt = Number.NEGATIVE_INFINITY;
|
|
1229
|
+
let sttFailureCount = 0;
|
|
1230
|
+
const sttFailureReplyWindowMs = Math.max(debounceMs, DEFAULT_SPEECH_DEBOUNCE_MS);
|
|
39
1231
|
let inFlight = false;
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
1232
|
+
let activeWav;
|
|
1233
|
+
let activeTurnId;
|
|
1234
|
+
let disposed = false;
|
|
1235
|
+
let liveSeq = 0; // unique dedup key for live-mic finals (there's no WAV to key on)
|
|
1236
|
+
let turnSeq = 0;
|
|
1237
|
+
let pendingSpeechStartedAtMs;
|
|
1238
|
+
let pendingSpeechTurnId;
|
|
1239
|
+
let pendingSpeechPartialText;
|
|
1240
|
+
let bargedSpeechTurnId;
|
|
1241
|
+
let suspectedOwnPlaybackTurnId;
|
|
1242
|
+
let leakagePlaybackStartedAtMs;
|
|
1243
|
+
let leakageSamples = [];
|
|
1244
|
+
const resetLeakageReference = (playbackStartedAtMs) => {
|
|
1245
|
+
leakagePlaybackStartedAtMs = playbackStartedAtMs;
|
|
1246
|
+
leakageSamples = [];
|
|
1247
|
+
};
|
|
1248
|
+
const payloadRms = (payload) => {
|
|
1249
|
+
for (const key of ['avgRms', 'rms', 'peakRms']) {
|
|
1250
|
+
const value = payload[key];
|
|
1251
|
+
if (typeof value === 'number' && Number.isFinite(value) && value > 0)
|
|
1252
|
+
return value;
|
|
1253
|
+
}
|
|
1254
|
+
return undefined;
|
|
1255
|
+
};
|
|
1256
|
+
const payloadNoiseFloorRms = (payload) => {
|
|
1257
|
+
const value = payload.noiseFloorRms;
|
|
1258
|
+
return typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : undefined;
|
|
1259
|
+
};
|
|
1260
|
+
/**
|
|
1261
|
+
* Measure the playback leakage reference from the first 300 ms of reported mic energy.
|
|
1262
|
+
* A missing reference fails closed; the independent 250 ms duration path remains available.
|
|
1263
|
+
*/
|
|
1264
|
+
const shouldTriggerAcousticBargeIn = (payload, speechStartedAtMs) => {
|
|
1265
|
+
const eligibility = assessAcousticBargeInEligibility(payload, env);
|
|
1266
|
+
if (!voiceBargeInEnabled(env)
|
|
1267
|
+
|| !eligibility.trustedAec
|
|
1268
|
+
|| speechStartedAtMs === undefined)
|
|
1269
|
+
return false;
|
|
1270
|
+
const timing = measureVoiceResumeTiming(speechStartedAtMs);
|
|
1271
|
+
if (timing?.kind !== 'during_playback') {
|
|
1272
|
+
resetLeakageReference();
|
|
1273
|
+
return false;
|
|
1274
|
+
}
|
|
1275
|
+
const playbackStartedAtMs = speechStartedAtMs - timing.afterPlaybackStartMs;
|
|
1276
|
+
if (leakagePlaybackStartedAtMs !== playbackStartedAtMs) {
|
|
1277
|
+
resetLeakageReference(playbackStartedAtMs);
|
|
1278
|
+
}
|
|
1279
|
+
const rms = payloadRms(payload);
|
|
1280
|
+
const noiseFloorRms = payloadNoiseFloorRms(payload);
|
|
1281
|
+
const referenceSample = noiseFloorRms ?? rms;
|
|
1282
|
+
// The VAD's calibrated floor is already a leakage measurement. Use it at once when the
|
|
1283
|
+
// speech-start sample clears the margin; otherwise collect the first 300 ms before deciding.
|
|
1284
|
+
if (eligibility.sustainedSpeech
|
|
1285
|
+
&& rms !== undefined && noiseFloorRms !== undefined
|
|
1286
|
+
&& exceedsVoiceLeakageMargin(rms, noiseFloorRms, resolveVoiceBargeInMarginDb(env))) {
|
|
1287
|
+
return true;
|
|
1288
|
+
}
|
|
1289
|
+
if (timing.afterPlaybackStartMs <= VOICE_BARGEIN_LEAKAGE_REFERENCE_MS) {
|
|
1290
|
+
if (referenceSample !== undefined)
|
|
1291
|
+
leakageSamples.push(referenceSample);
|
|
1292
|
+
return false;
|
|
1293
|
+
}
|
|
1294
|
+
const leakageRms = leakageSamples.length > 0
|
|
1295
|
+
? leakageSamples.reduce((sum, sample) => sum + sample, 0) / leakageSamples.length
|
|
1296
|
+
: noiseFloorRms;
|
|
1297
|
+
if (!eligibility.sustainedSpeech || rms === undefined || leakageRms === undefined)
|
|
1298
|
+
return false;
|
|
1299
|
+
return exceedsVoiceLeakageMargin(rms, leakageRms, resolveVoiceBargeInMarginDb(env));
|
|
1300
|
+
};
|
|
1301
|
+
let pendingSpeech = null;
|
|
1302
|
+
let heldLiveTurn = null;
|
|
1303
|
+
const speakSttFailure = async (failure) => {
|
|
1304
|
+
if (options.onSpeechError) {
|
|
1305
|
+
return (await options.onSpeechError(failure)) === true;
|
|
1306
|
+
}
|
|
1307
|
+
// The ordinary `onHeard` path may invoke an LLM. STT failures must use only the
|
|
1308
|
+
// existing local speech path, and only when this wire is actually configured to speak.
|
|
1309
|
+
if (!options.onHeard)
|
|
1310
|
+
return false;
|
|
1311
|
+
try {
|
|
1312
|
+
const { sayNow } = await import('./voice-loop.js');
|
|
1313
|
+
return await sayNow(STT_FAILURE_REPLY, { phoneDelivery: 'never' });
|
|
1314
|
+
}
|
|
1315
|
+
catch (error) {
|
|
1316
|
+
logger.warn(`[speech] local STT failure recovery failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1317
|
+
return false;
|
|
1318
|
+
}
|
|
1319
|
+
};
|
|
1320
|
+
const heardHandler = options.onHeard;
|
|
1321
|
+
const cleanupSpeechJob = async (job) => {
|
|
1322
|
+
// `presetText` identifies buddy-sense's WAV-free live path. Only the
|
|
1323
|
+
// Python fallback's batch jobs own a disposable source file.
|
|
1324
|
+
if (job.presetText === undefined)
|
|
1325
|
+
await removeProcessedCompanionWav(job.wav);
|
|
1326
|
+
};
|
|
1327
|
+
const queuePendingSpeech = (job) => {
|
|
1328
|
+
const superseded = pendingSpeech;
|
|
1329
|
+
pendingSpeech = job;
|
|
1330
|
+
if (superseded && superseded.wav !== job.wav) {
|
|
1331
|
+
void cleanupSpeechJob(superseded);
|
|
1332
|
+
}
|
|
1333
|
+
};
|
|
1334
|
+
const startSpeechJob = (job, bypassDebounce = false) => {
|
|
47
1335
|
const t = now();
|
|
48
|
-
|
|
1336
|
+
const voiceResume = job.speechStartedAtMs !== undefined
|
|
1337
|
+
? measureVoiceResumeTiming(job.speechStartedAtMs)
|
|
1338
|
+
: undefined;
|
|
1339
|
+
const aecActive = job.p.payload?.aecActive === true;
|
|
1340
|
+
const aecTrusted = isSensoryAecTrusted(aecActive, env);
|
|
1341
|
+
// A turn that already barged in (CONV2) has stopped the playback: hear it. Otherwise the
|
|
1342
|
+
// half-duplex guard only opens for an explicitly trusted AEC (SENSE1) — never on aecActive alone.
|
|
1343
|
+
// A capture that STARTED after normal playback ended is different: transcribe it so the
|
|
1344
|
+
// in-memory fingerprint can distinguish room echo from a genuinely new quick reply.
|
|
1345
|
+
const bargedIn = job.turnId !== undefined && bargedSpeechTurnId === job.turnId;
|
|
1346
|
+
const canDiscriminateEchoTail = voiceResume?.kind === 'echo_tail';
|
|
1347
|
+
if (isSpeaking(t) && !aecTrusted && !bargedIn && !canDiscriminateEchoTail) {
|
|
1348
|
+
void cleanupSpeechJob(job);
|
|
1349
|
+
return; // half-duplex: ignore the mic while the robot is speaking (+ echo tail)
|
|
1350
|
+
}
|
|
1351
|
+
if (!bypassDebounce && t - lastAt < debounceMs) {
|
|
1352
|
+
void cleanupSpeechJob(job);
|
|
49
1353
|
return; // one transcription per utterance
|
|
50
|
-
|
|
51
|
-
return; // a prior STT (faster-whisper, seconds) is still running
|
|
1354
|
+
}
|
|
52
1355
|
lastAt = t;
|
|
53
1356
|
inFlight = true;
|
|
1357
|
+
activeWav = job.wav;
|
|
1358
|
+
const turnId = job.turnId ?? `voice_${t}_${++turnSeq}`;
|
|
1359
|
+
if (!job.turnId) {
|
|
1360
|
+
turnCoordinator.transition(turnId, 'listening', {
|
|
1361
|
+
aecActive: job.p.payload?.aecActive === true,
|
|
1362
|
+
});
|
|
1363
|
+
}
|
|
1364
|
+
turnCoordinator.transition(turnId, 'transcribing');
|
|
1365
|
+
activeTurnId = turnId;
|
|
54
1366
|
void (async () => {
|
|
1367
|
+
const payload = job.p.payload ?? {};
|
|
1368
|
+
const eventTimestamp = finiteTimestamp(job.p.receivedAt);
|
|
1369
|
+
const captureStartedAtMs = job.speechStartedAtMs ?? finiteTimestamp(payload.startedAtMs);
|
|
1370
|
+
const captureEndedAtMs = finiteTimestamp(payload.endedAtMs) ?? finiteTimestamp(job.p.tsMs);
|
|
1371
|
+
const endpointMs = finiteTimestamp(payload.endpointMs);
|
|
1372
|
+
const decodeMs = finiteTimestamp(payload.decodeMs);
|
|
1373
|
+
const turnDetectionMs = finiteTimestamp(payload.turnDetectionMs);
|
|
1374
|
+
const transcribeStartMs = now();
|
|
1375
|
+
let sttMs = 0;
|
|
1376
|
+
let decisionMs = 0;
|
|
1377
|
+
let actionMs = 0;
|
|
1378
|
+
let decisionReason;
|
|
1379
|
+
let spoke = false; // did the robot actually emit audio this turn? gates the echo re-stamp
|
|
1380
|
+
let responseTiming;
|
|
1381
|
+
let audioScene;
|
|
55
1382
|
try {
|
|
56
|
-
|
|
57
|
-
|
|
1383
|
+
// Live-mic path (buddy-sense `live-audio`): the daemon already decoded the
|
|
1384
|
+
// utterance in-process, so the transcript rides in the event payload — no
|
|
1385
|
+
// WAV, no STT here. Everything downstream (respond gate, onHeard, percept,
|
|
1386
|
+
// debounce/echo guard) is shared with the WAV path.
|
|
1387
|
+
let rawText = '';
|
|
1388
|
+
let actualSttEngine;
|
|
1389
|
+
let sttFailure;
|
|
1390
|
+
try {
|
|
1391
|
+
if (job.presetText !== undefined) {
|
|
1392
|
+
rawText = job.presetText;
|
|
1393
|
+
}
|
|
1394
|
+
else if (customTranscribe) {
|
|
1395
|
+
rawText = await customTranscribe(job.wav);
|
|
1396
|
+
}
|
|
1397
|
+
else {
|
|
1398
|
+
const outcome = await transcribeWavWithMetadata(job.wav);
|
|
1399
|
+
rawText = outcome.text;
|
|
1400
|
+
actualSttEngine = outcome.engine;
|
|
1401
|
+
}
|
|
1402
|
+
}
|
|
1403
|
+
catch (error) {
|
|
1404
|
+
sttFailure = error instanceof Error ? error : new Error(String(error));
|
|
1405
|
+
}
|
|
1406
|
+
const normalizedText = sttFailure
|
|
1407
|
+
? { text: '' }
|
|
1408
|
+
: normalizeSpeechTranscript(rawText);
|
|
1409
|
+
const text = normalizedText.text;
|
|
1410
|
+
const ingestMs = elapsedSince(transcribeStartMs, now);
|
|
1411
|
+
// On live `transcript_final`, the real decode happened upstream in
|
|
1412
|
+
// buddy-sense. Report its payload timing instead of the near-zero cost of
|
|
1413
|
+
// copying preset text into the brain.
|
|
1414
|
+
sttMs = job.presetText !== undefined && decodeMs !== undefined ? decodeMs : ingestMs;
|
|
1415
|
+
const perceptModule = import('../companion/percepts.js');
|
|
1416
|
+
const latencyPayload = {
|
|
1417
|
+
...(captureStartedAtMs !== undefined ? { captureStartedAtMs } : {}),
|
|
1418
|
+
...(captureEndedAtMs !== undefined ? { captureEndedAtMs } : {}),
|
|
1419
|
+
...(eventTimestamp !== undefined ? { eventReceivedAtMs: eventTimestamp } : {}),
|
|
1420
|
+
transcribeStartMs,
|
|
1421
|
+
sttMs,
|
|
1422
|
+
...(job.presetText !== undefined ? { ingestMs } : {}),
|
|
1423
|
+
...(endpointMs !== undefined ? { endpointMs } : {}),
|
|
1424
|
+
...(decodeMs !== undefined ? { decodeMs } : {}),
|
|
1425
|
+
...(turnDetectionMs !== undefined ? { turnDetectionMs } : {}),
|
|
1426
|
+
...(endpointMs !== undefined || decodeMs !== undefined || turnDetectionMs !== undefined
|
|
1427
|
+
? {
|
|
1428
|
+
inputReadyMs: (endpointMs ?? 0)
|
|
1429
|
+
+ (turnDetectionMs ?? 0)
|
|
1430
|
+
+ (job.presetText !== undefined ? sttMs : (decodeMs ?? 0) + sttMs),
|
|
1431
|
+
}
|
|
1432
|
+
: {}),
|
|
1433
|
+
decisionMs,
|
|
1434
|
+
actionMs,
|
|
1435
|
+
totalMs: sttMs,
|
|
1436
|
+
...(eventTimestamp !== undefined
|
|
1437
|
+
? { eventToSttStartMs: Math.max(0, transcribeStartMs - eventTimestamp) }
|
|
1438
|
+
: {}),
|
|
1439
|
+
};
|
|
1440
|
+
const capturePayload = {
|
|
1441
|
+
device: payload.device,
|
|
1442
|
+
rms: payload.rms,
|
|
1443
|
+
peakRms: payload.peakRms ?? payload.rms,
|
|
1444
|
+
avgRms: payload.avgRms,
|
|
1445
|
+
ms: payload.ms,
|
|
1446
|
+
audioMs: payload.audioMs,
|
|
1447
|
+
endpointMs: payload.endpointMs,
|
|
1448
|
+
decodeMs: payload.decodeMs,
|
|
1449
|
+
turnDetector: payload.turnDetector,
|
|
1450
|
+
turnProbability: payload.turnProbability,
|
|
1451
|
+
turnDetectionMs: payload.turnDetectionMs,
|
|
1452
|
+
turnForcedAfterHold: payload.turnForcedAfterHold,
|
|
1453
|
+
writeMs: payload.writeMs,
|
|
1454
|
+
vadHangMs: payload.vadHangMs,
|
|
1455
|
+
endedReason: payload.endedReason,
|
|
1456
|
+
sampleRate: payload.sampleRate,
|
|
1457
|
+
rmsOn: payload.rmsOn,
|
|
1458
|
+
rmsOff: payload.rmsOff,
|
|
1459
|
+
aecActive: payload.aecActive === true,
|
|
1460
|
+
captureSourceClass: payload.captureSourceClass,
|
|
1461
|
+
};
|
|
1462
|
+
if (sttFailure) {
|
|
1463
|
+
sttFailureCount += 1;
|
|
1464
|
+
const cause = sttFailure.message.slice(0, 300) || sttFailure.name;
|
|
1465
|
+
const signal = job.presetText === undefined
|
|
1466
|
+
? realSpeechCapture(job.wav, payload)
|
|
1467
|
+
: undefined;
|
|
1468
|
+
let recoverySpoke = false;
|
|
1469
|
+
if (signal
|
|
1470
|
+
&& now() - lastSttFailureReplyAt >= sttFailureReplyWindowMs) {
|
|
1471
|
+
lastSttFailureReplyAt = now();
|
|
1472
|
+
try {
|
|
1473
|
+
recoverySpoke = await speakSttFailure({
|
|
1474
|
+
wav: job.wav,
|
|
1475
|
+
cause,
|
|
1476
|
+
count: sttFailureCount,
|
|
1477
|
+
durationMs: signal.durationMs,
|
|
1478
|
+
rms: signal.rms,
|
|
1479
|
+
});
|
|
1480
|
+
}
|
|
1481
|
+
catch (error) {
|
|
1482
|
+
logger.warn(`[speech] STT failure recovery threw: ${error instanceof Error ? error.message : String(error)}`);
|
|
1483
|
+
}
|
|
1484
|
+
}
|
|
1485
|
+
spoke = recoverySpoke;
|
|
1486
|
+
turnCoordinator.transition(turnId, 'failed', {
|
|
1487
|
+
errorCategory: 'stt',
|
|
1488
|
+
sttMs,
|
|
1489
|
+
spoke: recoverySpoke,
|
|
1490
|
+
});
|
|
1491
|
+
logger.warn(`[speech] STT failed (#${sttFailureCount}): ${cause}`
|
|
1492
|
+
+ (signal ? `; local recovery spoke=${recoverySpoke}` : '; no real speech capture for local recovery'));
|
|
1493
|
+
await (await perceptModule).recordCompanionPercept({
|
|
1494
|
+
modality: 'hearing',
|
|
1495
|
+
source: 'sensory_speech_reaction',
|
|
1496
|
+
summary: 'Speech captured; STT failed',
|
|
1497
|
+
confidence: 0.35,
|
|
1498
|
+
payload: {
|
|
1499
|
+
text: '',
|
|
1500
|
+
wav: job.wav,
|
|
1501
|
+
responded: recoverySpoke,
|
|
1502
|
+
sttFailure: true,
|
|
1503
|
+
sttEmpty: true,
|
|
1504
|
+
sttEmptyReason: 'error',
|
|
1505
|
+
sttFailureCause: cause,
|
|
1506
|
+
sttFailureCount,
|
|
1507
|
+
...(recoverySpoke ? { sttFailureRecovery: STT_FAILURE_REPLY } : {}),
|
|
1508
|
+
latency: latencyPayload,
|
|
1509
|
+
capture: capturePayload,
|
|
1510
|
+
},
|
|
1511
|
+
tags: ['speech', 'stt', 'latency', 'error'],
|
|
1512
|
+
}, options.cwd ? { cwd: options.cwd } : {});
|
|
1513
|
+
return;
|
|
1514
|
+
}
|
|
1515
|
+
if (!text) {
|
|
1516
|
+
let repairAddressed = false;
|
|
1517
|
+
const repairCaptureAtMs = captureStartedAtMs ?? transcribeStartMs;
|
|
1518
|
+
const recentOwnPlaybackRisk = !isSensoryAecTrusted(payload.aecActive === true, env)
|
|
1519
|
+
&& hasRecentSpokenReference(repairCaptureAtMs);
|
|
1520
|
+
if (env.CODEBUDDY_SENSORY_REPAIR === 'true' && options.onConversationCue) {
|
|
1521
|
+
const repairHintIsOwnEcho = job.repairAddressHint
|
|
1522
|
+
? classifyRecentVoiceEcho(job.repairAddressHint, repairCaptureAtMs) === 'echo'
|
|
1523
|
+
: false;
|
|
1524
|
+
if (job.repairAddressHint && !repairHintIsOwnEcho && options.isAddressed) {
|
|
1525
|
+
try {
|
|
1526
|
+
repairAddressed = await options.isAddressed(job.repairAddressHint);
|
|
1527
|
+
}
|
|
1528
|
+
catch {
|
|
1529
|
+
repairAddressed = false;
|
|
1530
|
+
}
|
|
1531
|
+
}
|
|
1532
|
+
const attention = options.getAttentionSnapshot?.();
|
|
1533
|
+
repairAddressed ||= !recentOwnPlaybackRisk
|
|
1534
|
+
&& attention?.engaged === true
|
|
1535
|
+
&& attention.source === 'addressed';
|
|
1536
|
+
}
|
|
1537
|
+
const repairStartedAt = now();
|
|
1538
|
+
const repairSpoke = repairAddressed
|
|
1539
|
+
? await conversationCues.playRepair(turnId)
|
|
1540
|
+
: false;
|
|
1541
|
+
actionMs = repairAddressed ? elapsedSince(repairStartedAt, now) : 0;
|
|
1542
|
+
spoke = repairSpoke;
|
|
1543
|
+
turnCoordinator.transition(turnId, repairSpoke ? 'completed' : 'suppressed', {
|
|
1544
|
+
suppressionReason: repairAddressed
|
|
1545
|
+
? repairSpoke ? undefined : 'repair-cue-unavailable'
|
|
1546
|
+
: normalizedText.filteredReason ?? 'stt-empty',
|
|
1547
|
+
sttMs,
|
|
1548
|
+
spoke: repairSpoke,
|
|
1549
|
+
});
|
|
1550
|
+
const emptyReason = normalizedText.filteredReason ?? 'empty';
|
|
1551
|
+
logger.info(`[speech] empty transcript (${sttMs}ms STT, ${emptyReason})`);
|
|
1552
|
+
await (await perceptModule).recordCompanionPercept({
|
|
1553
|
+
modality: 'hearing',
|
|
1554
|
+
source: 'sensory_speech_reaction',
|
|
1555
|
+
summary: 'Speech captured; STT returned no text',
|
|
1556
|
+
confidence: 0.25,
|
|
1557
|
+
payload: {
|
|
1558
|
+
text: '',
|
|
1559
|
+
...(job.presetText !== undefined ? { live: true } : { wav: job.wav }),
|
|
1560
|
+
responded: repairSpoke,
|
|
1561
|
+
sttEmpty: true,
|
|
1562
|
+
sttEmptyReason: emptyReason,
|
|
1563
|
+
...(repairAddressed ? { repairTriggered: true, repairSpoke } : {}),
|
|
1564
|
+
...(normalizedText.filteredReason ? { rawText: rawText.trim().slice(0, 240) } : {}),
|
|
1565
|
+
latency: latencyPayload,
|
|
1566
|
+
capture: capturePayload,
|
|
1567
|
+
},
|
|
1568
|
+
tags: ['speech', 'stt', 'latency', 'empty'],
|
|
1569
|
+
}, options.cwd ? { cwd: options.cwd } : {});
|
|
58
1570
|
return;
|
|
59
|
-
|
|
60
|
-
|
|
1571
|
+
}
|
|
1572
|
+
const playbackCaptureKind = voiceResume?.kind === 'during_playback'
|
|
1573
|
+
|| voiceResume?.kind === 'echo_tail'
|
|
1574
|
+
? voiceResume.kind
|
|
1575
|
+
: undefined;
|
|
1576
|
+
const echoClassification = classifyRecentVoiceEcho(text, captureStartedAtMs ?? transcribeStartMs);
|
|
1577
|
+
const explicitBargeIn = playbackCaptureKind
|
|
1578
|
+
? shouldTriggerVoiceBargeIn(text, payload, env)
|
|
1579
|
+
// A turn that already cut the playback acoustically (CONV2 speech_start barge-in)
|
|
1580
|
+
// counts as an explicit interruption: the robot is no longer speaking over it.
|
|
1581
|
+
|| (job.turnId !== undefined && bargedSpeechTurnId === job.turnId)
|
|
1582
|
+
: false;
|
|
1583
|
+
const ownEcho = echoClassification === 'echo';
|
|
1584
|
+
const suppressPlaybackCapture = ownEcho || (playbackCaptureKind !== undefined
|
|
1585
|
+
? shouldSuppressPlaybackCapture(playbackCaptureKind, echoClassification, explicitBargeIn, isSensoryAecTrusted(payload.aecActive === true, env))
|
|
1586
|
+
: false);
|
|
1587
|
+
if (suppressPlaybackCapture) {
|
|
1588
|
+
const suppressionReason = ownEcho && playbackCaptureKind === undefined
|
|
1589
|
+
? 'own_echo'
|
|
1590
|
+
: playbackCaptureKind === 'during_playback'
|
|
1591
|
+
? ownEcho
|
|
1592
|
+
? 'during_playback_echo'
|
|
1593
|
+
: 'during_playback_non_explicit'
|
|
1594
|
+
: `echo_tail_${echoClassification}`;
|
|
1595
|
+
if (ownEcho) {
|
|
1596
|
+
logger.info('[speech] dropped own echo');
|
|
1597
|
+
}
|
|
1598
|
+
else {
|
|
1599
|
+
logger.info(`[speech] suppressed playback capture reason=${suppressionReason}`);
|
|
1600
|
+
}
|
|
1601
|
+
turnCoordinator.transition(turnId, 'suppressed', {
|
|
1602
|
+
suppressionReason,
|
|
1603
|
+
sttMs,
|
|
1604
|
+
scene: playbackCaptureKind ? 'assistant_playback' : 'assistant_echo',
|
|
1605
|
+
sceneConfidence: echoClassification === 'echo' ? 0.98 : 0.8,
|
|
1606
|
+
});
|
|
1607
|
+
await (await perceptModule).recordCompanionPercept({
|
|
1608
|
+
modality: 'hearing',
|
|
1609
|
+
source: 'sensory_speech_reaction',
|
|
1610
|
+
summary: playbackCaptureKind === 'during_playback'
|
|
1611
|
+
? 'Speech captured during loudspeaker playback suppressed'
|
|
1612
|
+
: 'Likely loudspeaker echo suppressed',
|
|
1613
|
+
confidence: echoClassification === 'echo' ? 0.95 : 0.75,
|
|
1614
|
+
payload: {
|
|
1615
|
+
responded: false,
|
|
1616
|
+
playbackEcho: echoClassification === 'echo',
|
|
1617
|
+
playbackCaptureSuppressed: true,
|
|
1618
|
+
suppressionReason,
|
|
1619
|
+
echoClassification,
|
|
1620
|
+
...(voiceResume ? { turnTaking: voiceResume } : {}),
|
|
1621
|
+
latency: latencyPayload,
|
|
1622
|
+
capture: capturePayload,
|
|
1623
|
+
},
|
|
1624
|
+
tags: ['speech', 'echo', 'playback-capture', 'turn-taking'],
|
|
1625
|
+
}, options.cwd ? { cwd: options.cwd } : {});
|
|
1626
|
+
return;
|
|
1627
|
+
}
|
|
1628
|
+
const configuredPlan = resolveSpeechTranscriptionPlan();
|
|
1629
|
+
const sttEngine = actualSttEngine ?? (typeof payload.sttEngine === 'string'
|
|
1630
|
+
? payload.sttEngine
|
|
1631
|
+
: job.presetText !== undefined
|
|
1632
|
+
? 'sherpa-rs'
|
|
1633
|
+
: configuredPlan.effectiveEngine);
|
|
1634
|
+
const sttLanguage = typeof payload.sttLanguage === 'string'
|
|
1635
|
+
? payload.sttLanguage
|
|
1636
|
+
: configuredPlan.language;
|
|
1637
|
+
const sttModel = typeof payload.sttModel === 'string'
|
|
1638
|
+
? payload.sttModel
|
|
1639
|
+
: sttEngine === 'faster-whisper'
|
|
1640
|
+
? process.env.CODEBUDDY_SPEECH_MODEL?.trim() || 'base'
|
|
1641
|
+
: resolveParakeetModelDir();
|
|
1642
|
+
const hotwordsState = payload.hotwordsApplied === false
|
|
1643
|
+
? 'ignored'
|
|
1644
|
+
: sttEngine === 'faster-whisper' && defaultSpeechHotwords()
|
|
1645
|
+
? 'applied'
|
|
1646
|
+
: 'none';
|
|
1647
|
+
logger.info(`[speech] heard (${sttMs}ms STT, engine=${sttEngine}, model=${sttModel}, language=${sttLanguage}, hotwords=${hotwordsState}) → ${text}`);
|
|
1648
|
+
let backchannel = null;
|
|
1649
|
+
const turnContext = {
|
|
1650
|
+
turnId,
|
|
1651
|
+
...(finiteTimestamp(payload.audioMs) !== undefined
|
|
1652
|
+
? { audioMs: finiteTimestamp(payload.audioMs) }
|
|
1653
|
+
: {}),
|
|
1654
|
+
...(finiteTimestamp(payload.ms) !== undefined
|
|
1655
|
+
? { captureMs: finiteTimestamp(payload.ms) }
|
|
1656
|
+
: {}),
|
|
1657
|
+
...(captureStartedAtMs !== undefined ? { speechStartedAtMs: captureStartedAtMs } : {}),
|
|
1658
|
+
...(captureEndedAtMs !== undefined ? { speechEndedAtMs: captureEndedAtMs } : {}),
|
|
1659
|
+
...(sensoryBackchannelEnabled
|
|
1660
|
+
? { onResponseAudioStart: () => backchannel?.cancel() }
|
|
1661
|
+
: {}),
|
|
1662
|
+
};
|
|
1663
|
+
let acceptedForSemanticIngress = true;
|
|
1664
|
+
let responded = Boolean(options.onHeard);
|
|
1665
|
+
let repaired = false;
|
|
1666
|
+
let repairReason;
|
|
1667
|
+
turnCoordinator.transition(turnId, 'deciding', {
|
|
1668
|
+
sttMs,
|
|
1669
|
+
wordCount: text.match(/[\p{L}\p{N}]+/gu)?.length ?? 0,
|
|
1670
|
+
});
|
|
1671
|
+
// Human-like gate: raw observation remains continuous; semantic dialogue and
|
|
1672
|
+
// speech proceed only when this turn was accepted as addressed/warranted.
|
|
1673
|
+
if (options.shouldRespond) {
|
|
1674
|
+
const decisionStartMs = now();
|
|
1675
|
+
const decision = await options.shouldRespond(text);
|
|
1676
|
+
decisionMs = elapsedSince(decisionStartMs, now);
|
|
1677
|
+
decisionReason = decision.reason;
|
|
1678
|
+
turnCoordinator.transition(turnId, 'deciding', {
|
|
1679
|
+
decisionReason,
|
|
1680
|
+
decisionMs,
|
|
1681
|
+
sttMs,
|
|
1682
|
+
});
|
|
1683
|
+
acceptedForSemanticIngress = decision.respond;
|
|
1684
|
+
if (!decision.respond) {
|
|
1685
|
+
logger.info(`[speech] silent (${decision.reason}, decision ${decisionMs}ms)`);
|
|
1686
|
+
responded = false;
|
|
1687
|
+
}
|
|
1688
|
+
else {
|
|
1689
|
+
responded = Boolean(options.onHeard);
|
|
1690
|
+
logger.info(`[speech] responding (${decision.reason}, decision ${decisionMs}ms)`);
|
|
1691
|
+
}
|
|
1692
|
+
const attention = options.getAttentionSnapshot?.();
|
|
1693
|
+
if (attention)
|
|
1694
|
+
turnCoordinator.updateAttention(attention);
|
|
1695
|
+
}
|
|
1696
|
+
audioScene = assessAudioScene({
|
|
1697
|
+
transcript: text,
|
|
1698
|
+
...(decisionReason ? { decisionReason } : {}),
|
|
1699
|
+
...(playbackCaptureKind ? { playbackCaptureKind } : {}),
|
|
1700
|
+
...(echoClassification ? { echoClassification } : {}),
|
|
1701
|
+
rms: finiteTimestamp(payload.rms),
|
|
1702
|
+
rmsOn: finiteTimestamp(payload.rmsOn),
|
|
1703
|
+
audioMs: finiteTimestamp(payload.audioMs),
|
|
1704
|
+
turnDetector: typeof payload.turnDetector === 'string' ? payload.turnDetector : undefined,
|
|
1705
|
+
speakerCount: finiteTimestamp(payload.speakerCount),
|
|
1706
|
+
aecActive: payload.aecActive === true,
|
|
1707
|
+
});
|
|
1708
|
+
turnCoordinator.transition(turnId, 'deciding', {
|
|
1709
|
+
...(decisionReason ? { decisionReason } : {}),
|
|
1710
|
+
scene: audioScene.scene,
|
|
1711
|
+
sceneConfidence: audioScene.confidence,
|
|
1712
|
+
decisionMs,
|
|
1713
|
+
sttMs,
|
|
1714
|
+
});
|
|
1715
|
+
const confidence = finiteTimestamp(payload.confidence);
|
|
1716
|
+
if (acceptedForSemanticIngress
|
|
1717
|
+
&& decisionReason === 'addressed'
|
|
1718
|
+
&& env.CODEBUDDY_SENSORY_REPAIR === 'true'
|
|
1719
|
+
&& options.onConversationCue
|
|
1720
|
+
&& shouldRepairTranscript(text, confidence)) {
|
|
1721
|
+
repairReason = (text.match(/[\p{L}\p{N}]+/gu)?.length ?? 0) <= 2
|
|
1722
|
+
? 'short'
|
|
1723
|
+
: 'low-confidence';
|
|
1724
|
+
const repairStartedAt = now();
|
|
1725
|
+
spoke = await conversationCues.playRepair(turnId);
|
|
1726
|
+
actionMs = elapsedSince(repairStartedAt, now);
|
|
1727
|
+
repaired = true;
|
|
1728
|
+
responded = spoke;
|
|
1729
|
+
acceptedForSemanticIngress = false;
|
|
1730
|
+
}
|
|
1731
|
+
if (acceptedForSemanticIngress && options.onRecognizedTurn) {
|
|
1732
|
+
try {
|
|
1733
|
+
const ingress = options.onRecognizedTurn({ turnId, text, context: turnContext });
|
|
1734
|
+
void Promise.resolve(ingress).catch((error) => {
|
|
1735
|
+
logger.warn(`[speech] background turn ingress failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1736
|
+
});
|
|
1737
|
+
}
|
|
1738
|
+
catch (error) {
|
|
1739
|
+
logger.warn(`[speech] background turn ingress failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1740
|
+
}
|
|
1741
|
+
}
|
|
1742
|
+
if (repaired) {
|
|
1743
|
+
turnCoordinator.transition(turnId, spoke ? 'completed' : 'suppressed', {
|
|
1744
|
+
decisionReason,
|
|
1745
|
+
suppressionReason: spoke ? undefined : 'repair-cue-unavailable',
|
|
1746
|
+
spoke,
|
|
1747
|
+
totalMs: elapsedSince(transcribeStartMs, now),
|
|
1748
|
+
});
|
|
1749
|
+
}
|
|
1750
|
+
else if (responded) {
|
|
1751
|
+
if (sensoryBackchannelEnabled && decisionReason === 'addressed') {
|
|
1752
|
+
backchannel = conversationCues.armBackchannel(turnId);
|
|
1753
|
+
}
|
|
1754
|
+
turnCoordinator.transition(turnId, 'thinking', {
|
|
1755
|
+
decisionReason,
|
|
1756
|
+
decisionMs,
|
|
1757
|
+
});
|
|
1758
|
+
const actionStartMs = now();
|
|
1759
|
+
try {
|
|
1760
|
+
await options.onHeard?.(text, turnContext);
|
|
1761
|
+
}
|
|
1762
|
+
finally {
|
|
1763
|
+
backchannel?.cancel();
|
|
1764
|
+
}
|
|
1765
|
+
actionMs = elapsedSince(actionStartMs, now);
|
|
1766
|
+
responseTiming = options.getResponseTiming?.();
|
|
1767
|
+
// The voice handler publishes the same fact through getResponseTiming or its
|
|
1768
|
+
// lastTiming property. An uninstrumented/no-op hook is not evidence of audio.
|
|
1769
|
+
spoke = responseTiming?.spoke ?? heardHandler?.lastTiming?.spoke ?? false;
|
|
1770
|
+
if (!responseTiming) {
|
|
1771
|
+
turnCoordinator.transition(turnId, 'completed', {
|
|
1772
|
+
decisionReason,
|
|
1773
|
+
spoke,
|
|
1774
|
+
totalMs: elapsedSince(transcribeStartMs, now),
|
|
1775
|
+
});
|
|
1776
|
+
}
|
|
1777
|
+
}
|
|
1778
|
+
else {
|
|
1779
|
+
turnCoordinator.transition(turnId, 'suppressed', {
|
|
1780
|
+
suppressionReason: decisionReason ?? 'no-response-handler',
|
|
1781
|
+
decisionMs,
|
|
1782
|
+
sttMs,
|
|
1783
|
+
});
|
|
1784
|
+
}
|
|
1785
|
+
const totalMs = elapsedSince(transcribeStartMs, now);
|
|
1786
|
+
const inputReadyMs = (endpointMs ?? 0)
|
|
1787
|
+
+ (turnDetectionMs ?? 0)
|
|
1788
|
+
+ (job.presetText !== undefined ? sttMs : (decodeMs ?? 0) + sttMs);
|
|
1789
|
+
const perceivedResponseMs = responseTiming?.firstAudioMs !== undefined
|
|
1790
|
+
? inputReadyMs + decisionMs + responseTiming.firstAudioMs
|
|
1791
|
+
: undefined;
|
|
1792
|
+
const perceivedContentResponseMs = responseTiming?.firstContentAudioMs !== undefined
|
|
1793
|
+
? inputReadyMs + decisionMs + responseTiming.firstContentAudioMs
|
|
1794
|
+
: undefined;
|
|
1795
|
+
await (await perceptModule).recordCompanionPercept({
|
|
61
1796
|
modality: 'hearing',
|
|
62
1797
|
source: 'sensory_speech_reaction',
|
|
63
1798
|
summary: `Heard: ${text}`,
|
|
64
1799
|
confidence: 0.8,
|
|
65
|
-
payload: {
|
|
66
|
-
|
|
1800
|
+
payload: {
|
|
1801
|
+
text,
|
|
1802
|
+
...(job.presetText !== undefined ? { live: true } : { wav: job.wav }),
|
|
1803
|
+
responded,
|
|
1804
|
+
stt: {
|
|
1805
|
+
requestedEngine: typeof payload.sttRequestedEngine === 'string'
|
|
1806
|
+
? payload.sttRequestedEngine
|
|
1807
|
+
: configuredPlan.requestedEngine,
|
|
1808
|
+
engine: sttEngine,
|
|
1809
|
+
model: sttModel,
|
|
1810
|
+
language: sttLanguage,
|
|
1811
|
+
hotwords: hotwordsState,
|
|
1812
|
+
...(typeof payload.sttFallbackReason === 'string'
|
|
1813
|
+
? { fallbackReason: payload.sttFallbackReason }
|
|
1814
|
+
: {}),
|
|
1815
|
+
},
|
|
1816
|
+
...(decisionReason ? { decisionReason } : {}),
|
|
1817
|
+
...(repaired ? { repairTriggered: true, repairReason } : {}),
|
|
1818
|
+
latency: {
|
|
1819
|
+
...latencyPayload,
|
|
1820
|
+
decisionMs,
|
|
1821
|
+
actionMs,
|
|
1822
|
+
totalMs,
|
|
1823
|
+
...(responseTiming?.promptReadyMs !== undefined
|
|
1824
|
+
? { promptReadyMs: responseTiming.promptReadyMs }
|
|
1825
|
+
: {}),
|
|
1826
|
+
...(responseTiming?.providerFirstDeltaMs !== undefined
|
|
1827
|
+
? { providerFirstDeltaMs: responseTiming.providerFirstDeltaMs }
|
|
1828
|
+
: {}),
|
|
1829
|
+
...(responseTiming?.generationCompleteMs !== undefined
|
|
1830
|
+
? { generationCompleteMs: responseTiming.generationCompleteMs }
|
|
1831
|
+
: {}),
|
|
1832
|
+
...(responseTiming?.semanticReviewCompleteMs !== undefined
|
|
1833
|
+
? { semanticReviewCompleteMs: responseTiming.semanticReviewCompleteMs }
|
|
1834
|
+
: {}),
|
|
1835
|
+
...(responseTiming?.spokenPrefix
|
|
1836
|
+
? { spokenPrefix: responseTiming.spokenPrefix }
|
|
1837
|
+
: {}),
|
|
1838
|
+
...(responseTiming?.continuation
|
|
1839
|
+
? { continuation: responseTiming.continuation }
|
|
1840
|
+
: {}),
|
|
1841
|
+
...(responseTiming?.firstSafeReleaseMs !== undefined
|
|
1842
|
+
? { firstSafeReleaseMs: responseTiming.firstSafeReleaseMs }
|
|
1843
|
+
: {}),
|
|
1844
|
+
...(responseTiming?.firstTextMs !== undefined
|
|
1845
|
+
? { firstTextMs: responseTiming.firstTextMs }
|
|
1846
|
+
: {}),
|
|
1847
|
+
...(responseTiming?.firstSegmentMs !== undefined
|
|
1848
|
+
? { firstSegmentMs: responseTiming.firstSegmentMs }
|
|
1849
|
+
: {}),
|
|
1850
|
+
...(responseTiming?.firstAudioMs !== undefined
|
|
1851
|
+
? {
|
|
1852
|
+
firstAudioMs: responseTiming.firstAudioMs,
|
|
1853
|
+
perceivedResponseMs,
|
|
1854
|
+
}
|
|
1855
|
+
: {}),
|
|
1856
|
+
...(responseTiming?.firstContentAudioMs !== undefined
|
|
1857
|
+
? {
|
|
1858
|
+
firstContentAudioMs: responseTiming.firstContentAudioMs,
|
|
1859
|
+
perceivedContentResponseMs,
|
|
1860
|
+
}
|
|
1861
|
+
: {}),
|
|
1862
|
+
...(responseTiming?.streamFallbackSegments !== undefined
|
|
1863
|
+
? { streamFallbackSegments: responseTiming.streamFallbackSegments }
|
|
1864
|
+
: {}),
|
|
1865
|
+
...(responseTiming ? { voiceTotalMs: responseTiming.totalMs } : {}),
|
|
1866
|
+
},
|
|
1867
|
+
...(responseTiming ? { responseMode: responseTiming.mode, spoke } : {}),
|
|
1868
|
+
...(responseTiming?.delivery ? { delivery: responseTiming.delivery } : {}),
|
|
1869
|
+
...(audioScene ? { audioScene } : {}),
|
|
1870
|
+
...(voiceResume ? { turnTaking: voiceResume } : {}),
|
|
1871
|
+
capture: {
|
|
1872
|
+
...capturePayload,
|
|
1873
|
+
},
|
|
1874
|
+
},
|
|
1875
|
+
tags: ['speech', 'stt', 'latency'],
|
|
67
1876
|
}, options.cwd ? { cwd: options.cwd } : {});
|
|
68
|
-
|
|
69
|
-
|
|
1877
|
+
if (actionMs > 0 || decisionMs > 0) {
|
|
1878
|
+
logger.info(`[speech] loop timings: stt=${sttMs}ms decision=${decisionMs}ms action=${actionMs}ms total=${totalMs}ms` +
|
|
1879
|
+
(perceivedResponseMs !== undefined
|
|
1880
|
+
? ` perceived=${perceivedResponseMs}ms`
|
|
1881
|
+
: '') +
|
|
1882
|
+
(perceivedContentResponseMs !== undefined
|
|
1883
|
+
? ` perceivedContent=${perceivedContentResponseMs}ms`
|
|
1884
|
+
: '') +
|
|
1885
|
+
(decisionReason ? ` reason=${decisionReason}` : ''));
|
|
1886
|
+
}
|
|
70
1887
|
}
|
|
71
1888
|
catch (err) {
|
|
1889
|
+
turnCoordinator.transition(turnId, 'failed', { errorCategory: 'unknown' });
|
|
72
1890
|
logger.warn(`[speech] reaction failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
73
1891
|
}
|
|
74
1892
|
finally {
|
|
1893
|
+
// The Python ear writes one source WAV per utterance. Delete it on every
|
|
1894
|
+
// terminal path (success, empty STT, error, or teardown during a job).
|
|
1895
|
+
// The helper fails closed for arbitrary paths and honours the debug
|
|
1896
|
+
// retention switch documented at the top of this module.
|
|
1897
|
+
await cleanupSpeechJob(job);
|
|
1898
|
+
// Re-stamp AFTER the full hear→think→speak cycle so the debounce window restarts from
|
|
1899
|
+
// end-of-playback — but ONLY when the robot actually spoke (that's the echo tail we must
|
|
1900
|
+
// not re-hear). After a silent turn (empty/filtered transcript, or the gate vetoed a
|
|
1901
|
+
// reply) there is no echo, so keep the job-start debounce anchor (set at startSpeechJob):
|
|
1902
|
+
// pushing lastAt out by the STT/decision duration would swallow a real address arriving
|
|
1903
|
+
// in the (debounceMs, debounceMs+sttMs] window after a silent turn.
|
|
1904
|
+
if (spoke)
|
|
1905
|
+
lastAt = now();
|
|
75
1906
|
inFlight = false;
|
|
1907
|
+
activeWav = undefined;
|
|
1908
|
+
if (activeTurnId === turnId)
|
|
1909
|
+
activeTurnId = undefined;
|
|
1910
|
+
const nextSpeech = pendingSpeech;
|
|
1911
|
+
pendingSpeech = null;
|
|
1912
|
+
if (nextSpeech && !disposed) {
|
|
1913
|
+
startSpeechJob(nextSpeech, true);
|
|
1914
|
+
}
|
|
76
1915
|
}
|
|
77
1916
|
})();
|
|
1917
|
+
};
|
|
1918
|
+
const id = bus.on('sensory:perception', (evt) => {
|
|
1919
|
+
const p = perceptionOf(evt);
|
|
1920
|
+
if (p.modality !== 'audio')
|
|
1921
|
+
return;
|
|
1922
|
+
// Rust live-audio publishes this on the exact VAD closed→open edge. Use the
|
|
1923
|
+
// human's speaking time for preparation, but keep the response decision on
|
|
1924
|
+
// `transcript_final`: television/noise can warm a standby, never make it talk.
|
|
1925
|
+
if (p.kind === 'speech_start') {
|
|
1926
|
+
const payload = p.payload ?? {};
|
|
1927
|
+
pendingSpeechStartedAtMs = finiteTimestamp(payload.startedAtMs)
|
|
1928
|
+
?? finiteTimestamp(p.receivedAt)
|
|
1929
|
+
?? now();
|
|
1930
|
+
pendingSpeechTurnId = `voice_${pendingSpeechStartedAtMs}_${++turnSeq}`;
|
|
1931
|
+
pendingSpeechPartialText = undefined;
|
|
1932
|
+
bargedSpeechTurnId = undefined;
|
|
1933
|
+
turnCoordinator.transition(pendingSpeechTurnId, 'listening', {
|
|
1934
|
+
aecActive: payload.aecActive === true,
|
|
1935
|
+
});
|
|
1936
|
+
const speechStartedAtMs = pendingSpeechStartedAtMs ?? now();
|
|
1937
|
+
const playbackActive = measureVoiceResumeTiming(speechStartedAtMs)?.kind === 'during_playback';
|
|
1938
|
+
const suspectedOwnPlayback = playbackActive
|
|
1939
|
+
&& payload.aecActive !== true
|
|
1940
|
+
&& (env.CODEBUDDY_SENSORY_REPAIR === 'true' || sensoryBackchannelEnabled)
|
|
1941
|
+
&& hasRecentSpokenReference(speechStartedAtMs);
|
|
1942
|
+
suspectedOwnPlaybackTurnId = suspectedOwnPlayback ? pendingSpeechTurnId : undefined;
|
|
1943
|
+
if (inFlight
|
|
1944
|
+
&& playbackActive
|
|
1945
|
+
&& !suspectedOwnPlayback
|
|
1946
|
+
&& voiceBargeInEnabled(env)
|
|
1947
|
+
&& (shouldTriggerVoiceBargeInOnSpeechStart(payload, env)
|
|
1948
|
+
|| shouldTriggerAcousticBargeIn(payload, speechStartedAtMs))
|
|
1949
|
+
&& (pendingSpeechTurnId === undefined || bargedSpeechTurnId !== pendingSpeechTurnId)) {
|
|
1950
|
+
bargedSpeechTurnId = pendingSpeechTurnId;
|
|
1951
|
+
try {
|
|
1952
|
+
if (options.onBargeInStart) {
|
|
1953
|
+
options.onBargeInStart(payload, activeTurnId);
|
|
1954
|
+
}
|
|
1955
|
+
else {
|
|
1956
|
+
options.onBargeIn?.('', activeTurnId);
|
|
1957
|
+
}
|
|
1958
|
+
}
|
|
1959
|
+
catch {
|
|
1960
|
+
/* interruption is best-effort */
|
|
1961
|
+
}
|
|
1962
|
+
}
|
|
1963
|
+
if (options.onSpeechStart) {
|
|
1964
|
+
void Promise.resolve().then(() => options.onSpeechStart(payload)).catch((error) => {
|
|
1965
|
+
logger.debug('[speech] predictive warmup skipped', {
|
|
1966
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1967
|
+
});
|
|
1968
|
+
});
|
|
1969
|
+
}
|
|
1970
|
+
return;
|
|
1971
|
+
}
|
|
1972
|
+
if (p.kind === 'transcript_partial') {
|
|
1973
|
+
const payload = p.payload ?? {};
|
|
1974
|
+
const text = typeof payload.text === 'string' ? payload.text.trim() : '';
|
|
1975
|
+
if (!text)
|
|
1976
|
+
return;
|
|
1977
|
+
pendingSpeechPartialText = text;
|
|
1978
|
+
const acousticBargeIn = shouldTriggerAcousticBargeIn(payload, pendingSpeechStartedAtMs);
|
|
1979
|
+
if (inFlight &&
|
|
1980
|
+
(options.onBargeIn || options.onBargeInStart) &&
|
|
1981
|
+
(shouldTriggerVoiceBargeIn(text, payload, env) || acousticBargeIn) &&
|
|
1982
|
+
(pendingSpeechTurnId === undefined || bargedSpeechTurnId !== pendingSpeechTurnId)) {
|
|
1983
|
+
if (pendingSpeechTurnId !== undefined)
|
|
1984
|
+
bargedSpeechTurnId = pendingSpeechTurnId;
|
|
1985
|
+
try {
|
|
1986
|
+
if (acousticBargeIn && options.onBargeInStart) {
|
|
1987
|
+
options.onBargeInStart(payload, activeTurnId);
|
|
1988
|
+
}
|
|
1989
|
+
else {
|
|
1990
|
+
options.onBargeIn?.(text, activeTurnId);
|
|
1991
|
+
}
|
|
1992
|
+
}
|
|
1993
|
+
catch {
|
|
1994
|
+
/* interruption is best-effort */
|
|
1995
|
+
}
|
|
1996
|
+
}
|
|
1997
|
+
if (!options.onSpeechPartial)
|
|
1998
|
+
return;
|
|
1999
|
+
const audioMs = finiteTimestamp(payload.audioMs);
|
|
2000
|
+
const decodeMs = finiteTimestamp(payload.decodeMs);
|
|
2001
|
+
void Promise.resolve()
|
|
2002
|
+
.then(() => options.onSpeechPartial({
|
|
2003
|
+
text,
|
|
2004
|
+
...(audioMs !== undefined ? { audioMs } : {}),
|
|
2005
|
+
...(decodeMs !== undefined ? { decodeMs } : {}),
|
|
2006
|
+
}))
|
|
2007
|
+
.catch((error) => {
|
|
2008
|
+
logger.debug('[speech] partial transcript prewarm skipped', {
|
|
2009
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2010
|
+
});
|
|
2011
|
+
});
|
|
2012
|
+
return;
|
|
2013
|
+
}
|
|
2014
|
+
// Live-mic path (buddy-sense `live-audio`): the transcript is ALREADY decoded
|
|
2015
|
+
// and carried in the payload — drive the same cognition with the text directly,
|
|
2016
|
+
// no WAV / no STT. Keyed on a synthetic id since there's no file to dedup on.
|
|
2017
|
+
if (p.kind === 'transcript_final') {
|
|
2018
|
+
const livePayload = p.payload;
|
|
2019
|
+
let speechStartedAtMs = finiteTimestamp(p.payload?.startedAtMs) ?? pendingSpeechStartedAtMs;
|
|
2020
|
+
let turnId = pendingSpeechTurnId;
|
|
2021
|
+
const suspectedOwnPlayback = turnId !== undefined
|
|
2022
|
+
&& suspectedOwnPlaybackTurnId === turnId;
|
|
2023
|
+
const repairAddressHint = pendingSpeechPartialText;
|
|
2024
|
+
pendingSpeechStartedAtMs = undefined;
|
|
2025
|
+
pendingSpeechTurnId = undefined;
|
|
2026
|
+
pendingSpeechPartialText = undefined;
|
|
2027
|
+
let text = livePayload?.text?.trim() ?? '';
|
|
2028
|
+
if (suspectedOwnPlayback)
|
|
2029
|
+
suspectedOwnPlaybackTurnId = undefined;
|
|
2030
|
+
if (suspectedOwnPlayback && isRecentVoiceFragmentEcho(text, speechStartedAtMs ?? now())) {
|
|
2031
|
+
if (turnId) {
|
|
2032
|
+
turnCoordinator.transition(turnId, 'suppressed', {
|
|
2033
|
+
suppressionReason: 'own_playback_fragment',
|
|
2034
|
+
scene: 'assistant_playback',
|
|
2035
|
+
sceneConfidence: 0.98,
|
|
2036
|
+
});
|
|
2037
|
+
}
|
|
2038
|
+
logger.info('[speech] dropped own playback fragment');
|
|
2039
|
+
return;
|
|
2040
|
+
}
|
|
2041
|
+
if (!text && env.CODEBUDDY_SENSORY_REPAIR !== 'true')
|
|
2042
|
+
return;
|
|
2043
|
+
const key = `live:${liveSeq++}`;
|
|
2044
|
+
if (heldLiveTurn) {
|
|
2045
|
+
clearTimeout(heldLiveTurn.timer);
|
|
2046
|
+
text = joinVoiceTurnFragments(heldLiveTurn.text, text);
|
|
2047
|
+
speechStartedAtMs = heldLiveTurn.speechStartedAtMs ?? speechStartedAtMs;
|
|
2048
|
+
turnId = heldLiveTurn.turnId ?? turnId;
|
|
2049
|
+
heldLiveTurn = null;
|
|
2050
|
+
}
|
|
2051
|
+
let turnDecision;
|
|
2052
|
+
try {
|
|
2053
|
+
turnDecision = resolveTurnDetectorDecision({ text, payload: p.payload ?? {} }, options.turnDecisionProvider);
|
|
2054
|
+
}
|
|
2055
|
+
catch (error) {
|
|
2056
|
+
logger.warn(`[speech] LiveKit turn decision unavailable: ${error instanceof Error ? error.message : String(error)}`);
|
|
2057
|
+
}
|
|
2058
|
+
// Smart Turn has already considered prosody and the complete audio. The
|
|
2059
|
+
// text heuristic is only a fail-open fallback for VAD-only sources.
|
|
2060
|
+
const conversationalEndSilenceMs = resolveConversationalTurnEndSilenceMs(text, env);
|
|
2061
|
+
const endpointWaitMs = finiteTimestamp(livePayload?.endpointWaitMs) ?? 0;
|
|
2062
|
+
const remainingIncompleteHoldMs = conversationalEndSilenceMs === null
|
|
2063
|
+
? incompleteTurnHoldMs
|
|
2064
|
+
: Math.max(0, conversationalEndSilenceMs - endpointWaitMs);
|
|
2065
|
+
if (
|
|
2066
|
+
// CONV1 shortens the hold by the endpoint silence already waited. A
|
|
2067
|
+
// probabilistic detector may confirm a complete turn, but may not
|
|
2068
|
+
// override an explicit syntactic suspension in the transcript.
|
|
2069
|
+
remainingIncompleteHoldMs > 0 &&
|
|
2070
|
+
(turnDecision?.endOfTurn === false || isLikelyIncompleteVoiceTurn(text))) {
|
|
2071
|
+
const timer = setTimeout(() => {
|
|
2072
|
+
const held = heldLiveTurn;
|
|
2073
|
+
if (!held || held.key !== key || disposed)
|
|
2074
|
+
return;
|
|
2075
|
+
heldLiveTurn = null;
|
|
2076
|
+
const job = {
|
|
2077
|
+
p: held.p,
|
|
2078
|
+
wav: held.key,
|
|
2079
|
+
presetText: held.text,
|
|
2080
|
+
...(held.speechStartedAtMs !== undefined
|
|
2081
|
+
? { speechStartedAtMs: held.speechStartedAtMs }
|
|
2082
|
+
: {}),
|
|
2083
|
+
...(held.turnId ? { turnId: held.turnId } : {}),
|
|
2084
|
+
...(repairAddressHint ? { repairAddressHint } : {}),
|
|
2085
|
+
};
|
|
2086
|
+
if (inFlight)
|
|
2087
|
+
queuePendingSpeech(job);
|
|
2088
|
+
else
|
|
2089
|
+
startSpeechJob(job);
|
|
2090
|
+
}, remainingIncompleteHoldMs);
|
|
2091
|
+
heldLiveTurn = {
|
|
2092
|
+
p,
|
|
2093
|
+
text,
|
|
2094
|
+
key,
|
|
2095
|
+
timer,
|
|
2096
|
+
...(speechStartedAtMs !== undefined ? { speechStartedAtMs } : {}),
|
|
2097
|
+
...(turnId ? { turnId } : {}),
|
|
2098
|
+
};
|
|
2099
|
+
logger.debug(`[speech] holding likely incomplete turn for ${remainingIncompleteHoldMs}ms → ${text}`);
|
|
2100
|
+
return;
|
|
2101
|
+
}
|
|
2102
|
+
if (inFlight) {
|
|
2103
|
+
const payload = p.payload ?? {};
|
|
2104
|
+
const acousticBargeIn = shouldTriggerAcousticBargeIn(payload, speechStartedAtMs);
|
|
2105
|
+
if ((options.onBargeIn || options.onBargeInStart) &&
|
|
2106
|
+
(shouldTriggerVoiceBargeIn(text, payload, env) || acousticBargeIn) &&
|
|
2107
|
+
(turnId === undefined || bargedSpeechTurnId !== turnId)) {
|
|
2108
|
+
if (turnId !== undefined)
|
|
2109
|
+
bargedSpeechTurnId = turnId;
|
|
2110
|
+
logger.info(`[speech] barge-in → ${text}`);
|
|
2111
|
+
try {
|
|
2112
|
+
if (acousticBargeIn && options.onBargeInStart) {
|
|
2113
|
+
options.onBargeInStart(payload, activeTurnId);
|
|
2114
|
+
}
|
|
2115
|
+
else {
|
|
2116
|
+
options.onBargeIn?.(text, activeTurnId);
|
|
2117
|
+
}
|
|
2118
|
+
}
|
|
2119
|
+
catch {
|
|
2120
|
+
/* interruption is best-effort; still queue the new utterance */
|
|
2121
|
+
}
|
|
2122
|
+
}
|
|
2123
|
+
if (key !== activeWav) {
|
|
2124
|
+
queuePendingSpeech({
|
|
2125
|
+
p,
|
|
2126
|
+
wav: key,
|
|
2127
|
+
presetText: text,
|
|
2128
|
+
...(speechStartedAtMs !== undefined ? { speechStartedAtMs } : {}),
|
|
2129
|
+
...(turnId ? { turnId } : {}),
|
|
2130
|
+
...(repairAddressHint ? { repairAddressHint } : {}),
|
|
2131
|
+
});
|
|
2132
|
+
}
|
|
2133
|
+
return;
|
|
2134
|
+
}
|
|
2135
|
+
startSpeechJob({
|
|
2136
|
+
p,
|
|
2137
|
+
wav: key,
|
|
2138
|
+
presetText: text,
|
|
2139
|
+
...(speechStartedAtMs !== undefined ? { speechStartedAtMs } : {}),
|
|
2140
|
+
...(turnId ? { turnId } : {}),
|
|
2141
|
+
...(repairAddressHint ? { repairAddressHint } : {}),
|
|
2142
|
+
});
|
|
2143
|
+
return;
|
|
2144
|
+
}
|
|
2145
|
+
if (p.kind !== 'speech_end')
|
|
2146
|
+
return;
|
|
2147
|
+
const wav = p.payload?.wav;
|
|
2148
|
+
const speechStartedAtMs = finiteTimestamp(p.payload?.startedAtMs) ?? pendingSpeechStartedAtMs;
|
|
2149
|
+
const turnId = pendingSpeechTurnId;
|
|
2150
|
+
pendingSpeechStartedAtMs = undefined;
|
|
2151
|
+
pendingSpeechTurnId = undefined;
|
|
2152
|
+
if (turnId !== undefined && suspectedOwnPlaybackTurnId === turnId) {
|
|
2153
|
+
suspectedOwnPlaybackTurnId = undefined;
|
|
2154
|
+
return;
|
|
2155
|
+
}
|
|
2156
|
+
if (!wav)
|
|
2157
|
+
return; // no audio to transcribe (the batch path needs a WAV)
|
|
2158
|
+
if (inFlight) {
|
|
2159
|
+
if (wav !== activeWav) {
|
|
2160
|
+
queuePendingSpeech({
|
|
2161
|
+
p,
|
|
2162
|
+
wav,
|
|
2163
|
+
...(speechStartedAtMs !== undefined ? { speechStartedAtMs } : {}),
|
|
2164
|
+
...(turnId ? { turnId } : {}),
|
|
2165
|
+
});
|
|
2166
|
+
}
|
|
2167
|
+
return;
|
|
2168
|
+
}
|
|
2169
|
+
startSpeechJob({
|
|
2170
|
+
p,
|
|
2171
|
+
wav,
|
|
2172
|
+
...(speechStartedAtMs !== undefined ? { speechStartedAtMs } : {}),
|
|
2173
|
+
...(turnId ? { turnId } : {}),
|
|
2174
|
+
});
|
|
78
2175
|
});
|
|
79
2176
|
return () => {
|
|
2177
|
+
disposed = true;
|
|
2178
|
+
conversationCues.dispose();
|
|
2179
|
+
if (heldLiveTurn)
|
|
2180
|
+
clearTimeout(heldLiveTurn.timer);
|
|
2181
|
+
heldLiveTurn = null;
|
|
2182
|
+
const abandonedSpeech = pendingSpeech;
|
|
2183
|
+
pendingSpeech = null;
|
|
2184
|
+
if (abandonedSpeech)
|
|
2185
|
+
void cleanupSpeechJob(abandonedSpeech);
|
|
80
2186
|
bus.off(id);
|
|
81
2187
|
};
|
|
82
2188
|
}
|