@phuetz/code-buddy 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +211 -163
- package/dist/agent/autonomous/agentic-coding-contract.d.ts +12 -12
- package/dist/agent/base-agent.d.ts +4 -0
- package/dist/agent/base-agent.js +6 -0
- package/dist/agent/codebuddy-agent.d.ts +1 -1
- package/dist/agent/codebuddy-agent.js +69 -14
- package/dist/agent/execution/agent-executor.js +122 -12
- package/dist/agent/execution/tool-dependency-graph.js +7 -0
- package/dist/agent/execution/tool-hooks.d.ts +6 -0
- package/dist/agent/execution/tool-hooks.js +13 -5
- package/dist/agent/execution/tool-selection-strategy.d.ts +11 -2
- package/dist/agent/execution/tool-selection-strategy.js +67 -10
- package/dist/agent/extended-thinking.d.ts +8 -0
- package/dist/agent/extended-thinking.js +27 -0
- package/dist/agent/facades/infrastructure-facade.d.ts +9 -2
- package/dist/agent/facades/infrastructure-facade.js +15 -6
- package/dist/agent/hermes-browser-backends.js +151 -11
- package/dist/agent/hermes-claw-migrate.d.ts +59 -0
- package/dist/agent/hermes-claw-migrate.js +427 -26
- package/dist/agent/hermes-parity-manifest.js +78 -71
- package/dist/agent/hermes-runtime-lifecycle.d.ts +67 -0
- package/dist/agent/hermes-runtime-lifecycle.js +468 -0
- package/dist/agent/middleware/index.d.ts +1 -0
- package/dist/agent/middleware/index.js +1 -0
- package/dist/agent/middleware/visual-validation-middleware.d.ts +17 -0
- package/dist/agent/middleware/visual-validation-middleware.js +67 -0
- package/dist/agent/model-benchmark.d.ts +77 -0
- package/dist/agent/model-benchmark.js +309 -0
- package/dist/agent/model-tier.d.ts +14 -0
- package/dist/agent/model-tier.js +71 -0
- package/dist/agent/multi-agent/agents/orchestrator-agent.js +17 -6
- package/dist/agent/multi-agent/multi-agent-system.js +3 -1
- package/dist/agent/multi-agent/session-fleet-bridge.d.ts +2 -0
- package/dist/agent/multi-agent/session-fleet-bridge.js +9 -1
- package/dist/agent/multi-agent/session-tools.js +22 -5
- package/dist/agent/reasoning/mcts.js +20 -6
- package/dist/agent/self-improvement/authored-artifact-gate.d.ts +18 -0
- package/dist/agent/self-improvement/authored-artifact-gate.js +42 -0
- package/dist/agent/self-improvement/authored-tool-runtime.d.ts +27 -0
- package/dist/agent/self-improvement/authored-tool-runtime.js +57 -0
- package/dist/agent/self-improvement/authored-tool-store.d.ts +24 -0
- package/dist/agent/self-improvement/authored-tool-store.js +57 -0
- package/dist/agent/self-improvement/llm-tool-proposer.d.ts +41 -0
- package/dist/agent/self-improvement/llm-tool-proposer.js +136 -0
- package/dist/agent/self-improvement/sandbox-scorer.d.ts +17 -0
- package/dist/agent/self-improvement/sandbox-scorer.js +43 -0
- package/dist/agent/self-improvement/self-knowledge.d.ts +8 -0
- package/dist/agent/self-improvement/self-knowledge.js +24 -0
- package/dist/agent/self-improvement/skill-benchmark.d.ts +9 -0
- package/dist/agent/self-improvement/skill-benchmark.js +22 -0
- package/dist/agent/self-improvement/skill-consolidator.d.ts +71 -0
- package/dist/agent/self-improvement/skill-consolidator.js +137 -0
- package/dist/agent/self-improvement/skill-engine.d.ts +42 -0
- package/dist/agent/self-improvement/skill-engine.js +87 -0
- package/dist/agent/self-improvement/skill-gate.d.ts +19 -0
- package/dist/agent/self-improvement/skill-gate.js +62 -0
- package/dist/agent/self-improvement/skill-mutator.d.ts +74 -0
- package/dist/agent/self-improvement/skill-mutator.js +223 -0
- package/dist/agent/self-improvement/skill-proposer.d.ts +40 -0
- package/dist/agent/self-improvement/skill-proposer.js +82 -0
- package/dist/agent/self-improvement/skill-types.d.ts +41 -0
- package/dist/agent/self-improvement/skill-types.js +13 -0
- package/dist/agent/self-improvement/tool-benchmark.d.ts +10 -0
- package/dist/agent/self-improvement/tool-benchmark.js +37 -0
- package/dist/agent/self-improvement/tool-engine.d.ts +54 -0
- package/dist/agent/self-improvement/tool-engine.js +101 -0
- package/dist/agent/self-improvement/tool-gate.d.ts +20 -0
- package/dist/agent/self-improvement/tool-gate.js +78 -0
- package/dist/agent/self-improvement/tool-proposer.d.ts +31 -0
- package/dist/agent/self-improvement/tool-proposer.js +34 -0
- package/dist/agent/self-improvement/tool-skill-mutator.d.ts +40 -0
- package/dist/agent/self-improvement/tool-skill-mutator.js +79 -0
- package/dist/agent/self-improvement/tool-types.d.ts +48 -0
- package/dist/agent/self-improvement/tool-types.js +9 -0
- package/dist/agent/self-improvement/types.d.ts +3 -1
- package/dist/agent/session-end-flush.d.ts +6 -1
- package/dist/agent/session-end-flush.js +20 -4
- package/dist/agent/streaming/streaming-handler.d.ts +7 -0
- package/dist/agent/streaming/streaming-handler.js +8 -1
- package/dist/agent/tool-executor.js +1 -0
- package/dist/agent/tool-handler.d.ts +5 -0
- package/dist/agent/tool-handler.js +133 -4
- package/dist/browser-automation/browser-operator-executor.js +15 -0
- package/dist/browser-automation/browser-use-runner.d.ts +96 -0
- package/dist/browser-automation/browser-use-runner.js +492 -0
- package/dist/browser-automation/camofox-runner.d.ts +107 -0
- package/dist/browser-automation/camofox-runner.js +287 -0
- package/dist/channels/dingtalk/index.js +3 -0
- package/dist/channels/discord/client.d.ts +9 -0
- package/dist/channels/discord/client.js +12 -0
- package/dist/channels/feishu/index.d.ts +167 -1
- package/dist/channels/feishu/index.js +432 -7
- package/dist/channels/gateway-lifecycle.d.ts +132 -0
- package/dist/channels/gateway-lifecycle.js +219 -0
- package/dist/channels/google-chat/index.d.ts +5 -1
- package/dist/channels/google-chat/index.js +5 -1
- package/dist/channels/imessage/index.d.ts +14 -0
- package/dist/channels/imessage/index.js +64 -17
- package/dist/channels/index.d.ts +4 -0
- package/dist/channels/index.js +4 -0
- package/dist/channels/irc/index.d.ts +73 -3
- package/dist/channels/irc/index.js +446 -11
- package/dist/channels/line/index.js +3 -0
- package/dist/channels/mattermost/index.d.ts +51 -4
- package/dist/channels/mattermost/index.js +303 -20
- package/dist/channels/nextcloud-talk/index.d.ts +89 -8
- package/dist/channels/nextcloud-talk/index.js +367 -16
- package/dist/channels/nostr/index.d.ts +121 -1
- package/dist/channels/nostr/index.js +396 -8
- package/dist/channels/ntfy/index.js +3 -0
- package/dist/channels/qq/index.js +3 -0
- package/dist/channels/slash-parity.d.ts +117 -0
- package/dist/channels/slash-parity.js +185 -0
- package/dist/channels/synology-chat/index.js +3 -0
- package/dist/channels/teams/index.d.ts +3 -0
- package/dist/channels/teams/index.js +3 -0
- package/dist/channels/telegram/client.d.ts +9 -0
- package/dist/channels/telegram/client.js +12 -0
- package/dist/channels/twilio-voice/index.js +3 -0
- package/dist/channels/webchat/index.d.ts +7 -1
- package/dist/channels/webchat/index.js +7 -1
- package/dist/channels/wecom/index.js +3 -0
- package/dist/channels/weixin/index.js +3 -0
- package/dist/channels/zalo/index.js +3 -0
- package/dist/cli/config-loader.js +1 -1
- package/dist/codebuddy/client.d.ts +30 -1
- package/dist/codebuddy/client.js +167 -20
- package/dist/codebuddy/providers/provider-chatgpt-responses.d.ts +5 -0
- package/dist/codebuddy/providers/provider-chatgpt-responses.js +7 -1
- package/dist/codebuddy/providers/provider-gemini-native.js +1 -1
- package/dist/codebuddy/providers/provider-openai-compat.d.ts +5 -0
- package/dist/codebuddy/providers/provider-openai-compat.js +82 -2
- package/dist/codebuddy/tool-definitions/computer-control-tools.js +32 -2
- package/dist/codebuddy/tool-definitions/core-tools.js +3 -3
- package/dist/codebuddy/tool-definitions/index.d.ts +2 -1
- package/dist/codebuddy/tool-definitions/index.js +2 -1
- package/dist/codebuddy/tool-definitions/multimodal-tools.d.ts +1 -0
- package/dist/codebuddy/tool-definitions/multimodal-tools.js +44 -1
- package/dist/codebuddy/tool-definitions/windows-tools.d.ts +2 -0
- package/dist/codebuddy/tool-definitions/windows-tools.js +34 -0
- package/dist/codebuddy/tools.d.ts +8 -1
- package/dist/codebuddy/tools.js +67 -24
- package/dist/commands/cli/hermes-commands.js +108 -0
- package/dist/commands/cli/improve-command.js +123 -0
- package/dist/commands/cli/native-engine-commands.js +102 -3
- package/dist/commands/cli/utility-commands.js +45 -0
- package/dist/commands/enhanced-command-handler.js +2 -2
- package/dist/commands/goal-cli.d.ts +30 -0
- package/dist/commands/goal-cli.js +210 -27
- package/dist/commands/handlers/goal-handler.d.ts +9 -2
- package/dist/commands/handlers/goal-handler.js +36 -9
- package/dist/commands/handlers/infra-handlers.js +36 -0
- package/dist/commands/handlers/memory-handlers.js +96 -1
- package/dist/commands/handlers/missing-handlers.d.ts +1 -1
- package/dist/commands/handlers/missing-handlers.js +26 -3
- package/dist/commands/headless-slash.d.ts +2 -0
- package/dist/commands/headless-slash.js +12 -0
- package/dist/commands/llm-provider-resolution.d.ts +5 -3
- package/dist/commands/llm-provider-resolution.js +87 -33
- package/dist/commands/ollama.d.ts +25 -0
- package/dist/commands/ollama.js +100 -0
- package/dist/commands/provider.d.ts +5 -0
- package/dist/commands/provider.js +106 -55
- package/dist/commands/skills-cli/index.js +123 -0
- package/dist/commands/slash/builtin-commands.js +1 -1
- package/dist/commands/spec-next.js +2 -1
- package/dist/commands/spec-plan.js +5 -16
- package/dist/commands/tunnel.d.ts +2 -0
- package/dist/commands/tunnel.js +48 -0
- package/dist/companion/percepts.js +11 -1
- package/dist/config/config-resolver.d.ts +2 -1
- package/dist/config/config-resolver.js +54 -41
- package/dist/config/constants.d.ts +28 -0
- package/dist/config/constants.js +7 -0
- package/dist/config/env-schema.js +756 -0
- package/dist/config/model-tools.js +4 -4
- package/dist/config/toml-config.d.ts +16 -0
- package/dist/config/toml-config.js +3 -0
- package/dist/context/bootstrap-loader.js +6 -23
- package/dist/context/context-manager-v2.js +1 -0
- package/dist/context/import-directive-parser.d.ts +4 -0
- package/dist/context/import-directive-parser.js +51 -6
- package/dist/context/instruction-excludes.d.ts +30 -1
- package/dist/context/instruction-excludes.js +71 -1
- package/dist/context/jit-context.d.ts +8 -10
- package/dist/context/jit-context.js +28 -106
- package/dist/context/project-context.d.ts +90 -0
- package/dist/context/project-context.js +295 -0
- package/dist/daemon/autonomous-daemon.d.ts +1 -1
- package/dist/daemon/autonomous-daemon.js +3 -3
- package/dist/daemon/autonomous-loop.d.ts +31 -1
- package/dist/daemon/autonomous-loop.js +91 -4
- package/dist/daemon/colab-goal.js +9 -1
- package/dist/daemon/cron-agent-bridge.d.ts +12 -3
- package/dist/daemon/cron-agent-bridge.js +25 -9
- package/dist/desktop/codebuddy-engine-adapter.d.ts +32 -0
- package/dist/desktop/codebuddy-engine-adapter.js +257 -80
- package/dist/desktop/engine-adapter.d.ts +14 -0
- package/dist/desktop-automation/automation-manager.js +16 -0
- package/dist/desktop-automation/omniparser-runner.d.ts +63 -0
- package/dist/desktop-automation/omniparser-runner.js +115 -0
- package/dist/desktop-automation/smart-snapshot.d.ts +11 -0
- package/dist/desktop-automation/smart-snapshot.js +72 -5
- package/dist/doctor/index.js +40 -23
- package/dist/events/types.d.ts +11 -0
- package/dist/fleet/colab-store.js +14 -2
- package/dist/fleet/peer-session-bridge.js +50 -12
- package/dist/fleet/peer-tool-bridge.js +14 -0
- package/dist/goals/goal-decomposer.d.ts +27 -0
- package/dist/goals/goal-decomposer.js +293 -0
- package/dist/goals/goal-judge-client.d.ts +15 -0
- package/dist/goals/goal-judge-client.js +45 -0
- package/dist/goals/goal-judge.d.ts +2 -0
- package/dist/goals/goal-judge.js +35 -6
- package/dist/goals/goal-loop.d.ts +14 -0
- package/dist/goals/goal-loop.js +47 -2
- package/dist/goals/goal-manager.d.ts +6 -0
- package/dist/goals/goal-manager.js +67 -14
- package/dist/goals/goal-state.d.ts +13 -0
- package/dist/goals/goal-state.js +83 -13
- package/dist/goals/goal-store.d.ts +3 -0
- package/dist/goals/goal-store.js +31 -15
- package/dist/harness/contract.d.ts +28 -28
- package/dist/hooks/user-hooks.js +17 -3
- package/dist/identity/identity-manager.js +3 -2
- package/dist/index.js +481 -26
- package/dist/input/text-to-speech.d.ts +2 -6
- package/dist/input/text-to-speech.js +2 -27
- package/dist/integrations/screenpipe/screenpipe-client.d.ts +10 -0
- package/dist/integrations/screenpipe/screenpipe-client.js +12 -2
- package/dist/integrations/tailscale.d.ts +13 -0
- package/dist/integrations/tailscale.js +87 -34
- package/dist/knowledge/workspace-indexer.js +53 -9
- package/dist/mcp/client.js +1 -0
- package/dist/mcp/mcp-desktop-tools.d.ts +23 -0
- package/dist/mcp/mcp-desktop-tools.js +158 -0
- package/dist/mcp/mcp-resources.js +2 -3
- package/dist/mcp/mcp-server.js +2 -0
- package/dist/memory/enhanced-memory.d.ts +10 -0
- package/dist/memory/enhanced-memory.js +33 -2
- package/dist/memory/index.d.ts +3 -1
- package/dist/memory/index.js +4 -1
- package/dist/memory/memory-auto-proposer.d.ts +23 -0
- package/dist/memory/memory-auto-proposer.js +308 -0
- package/dist/memory/memory-candidate-queue.d.ts +92 -0
- package/dist/memory/memory-candidate-queue.js +261 -0
- package/dist/memory/persistent-memory.d.ts +45 -2
- package/dist/memory/persistent-memory.js +236 -40
- package/dist/ml/bayesian-qualifier.d.ts +1 -1
- package/dist/ml/bayesian-qualifier.js +35 -2
- package/dist/openclaw/gateway-bridge.js +5 -0
- package/dist/plugins/marketplace.d.ts +1 -0
- package/dist/plugins/marketplace.js +7 -0
- package/dist/prompts/execution-discipline.d.ts +14 -0
- package/dist/prompts/execution-discipline.js +29 -0
- package/dist/prompts/variation-injector.js +9 -2
- package/dist/protocols/acp/acp-agentic-runner.d.ts +7 -6
- package/dist/protocols/acp/acp-agentic-runner.js +115 -18
- package/dist/protocols/acp/acp-session-store.d.ts +23 -0
- package/dist/protocols/acp/acp-session-store.js +77 -0
- package/dist/protocols/acp/acp-stdio-server.d.ts +7 -0
- package/dist/protocols/acp/acp-stdio-server.js +71 -16
- package/dist/providers/active-llm-registry.d.ts +37 -0
- package/dist/providers/active-llm-registry.js +186 -0
- package/dist/providers/auxiliary-provider.d.ts +25 -0
- package/dist/providers/auxiliary-provider.js +192 -0
- package/dist/providers/codex-oauth.d.ts +1 -1
- package/dist/providers/codex-oauth.js +27 -4
- package/dist/providers/index.d.ts +3 -0
- package/dist/providers/index.js +4 -0
- package/dist/providers/provider-catalog.d.ts +70 -0
- package/dist/providers/provider-catalog.js +738 -0
- package/dist/providers/provider-fallback.d.ts +35 -0
- package/dist/providers/provider-fallback.js +171 -0
- package/dist/providers/xai-oauth.d.ts +128 -0
- package/dist/providers/xai-oauth.js +735 -0
- package/dist/scheduler/cron-scheduler.d.ts +15 -3
- package/dist/scheduler/cron-scheduler.js +53 -7
- package/dist/scheduler/script-runner.d.ts +2 -0
- package/dist/scheduler/script-runner.js +19 -4
- package/dist/scheduler/watchdog-handlers.js +7 -20
- package/dist/search/usearch-index.js +7 -2
- package/dist/security/tool-policy/tool-groups.js +2 -0
- package/dist/sensory/dreaming.d.ts +45 -0
- package/dist/sensory/dreaming.js +114 -0
- package/dist/sensory/heartbeat-scheduler.d.ts +38 -0
- package/dist/sensory/heartbeat-scheduler.js +72 -0
- package/dist/sensory/reactions.d.ts +24 -0
- package/dist/sensory/reactions.js +31 -0
- package/dist/sensory/screen-reaction.d.ts +23 -0
- package/dist/sensory/screen-reaction.js +59 -0
- package/dist/sensory/sensory-bridge.d.ts +23 -0
- package/dist/sensory/sensory-bridge.js +85 -0
- package/dist/sensory/sensory-memory.d.ts +20 -0
- package/dist/sensory/sensory-memory.js +39 -0
- package/dist/sensory/speech-reaction.d.ts +21 -0
- package/dist/sensory/speech-reaction.js +83 -0
- package/dist/sensory/vision-reaction.d.ts +31 -0
- package/dist/sensory/vision-reaction.js +74 -0
- package/dist/server/index.js +106 -2
- package/dist/server/routes/mobile.d.ts +12 -4
- package/dist/server/routes/mobile.js +116 -1
- package/dist/server/tls-config.d.ts +35 -0
- package/dist/server/tls-config.js +142 -0
- package/dist/server/tunnel-manager.d.ts +20 -0
- package/dist/server/tunnel-manager.js +58 -0
- package/dist/server/websocket/fleet-bridge.js +5 -0
- package/dist/services/prompt-builder.d.ts +11 -0
- package/dist/services/prompt-builder.js +124 -17
- package/dist/shared/engine-types.d.ts +15 -1
- package/dist/sidecar/sidecar-bridge.d.ts +1 -0
- package/dist/sidecar/sidecar-bridge.js +21 -10
- package/dist/skills/parser.js +3 -0
- package/dist/skills/skill-importer.d.ts +58 -0
- package/dist/skills/skill-importer.js +261 -0
- package/dist/skills/skill-sources.d.ts +20 -0
- package/dist/skills/skill-sources.js +102 -0
- package/dist/skills/types.d.ts +6 -0
- package/dist/spec/spec-store.js +8 -1
- package/dist/talk-mode/providers/audioreader-tts.d.ts +1 -1
- package/dist/talk-mode/providers/audioreader-tts.js +8 -8
- package/dist/templates/project-scaffolding.js +3 -2
- package/dist/themes/theme-schema.d.ts +10 -10
- package/dist/tools/application-profiles.js +38 -0
- package/dist/tools/bash/bash-tool.d.ts +1 -0
- package/dist/tools/bash/bash-tool.js +16 -2
- package/dist/tools/bash/command-validator.js +3 -0
- package/dist/tools/bash/rtk-rewrite.d.ts +16 -0
- package/dist/tools/bash/rtk-rewrite.js +100 -0
- package/dist/tools/bash/streaming-executor.js +14 -1
- package/dist/tools/computer-control-tool.d.ts +16 -1
- package/dist/tools/computer-control-tool.js +317 -5
- package/dist/tools/document-generator.d.ts +14 -0
- package/dist/tools/document-generator.js +79 -0
- package/dist/tools/execute-code-rpc-invoker.d.ts +26 -2
- package/dist/tools/execute-code-rpc-invoker.js +46 -5
- package/dist/tools/metadata.js +28 -0
- package/dist/tools/office-macro-tool.d.ts +10 -0
- package/dist/tools/office-macro-tool.js +93 -0
- package/dist/tools/register-tool-handler.d.ts +25 -0
- package/dist/tools/register-tool-handler.js +100 -0
- package/dist/tools/registry/index.d.ts +2 -1
- package/dist/tools/registry/index.js +2 -1
- package/dist/tools/registry/memory-tools.d.ts +20 -0
- package/dist/tools/registry/memory-tools.js +225 -3
- package/dist/tools/registry/multimodal-tools.js +2 -2
- package/dist/tools/registry/vision-tools.d.ts +33 -0
- package/dist/tools/registry/vision-tools.js +308 -6
- package/dist/tools/registry/windows-tools.d.ts +2 -0
- package/dist/tools/registry/windows-tools.js +47 -0
- package/dist/tools/registry.d.ts +6 -0
- package/dist/tools/registry.js +8 -0
- package/dist/tools/route-peer-tool.js +14 -0
- package/dist/tools/screenshot-tool.js +14 -2
- package/dist/tools/text-to-speech-tool.d.ts +1 -1
- package/dist/tools/text-to-speech-tool.js +2 -12
- package/dist/tools/tool-selector.js +3 -59
- package/dist/utils/config-validation/schema.d.ts +40 -7
- package/dist/utils/config-validation/schema.js +130 -1
- package/dist/utils/cost-tracker.js +21 -1
- package/dist/utils/disk-guard.d.ts +180 -0
- package/dist/utils/disk-guard.js +404 -0
- package/dist/utils/init-project.d.ts +7 -0
- package/dist/utils/init-project.js +37 -0
- package/dist/utils/interactive-setup.js +35 -2
- package/dist/utils/model-utils.js +11 -2
- package/dist/utils/output-sanitizer.js +11 -0
- package/dist/utils/provider-detector.d.ts +8 -7
- package/dist/utils/provider-detector.js +19 -77
- package/dist/utils/settings-manager.d.ts +19 -0
- package/dist/utils/token-display.d.ts +3 -3
- package/dist/utils/token-display.js +22 -4
- package/dist/wizard/onboarding.d.ts +39 -0
- package/dist/wizard/onboarding.js +433 -21
- package/dist/wizard/provider-onboarding.d.ts +8 -23
- package/dist/wizard/provider-onboarding.js +39 -158
- package/package.json +4 -2
- package/dist/plugins/provider-onboarding.d.ts +0 -23
- package/dist/plugins/provider-onboarding.js +0 -116
- package/dist/utils/kokoro-tts.d.ts +0 -8
- package/dist/utils/kokoro-tts.js +0 -67
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
import type { ToolResult } from '../../types/index.js';
|
|
7
7
|
import type { ITool, ToolSchema, IToolMetadata, IValidationResult, IToolExecutionContext } from './types.js';
|
|
8
8
|
import { type VisionAnalysisOptions } from '../vision/vision-analysis.js';
|
|
9
|
+
import { captureCameraSnapshot } from '../../companion/camera.js';
|
|
9
10
|
export declare class VisionAnalyzeTool implements ITool {
|
|
10
11
|
private readonly options;
|
|
11
12
|
readonly name = "vision_analyze";
|
|
@@ -29,6 +30,7 @@ export declare class BrowserVisionTool implements ITool {
|
|
|
29
30
|
getMetadata(): IToolMetadata;
|
|
30
31
|
isAvailable(): boolean;
|
|
31
32
|
dispose(): Promise<void>;
|
|
33
|
+
private captureScreenshotWithRetry;
|
|
32
34
|
}
|
|
33
35
|
export declare class OcrExtractTool implements ITool {
|
|
34
36
|
readonly name = "ocr_extract";
|
|
@@ -57,4 +59,35 @@ export declare class CameraSnapshotTool implements ITool {
|
|
|
57
59
|
getMetadata(): IToolMetadata;
|
|
58
60
|
isAvailable(): boolean;
|
|
59
61
|
}
|
|
62
|
+
export interface CameraAnalyzeRuntime {
|
|
63
|
+
/** Injected for tests; defaults to global fetch. */
|
|
64
|
+
fetch?: typeof fetch;
|
|
65
|
+
/** Injected for tests; defaults to companion captureCameraSnapshot. */
|
|
66
|
+
captureSnapshot?: typeof captureCameraSnapshot;
|
|
67
|
+
/** Injected for tests; defaults to process.env. */
|
|
68
|
+
env?: NodeJS.ProcessEnv;
|
|
69
|
+
}
|
|
70
|
+
export declare class CameraAnalyzeTool implements ITool {
|
|
71
|
+
private readonly options;
|
|
72
|
+
private readonly runtime;
|
|
73
|
+
readonly name = "camera_analyze";
|
|
74
|
+
readonly description = "Capture one local webcam frame and return a natural-language description from a local multimodal vision model (default Ollama gemma4:12b). Requires ffmpeg, OS camera permission, and a reachable local vision model.";
|
|
75
|
+
constructor(options?: VisionAnalysisOptions, runtime?: CameraAnalyzeRuntime);
|
|
76
|
+
execute(input: Record<string, unknown>, context?: IToolExecutionContext): Promise<ToolResult>;
|
|
77
|
+
private describeImage;
|
|
78
|
+
getSchema(): ToolSchema;
|
|
79
|
+
validate(input: unknown): IValidationResult;
|
|
80
|
+
getMetadata(): IToolMetadata;
|
|
81
|
+
isAvailable(): boolean;
|
|
82
|
+
}
|
|
60
83
|
export declare function createVisionTools(options?: VisionAnalysisOptions): ITool[];
|
|
84
|
+
/**
|
|
85
|
+
* Resolve the Ollama OpenAI-compatible chat endpoint, honoring OLLAMA_HOST.
|
|
86
|
+
* OLLAMA_HOST is typically `host:port` with no scheme and no /v1 path.
|
|
87
|
+
*/
|
|
88
|
+
export declare function resolveOllamaChatEndpoint(env: NodeJS.ProcessEnv): string;
|
|
89
|
+
/**
|
|
90
|
+
* Extract assistant text from an OpenAI-compatible chat completion. The
|
|
91
|
+
* `content` field may be a plain string or an array of typed parts.
|
|
92
|
+
*/
|
|
93
|
+
export declare function extractCompletionText(response: unknown): string | undefined;
|
|
@@ -123,12 +123,7 @@ export class BrowserVisionTool {
|
|
|
123
123
|
const screenshotDir = path.join(rootDir, '.codebuddy', 'browser-vision');
|
|
124
124
|
await fs.mkdir(screenshotDir, { recursive: true });
|
|
125
125
|
const screenshotPath = path.join(screenshotDir, `browser-vision-${sanitizeFilename(this.options.createId?.() ?? String(Date.now()))}.png`);
|
|
126
|
-
const screenshot = await this.
|
|
127
|
-
action: 'screenshot',
|
|
128
|
-
fullPage: input.full_page === true,
|
|
129
|
-
format: 'png',
|
|
130
|
-
outputPath: screenshotPath,
|
|
131
|
-
});
|
|
126
|
+
const screenshot = await this.captureScreenshotWithRetry(input, screenshotPath);
|
|
132
127
|
if (!screenshot.success)
|
|
133
128
|
return screenshot;
|
|
134
129
|
const analysis = await analyzeVisionImage(screenshotPath, {
|
|
@@ -247,6 +242,33 @@ export class BrowserVisionTool {
|
|
|
247
242
|
await this.browser.execute({ action: 'close' }).catch(() => undefined);
|
|
248
243
|
await this.browser.dispose?.();
|
|
249
244
|
}
|
|
245
|
+
async captureScreenshotWithRetry(input, screenshotPath) {
|
|
246
|
+
const fullPage = input.full_page === true;
|
|
247
|
+
const baseInput = {
|
|
248
|
+
action: 'screenshot',
|
|
249
|
+
fullPage,
|
|
250
|
+
format: 'png',
|
|
251
|
+
outputPath: screenshotPath,
|
|
252
|
+
};
|
|
253
|
+
const first = await this.browser.execute(baseInput);
|
|
254
|
+
if (first.success)
|
|
255
|
+
return first;
|
|
256
|
+
await delay(150);
|
|
257
|
+
const second = await this.browser.execute(baseInput);
|
|
258
|
+
if (second.success)
|
|
259
|
+
return second;
|
|
260
|
+
if (!fullPage) {
|
|
261
|
+
await delay(150);
|
|
262
|
+
const fullPageRetry = await this.browser.execute({
|
|
263
|
+
...baseInput,
|
|
264
|
+
fullPage: true,
|
|
265
|
+
});
|
|
266
|
+
if (fullPageRetry.success)
|
|
267
|
+
return fullPageRetry;
|
|
268
|
+
return combineScreenshotFailures(first, second, fullPageRetry);
|
|
269
|
+
}
|
|
270
|
+
return combineScreenshotFailures(first, second);
|
|
271
|
+
}
|
|
250
272
|
}
|
|
251
273
|
// ============================================================================
|
|
252
274
|
// OcrExtractTool
|
|
@@ -444,6 +466,234 @@ export class CameraSnapshotTool {
|
|
|
444
466
|
}
|
|
445
467
|
isAvailable() { return true; }
|
|
446
468
|
}
|
|
469
|
+
const DEFAULT_CAMERA_VISION_MODEL = 'gemma4:12b';
|
|
470
|
+
const CAMERA_VISION_TIMEOUT_MS = 180_000;
|
|
471
|
+
export class CameraAnalyzeTool {
|
|
472
|
+
options;
|
|
473
|
+
runtime;
|
|
474
|
+
name = 'camera_analyze';
|
|
475
|
+
description = 'Capture one local webcam frame and return a natural-language description from a local multimodal vision model (default Ollama gemma4:12b). Requires ffmpeg, OS camera permission, and a reachable local vision model.';
|
|
476
|
+
constructor(options = {}, runtime = {}) {
|
|
477
|
+
this.options = options;
|
|
478
|
+
this.runtime = runtime;
|
|
479
|
+
}
|
|
480
|
+
async execute(input, context) {
|
|
481
|
+
const cwd = this.options.rootDir ?? context?.cwd;
|
|
482
|
+
const prompt = optionalString(input, 'prompt') ?? 'Describe what you see.';
|
|
483
|
+
const model = optionalString(input, 'model') ?? DEFAULT_CAMERA_VISION_MODEL;
|
|
484
|
+
const device = optionalString(input, 'device');
|
|
485
|
+
const outputPath = optionalString(input, 'output_path');
|
|
486
|
+
const includeOcr = input.include_ocr === true;
|
|
487
|
+
// (1) Capture a frame from the local webcam.
|
|
488
|
+
const captureSnapshot = this.runtime.captureSnapshot ?? captureCameraSnapshot;
|
|
489
|
+
const snapshot = await captureSnapshot({
|
|
490
|
+
cwd,
|
|
491
|
+
...(outputPath ? { outputPath } : {}),
|
|
492
|
+
...(device ? { device } : {}),
|
|
493
|
+
timeoutMs: typeof input.timeout_ms === 'number' ? input.timeout_ms : undefined,
|
|
494
|
+
});
|
|
495
|
+
if (!snapshot.success || !snapshot.path) {
|
|
496
|
+
return {
|
|
497
|
+
success: false,
|
|
498
|
+
error: snapshot.error || 'Camera snapshot failed before analysis',
|
|
499
|
+
output: snapshot.command,
|
|
500
|
+
};
|
|
501
|
+
}
|
|
502
|
+
// (2) base64 the captured PNG into a data URL.
|
|
503
|
+
let dataUrl;
|
|
504
|
+
try {
|
|
505
|
+
const bytes = await fs.readFile(snapshot.path);
|
|
506
|
+
dataUrl = `data:image/png;base64,${bytes.toString('base64')}`;
|
|
507
|
+
}
|
|
508
|
+
catch (error) {
|
|
509
|
+
return {
|
|
510
|
+
success: false,
|
|
511
|
+
error: `Failed to read captured frame ${snapshot.path}: ${error instanceof Error ? error.message : String(error)}`,
|
|
512
|
+
};
|
|
513
|
+
}
|
|
514
|
+
// (3) Make a REAL vision completion against the local Ollama /v1 endpoint.
|
|
515
|
+
let description;
|
|
516
|
+
try {
|
|
517
|
+
description = await this.describeImage(model, prompt, dataUrl);
|
|
518
|
+
}
|
|
519
|
+
catch (error) {
|
|
520
|
+
return {
|
|
521
|
+
success: false,
|
|
522
|
+
error: error instanceof Error ? error.message : String(error),
|
|
523
|
+
output: `Captured frame saved to ${snapshot.path}, but the vision model could not describe it.`,
|
|
524
|
+
};
|
|
525
|
+
}
|
|
526
|
+
// (4) Optionally add local metadata/OCR evidence.
|
|
527
|
+
let ocr;
|
|
528
|
+
if (includeOcr) {
|
|
529
|
+
try {
|
|
530
|
+
const analysis = await analyzeVisionImage(snapshot.path, {
|
|
531
|
+
...this.options,
|
|
532
|
+
rootDir: cwd,
|
|
533
|
+
includeOcr: true,
|
|
534
|
+
ocrLanguage: optionalString(input, 'ocr_language') ?? 'eng',
|
|
535
|
+
source: 'image',
|
|
536
|
+
});
|
|
537
|
+
ocr = analysis.ocr;
|
|
538
|
+
}
|
|
539
|
+
catch {
|
|
540
|
+
// OCR is best-effort evidence; never fail the description on OCR errors.
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
// (5) Return the description text as the primary output.
|
|
544
|
+
const data = {
|
|
545
|
+
imagePath: snapshot.path,
|
|
546
|
+
description,
|
|
547
|
+
model,
|
|
548
|
+
...(ocr ? { ocr } : {}),
|
|
549
|
+
};
|
|
550
|
+
return {
|
|
551
|
+
success: true,
|
|
552
|
+
output: description,
|
|
553
|
+
data,
|
|
554
|
+
};
|
|
555
|
+
}
|
|
556
|
+
async describeImage(model, prompt, dataUrl) {
|
|
557
|
+
const fetchImpl = this.runtime.fetch ?? fetch;
|
|
558
|
+
const endpoint = resolveOllamaChatEndpoint(this.runtime.env ?? process.env);
|
|
559
|
+
const controller = new AbortController();
|
|
560
|
+
let timedOut = false;
|
|
561
|
+
const timer = setTimeout(() => {
|
|
562
|
+
timedOut = true;
|
|
563
|
+
controller.abort();
|
|
564
|
+
}, CAMERA_VISION_TIMEOUT_MS);
|
|
565
|
+
let response;
|
|
566
|
+
let text;
|
|
567
|
+
try {
|
|
568
|
+
response = await fetchImpl(endpoint, {
|
|
569
|
+
method: 'POST',
|
|
570
|
+
headers: { 'Content-Type': 'application/json' },
|
|
571
|
+
signal: controller.signal,
|
|
572
|
+
body: JSON.stringify({
|
|
573
|
+
model,
|
|
574
|
+
temperature: 0.2,
|
|
575
|
+
messages: [
|
|
576
|
+
{
|
|
577
|
+
role: 'user',
|
|
578
|
+
content: [
|
|
579
|
+
{ type: 'text', text: prompt },
|
|
580
|
+
{ type: 'image_url', image_url: { url: dataUrl } },
|
|
581
|
+
],
|
|
582
|
+
},
|
|
583
|
+
],
|
|
584
|
+
}),
|
|
585
|
+
});
|
|
586
|
+
// Read the body INSIDE the try so the abort timeout also covers a server
|
|
587
|
+
// that sends headers then stalls the body (otherwise response.text() hangs).
|
|
588
|
+
text = await response.text();
|
|
589
|
+
}
|
|
590
|
+
catch (error) {
|
|
591
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
592
|
+
if (timedOut) {
|
|
593
|
+
throw new Error(`Vision model ${model} timed out after ${CAMERA_VISION_TIMEOUT_MS}ms at ${endpoint}. ` +
|
|
594
|
+
'A cold local model load can be slow; retry once the model is warm.');
|
|
595
|
+
}
|
|
596
|
+
throw new Error(`Vision model unreachable at ${endpoint} (model ${model}): ${reason}. ` +
|
|
597
|
+
'Start Ollama (ollama serve) and pull a multimodal model, or set OLLAMA_HOST.');
|
|
598
|
+
}
|
|
599
|
+
finally {
|
|
600
|
+
clearTimeout(timer);
|
|
601
|
+
}
|
|
602
|
+
if (!response.ok) {
|
|
603
|
+
throw new Error(`Vision model returned HTTP ${response.status} from ${endpoint}: ${text.slice(0, 500)}`);
|
|
604
|
+
}
|
|
605
|
+
let parsed;
|
|
606
|
+
try {
|
|
607
|
+
parsed = JSON.parse(text);
|
|
608
|
+
}
|
|
609
|
+
catch {
|
|
610
|
+
throw new Error(`Vision model returned non-JSON response from ${endpoint}: ${text.slice(0, 300)}`);
|
|
611
|
+
}
|
|
612
|
+
const description = extractCompletionText(parsed);
|
|
613
|
+
if (!description) {
|
|
614
|
+
throw new Error(`Vision model ${model} returned an empty description.`);
|
|
615
|
+
}
|
|
616
|
+
return description;
|
|
617
|
+
}
|
|
618
|
+
getSchema() {
|
|
619
|
+
return {
|
|
620
|
+
name: this.name,
|
|
621
|
+
description: this.description,
|
|
622
|
+
parameters: {
|
|
623
|
+
type: 'object',
|
|
624
|
+
properties: {
|
|
625
|
+
prompt: {
|
|
626
|
+
type: 'string',
|
|
627
|
+
description: 'What to ask the vision model about the frame. Default "Describe what you see."',
|
|
628
|
+
},
|
|
629
|
+
device: {
|
|
630
|
+
type: 'string',
|
|
631
|
+
description: 'Optional ffmpeg camera device. Linux example: /dev/video0; Windows: video=Integrated Camera; macOS: 0.',
|
|
632
|
+
},
|
|
633
|
+
model: {
|
|
634
|
+
type: 'string',
|
|
635
|
+
description: 'Local multimodal model id served by Ollama. Default gemma4:12b.',
|
|
636
|
+
},
|
|
637
|
+
include_ocr: {
|
|
638
|
+
type: 'boolean',
|
|
639
|
+
description: 'Also attach local OCR text evidence from the captured frame. Default false.',
|
|
640
|
+
},
|
|
641
|
+
ocr_language: {
|
|
642
|
+
type: 'string',
|
|
643
|
+
description: 'OCR language code when include_ocr is true. Default eng.',
|
|
644
|
+
},
|
|
645
|
+
output_path: {
|
|
646
|
+
type: 'string',
|
|
647
|
+
description: 'Optional output image path. Defaults to .codebuddy/camera/camera-<timestamp>.png in the active workspace.',
|
|
648
|
+
},
|
|
649
|
+
timeout_ms: {
|
|
650
|
+
type: 'number',
|
|
651
|
+
description: 'Capture timeout in milliseconds (default: 10000).',
|
|
652
|
+
minimum: 1000,
|
|
653
|
+
maximum: 60000,
|
|
654
|
+
},
|
|
655
|
+
},
|
|
656
|
+
required: [],
|
|
657
|
+
},
|
|
658
|
+
};
|
|
659
|
+
}
|
|
660
|
+
validate(input) {
|
|
661
|
+
if (input === undefined || input === null)
|
|
662
|
+
return { valid: true };
|
|
663
|
+
if (typeof input !== 'object')
|
|
664
|
+
return { valid: false, errors: ['Input must be an object'] };
|
|
665
|
+
const data = input;
|
|
666
|
+
if (data.prompt !== undefined && typeof data.prompt !== 'string') {
|
|
667
|
+
return { valid: false, errors: ['prompt must be a string'] };
|
|
668
|
+
}
|
|
669
|
+
if (data.device !== undefined && typeof data.device !== 'string') {
|
|
670
|
+
return { valid: false, errors: ['device must be a string'] };
|
|
671
|
+
}
|
|
672
|
+
if (data.model !== undefined && typeof data.model !== 'string') {
|
|
673
|
+
return { valid: false, errors: ['model must be a string'] };
|
|
674
|
+
}
|
|
675
|
+
if (data.output_path !== undefined && typeof data.output_path !== 'string') {
|
|
676
|
+
return { valid: false, errors: ['output_path must be a string'] };
|
|
677
|
+
}
|
|
678
|
+
if (data.timeout_ms !== undefined && typeof data.timeout_ms !== 'number') {
|
|
679
|
+
return { valid: false, errors: ['timeout_ms must be a number'] };
|
|
680
|
+
}
|
|
681
|
+
return { valid: true };
|
|
682
|
+
}
|
|
683
|
+
getMetadata() {
|
|
684
|
+
return {
|
|
685
|
+
name: this.name,
|
|
686
|
+
description: this.description,
|
|
687
|
+
category: 'media',
|
|
688
|
+
keywords: ['camera', 'webcam', 'see', 'vision', 'describe', 'look', 'photo', 'companion', 'eyes', 'analyze'],
|
|
689
|
+
priority: 7,
|
|
690
|
+
modifiesFiles: true,
|
|
691
|
+
requiresConfirmation: true,
|
|
692
|
+
makesNetworkRequests: true,
|
|
693
|
+
};
|
|
694
|
+
}
|
|
695
|
+
isAvailable() { return true; }
|
|
696
|
+
}
|
|
447
697
|
// ============================================================================
|
|
448
698
|
// Factory Function
|
|
449
699
|
// ============================================================================
|
|
@@ -454,8 +704,48 @@ export function createVisionTools(options = {}) {
|
|
|
454
704
|
new OcrExtractTool(),
|
|
455
705
|
new ImageAnalyzeTool(),
|
|
456
706
|
new CameraSnapshotTool(),
|
|
707
|
+
new CameraAnalyzeTool(options),
|
|
457
708
|
];
|
|
458
709
|
}
|
|
710
|
+
/**
|
|
711
|
+
* Resolve the Ollama OpenAI-compatible chat endpoint, honoring OLLAMA_HOST.
|
|
712
|
+
* OLLAMA_HOST is typically `host:port` with no scheme and no /v1 path.
|
|
713
|
+
*/
|
|
714
|
+
export function resolveOllamaChatEndpoint(env) {
|
|
715
|
+
const raw = (env.OLLAMA_HOST ?? '').trim();
|
|
716
|
+
if (!raw)
|
|
717
|
+
return 'http://localhost:11434/v1/chat/completions';
|
|
718
|
+
const withScheme = /^https?:\/\//i.test(raw) ? raw : `http://${raw}`;
|
|
719
|
+
const base = withScheme.replace(/\/+$/, '').replace(/\/v1$/i, '');
|
|
720
|
+
return `${base}/v1/chat/completions`;
|
|
721
|
+
}
|
|
722
|
+
/**
|
|
723
|
+
* Extract assistant text from an OpenAI-compatible chat completion. The
|
|
724
|
+
* `content` field may be a plain string or an array of typed parts.
|
|
725
|
+
*/
|
|
726
|
+
export function extractCompletionText(response) {
|
|
727
|
+
if (!response || typeof response !== 'object')
|
|
728
|
+
return undefined;
|
|
729
|
+
const choices = response.choices;
|
|
730
|
+
if (!Array.isArray(choices) || choices.length === 0)
|
|
731
|
+
return undefined;
|
|
732
|
+
const first = choices[0];
|
|
733
|
+
if (!first || typeof first !== 'object')
|
|
734
|
+
return undefined;
|
|
735
|
+
const message = first.message;
|
|
736
|
+
if (!message || typeof message !== 'object')
|
|
737
|
+
return undefined;
|
|
738
|
+
const content = message.content;
|
|
739
|
+
if (typeof content === 'string')
|
|
740
|
+
return content.trim() || undefined;
|
|
741
|
+
if (Array.isArray(content)) {
|
|
742
|
+
const parts = content
|
|
743
|
+
.map(part => (part && typeof part === 'object' ? part.text : undefined))
|
|
744
|
+
.filter((part) => typeof part === 'string' && part.trim().length > 0);
|
|
745
|
+
return parts.join('\n').trim() || undefined;
|
|
746
|
+
}
|
|
747
|
+
return undefined;
|
|
748
|
+
}
|
|
459
749
|
function requiredString(data, key) {
|
|
460
750
|
const value = optionalString(data, key);
|
|
461
751
|
if (!value) {
|
|
@@ -470,4 +760,16 @@ function optionalString(data, key) {
|
|
|
470
760
|
function sanitizeFilename(id) {
|
|
471
761
|
return id.trim().replace(/[^A-Za-z0-9_.-]+/g, '-').replace(/^-+|-+$/g, '') || String(Date.now());
|
|
472
762
|
}
|
|
763
|
+
async function delay(ms) {
|
|
764
|
+
await new Promise(resolve => setTimeout(resolve, ms));
|
|
765
|
+
}
|
|
766
|
+
function combineScreenshotFailures(...results) {
|
|
767
|
+
const errors = results
|
|
768
|
+
.map(result => result.error || result.output)
|
|
769
|
+
.filter((message) => typeof message === 'string' && message.trim().length > 0);
|
|
770
|
+
return {
|
|
771
|
+
success: false,
|
|
772
|
+
error: `Browser screenshot failed after retries: ${errors.join(' | ') || 'unknown error'}`,
|
|
773
|
+
};
|
|
774
|
+
}
|
|
473
775
|
//# sourceMappingURL=vision-tools.js.map
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { OfficeMacroTool } from '../office-macro-tool.js';
|
|
2
|
+
class OfficeMacroExecuteAdapter {
|
|
3
|
+
name = 'office_macro_execute';
|
|
4
|
+
description = 'Executes VBA or PowerShell macros directly in Windows Microsoft Office applications.';
|
|
5
|
+
impl = new OfficeMacroTool();
|
|
6
|
+
schema = {
|
|
7
|
+
name: 'office_macro_execute',
|
|
8
|
+
description: 'Executes VBA or PowerShell macros directly in Windows Microsoft Office applications.',
|
|
9
|
+
parameters: {
|
|
10
|
+
type: "object",
|
|
11
|
+
properties: {
|
|
12
|
+
application: { type: "string" },
|
|
13
|
+
macroCode: { type: "string" },
|
|
14
|
+
type: { type: "string" },
|
|
15
|
+
runHeadless: { type: "boolean" },
|
|
16
|
+
},
|
|
17
|
+
required: ["application", "macroCode", "type"],
|
|
18
|
+
},
|
|
19
|
+
};
|
|
20
|
+
metadata = {
|
|
21
|
+
name: 'office_macro_execute',
|
|
22
|
+
category: 'system',
|
|
23
|
+
priority: 1,
|
|
24
|
+
description: 'Executes macros in Microsoft Office (Windows only).',
|
|
25
|
+
keywords: ['office', 'excel', 'word', 'powerpoint', 'vba', 'macro', 'windows', 'com'],
|
|
26
|
+
};
|
|
27
|
+
getSchema() {
|
|
28
|
+
return this.schema;
|
|
29
|
+
}
|
|
30
|
+
async execute(args, context) {
|
|
31
|
+
const input = args;
|
|
32
|
+
return this.impl.execute(input);
|
|
33
|
+
}
|
|
34
|
+
validate(args) {
|
|
35
|
+
if (!args.application)
|
|
36
|
+
return { valid: false, errors: ['Missing application'] };
|
|
37
|
+
if (!args.macroCode)
|
|
38
|
+
return { valid: false, errors: ['Missing macroCode'] };
|
|
39
|
+
if (!args.type)
|
|
40
|
+
return { valid: false, errors: ['Missing type'] };
|
|
41
|
+
return { valid: true };
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
export function createWindowsTools() {
|
|
45
|
+
return [new OfficeMacroExecuteAdapter()];
|
|
46
|
+
}
|
|
47
|
+
//# sourceMappingURL=windows-tools.js.map
|
package/dist/tools/registry.d.ts
CHANGED
|
@@ -67,6 +67,12 @@ export declare class ToolRegistry {
|
|
|
67
67
|
* @returns The registered tool object or undefined if not found
|
|
68
68
|
*/
|
|
69
69
|
getTool(name: string): RegisteredTool | undefined;
|
|
70
|
+
/**
|
|
71
|
+
* Remove a registered tool by name (used to revert self-authored tools).
|
|
72
|
+
*
|
|
73
|
+
* @returns true if a tool was removed, false if it was not registered
|
|
74
|
+
*/
|
|
75
|
+
removeTool(name: string): boolean;
|
|
70
76
|
/**
|
|
71
77
|
* Check if a tool is registered and currently enabled.
|
|
72
78
|
*
|
package/dist/tools/registry.js
CHANGED
|
@@ -87,6 +87,14 @@ export class ToolRegistry {
|
|
|
87
87
|
getTool(name) {
|
|
88
88
|
return this.tools.get(name);
|
|
89
89
|
}
|
|
90
|
+
/**
|
|
91
|
+
* Remove a registered tool by name (used to revert self-authored tools).
|
|
92
|
+
*
|
|
93
|
+
* @returns true if a tool was removed, false if it was not registered
|
|
94
|
+
*/
|
|
95
|
+
removeTool(name) {
|
|
96
|
+
return this.tools.delete(name);
|
|
97
|
+
}
|
|
90
98
|
/**
|
|
91
99
|
* Check if a tool is registered and currently enabled.
|
|
92
100
|
*
|
|
@@ -10,6 +10,7 @@ import { DEFAULT_DISPATCH_POLICY_PREVIEW_TOOLS, FLEET_DISPATCH_PROFILES, buildHe
|
|
|
10
10
|
import { resolveActiveCustomAgentDispatchProfile, shouldPropagateResolvedDispatchProfile, } from '../agent/custom/custom-agent-runtime.js';
|
|
11
11
|
import { getFleetRegistry } from '../fleet/fleet-registry.js';
|
|
12
12
|
import { classifyTaskComplexity } from '../optimization/model-routing.js';
|
|
13
|
+
import { getGlobalEventBus } from '../events/event-bus.js';
|
|
13
14
|
const DEFAULT_DESCRIBE_TIMEOUT_MS = 5_000;
|
|
14
15
|
const MAX_CHAIN_ROLES = 6;
|
|
15
16
|
export async function executeRoutePeer(params) {
|
|
@@ -153,6 +154,19 @@ export async function executeRoutePeer(params) {
|
|
|
153
154
|
},
|
|
154
155
|
...(chainNextCalls ? { nextCalls: chainNextCalls } : {}),
|
|
155
156
|
};
|
|
157
|
+
try {
|
|
158
|
+
if (output.recommendation && output.recommendation.peer) {
|
|
159
|
+
getGlobalEventBus().emit('fleet:activity', {
|
|
160
|
+
activityType: 'fleet.route',
|
|
161
|
+
title: 'Fleet Route Planned',
|
|
162
|
+
description: `Task routed to peer ${output.recommendation.peer}`,
|
|
163
|
+
metadata: { peer: output.recommendation.peer, model: output.recommendation.model, prompt: params.prompt }
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
catch (err) {
|
|
168
|
+
// ignore
|
|
169
|
+
}
|
|
156
170
|
return {
|
|
157
171
|
success: true,
|
|
158
172
|
output: JSON.stringify(output, null, 2),
|
|
@@ -167,8 +167,20 @@ export class ScreenshotTool {
|
|
|
167
167
|
args.push('-f', outputPath);
|
|
168
168
|
}
|
|
169
169
|
else if (availableTool === 'import') {
|
|
170
|
-
// ImageMagick import
|
|
171
|
-
|
|
170
|
+
// ImageMagick import. Without an explicit window/region, `import` enters
|
|
171
|
+
// interactive crosshair mode and blocks forever waiting for a click — so
|
|
172
|
+
// a headless fullscreen capture MUST target the root window. Only an
|
|
173
|
+
// explicit region (-crop) leaves the root grab intact.
|
|
174
|
+
if (options.region) {
|
|
175
|
+
args.push('-window', 'root');
|
|
176
|
+
args.push('-crop', `${options.region.width}x${options.region.height}+${options.region.x}+${options.region.y}`,
|
|
177
|
+
// Reset the virtual canvas so the saved PNG is a clean WxH+0+0, not the
|
|
178
|
+
// full-screen page geometry with a crop offset.
|
|
179
|
+
'+repage');
|
|
180
|
+
}
|
|
181
|
+
else {
|
|
182
|
+
// fullscreen or active-window: capture the whole root (import's -window
|
|
183
|
+
// can't reliably target a foreign window by title on X11 without an id).
|
|
172
184
|
args.push('-window', 'root');
|
|
173
185
|
}
|
|
174
186
|
args.push(outputPath);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { spawn } from 'child_process';
|
|
2
|
-
export type TextToSpeechProvider = 'auto' | 'system' | 'edge-tts' | 'espeak' | 'say' | '
|
|
2
|
+
export type TextToSpeechProvider = 'auto' | 'system' | 'edge-tts' | 'espeak' | 'say' | 'audioreader';
|
|
3
3
|
export interface TextToSpeechOptions {
|
|
4
4
|
rootDir?: string;
|
|
5
5
|
now?: () => Date;
|
|
@@ -100,7 +100,7 @@ async function resolveProvider(provider, options) {
|
|
|
100
100
|
if (await commandExists('espeak', { platform })) {
|
|
101
101
|
return 'espeak';
|
|
102
102
|
}
|
|
103
|
-
throw new Error('No local TTS provider available. Install edge-tts/espeak, use macOS say, Windows PowerShell SAPI, or choose
|
|
103
|
+
throw new Error('No local TTS provider available. Install edge-tts/espeak, use macOS say, Windows PowerShell SAPI, or choose audioreader when configured.');
|
|
104
104
|
}
|
|
105
105
|
function resolveOutputFormat(requested, provider, outputPath) {
|
|
106
106
|
if (outputPath) {
|
|
@@ -126,7 +126,6 @@ function validateProviderFormat(provider, format) {
|
|
|
126
126
|
'edge-tts': ['mp3'],
|
|
127
127
|
espeak: ['wav'],
|
|
128
128
|
say: ['aiff'],
|
|
129
|
-
kokoro: ['wav'],
|
|
130
129
|
audioreader: ['wav'],
|
|
131
130
|
};
|
|
132
131
|
if (!supported[provider].includes(format)) {
|
|
@@ -180,15 +179,6 @@ function buildProviderCommand(provider, input, options) {
|
|
|
180
179
|
input.text,
|
|
181
180
|
],
|
|
182
181
|
};
|
|
183
|
-
case 'kokoro':
|
|
184
|
-
return {
|
|
185
|
-
kind: 'node',
|
|
186
|
-
run: async () => {
|
|
187
|
-
const { kokoroTtsService } = await import('../utils/kokoro-tts.js');
|
|
188
|
-
const buffer = await kokoroTtsService.generateSpeech(input.text, input.voice ?? 'af_bella');
|
|
189
|
-
await fs.writeFile(input.outputPath, buffer);
|
|
190
|
-
},
|
|
191
|
-
};
|
|
192
182
|
case 'audioreader':
|
|
193
183
|
return {
|
|
194
184
|
kind: 'node',
|
|
@@ -197,7 +187,7 @@ function buildProviderCommand(provider, input, options) {
|
|
|
197
187
|
method: 'POST',
|
|
198
188
|
headers: { 'Content-Type': 'application/json' },
|
|
199
189
|
body: JSON.stringify({
|
|
200
|
-
model: '
|
|
190
|
+
model: 'audioreader',
|
|
201
191
|
input: input.text,
|
|
202
192
|
voice: input.voice ?? 'ff_siwis',
|
|
203
193
|
speed: 1.0,
|
|
@@ -14,63 +14,7 @@
|
|
|
14
14
|
* - Adaptive thresholds based on success metrics
|
|
15
15
|
* - LRU cache for repeated queries
|
|
16
16
|
*/
|
|
17
|
-
|
|
18
|
-
* Simple LRU Cache implementation
|
|
19
|
-
*/
|
|
20
|
-
class LRUCache {
|
|
21
|
-
cache;
|
|
22
|
-
maxSize;
|
|
23
|
-
ttlMs;
|
|
24
|
-
constructor(maxSize = 100, ttlMs = 5 * 60 * 1000) {
|
|
25
|
-
this.cache = new Map();
|
|
26
|
-
this.maxSize = maxSize;
|
|
27
|
-
this.ttlMs = ttlMs;
|
|
28
|
-
}
|
|
29
|
-
get(key) {
|
|
30
|
-
const entry = this.cache.get(key);
|
|
31
|
-
if (!entry)
|
|
32
|
-
return undefined;
|
|
33
|
-
// Check TTL
|
|
34
|
-
if (Date.now() - entry.timestamp > this.ttlMs) {
|
|
35
|
-
this.cache.delete(key);
|
|
36
|
-
return undefined;
|
|
37
|
-
}
|
|
38
|
-
// Update access count and move to end (most recently used)
|
|
39
|
-
entry.accessCount++;
|
|
40
|
-
this.cache.delete(key);
|
|
41
|
-
this.cache.set(key, entry);
|
|
42
|
-
return entry.value;
|
|
43
|
-
}
|
|
44
|
-
set(key, value) {
|
|
45
|
-
// Remove oldest if at capacity
|
|
46
|
-
if (this.cache.size >= this.maxSize) {
|
|
47
|
-
const firstKey = this.cache.keys().next().value;
|
|
48
|
-
if (firstKey !== undefined) {
|
|
49
|
-
this.cache.delete(firstKey);
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
this.cache.set(key, {
|
|
53
|
-
value,
|
|
54
|
-
timestamp: Date.now(),
|
|
55
|
-
accessCount: 1
|
|
56
|
-
});
|
|
57
|
-
}
|
|
58
|
-
has(key) {
|
|
59
|
-
return this.get(key) !== undefined;
|
|
60
|
-
}
|
|
61
|
-
clear() {
|
|
62
|
-
this.cache.clear();
|
|
63
|
-
}
|
|
64
|
-
get size() {
|
|
65
|
-
return this.cache.size;
|
|
66
|
-
}
|
|
67
|
-
getStats() {
|
|
68
|
-
return {
|
|
69
|
-
size: this.cache.size,
|
|
70
|
-
hitRate: 0 // Would need to track hits/misses
|
|
71
|
-
};
|
|
72
|
-
}
|
|
73
|
-
}
|
|
17
|
+
import { LRUCache } from '../utils/lru-cache.js';
|
|
74
18
|
import { TOOL_METADATA, CATEGORY_KEYWORDS } from "./metadata.js";
|
|
75
19
|
/**
|
|
76
20
|
* RAG-based Tool Selector.
|
|
@@ -116,8 +60,8 @@ export class ToolSelector {
|
|
|
116
60
|
lastUpdated: new Date()
|
|
117
61
|
};
|
|
118
62
|
// Initialize caches
|
|
119
|
-
this.classificationCache = new LRUCache(100, 5 * 60 * 1000);
|
|
120
|
-
this.selectionCache = new LRUCache(50, 2 * 60 * 1000);
|
|
63
|
+
this.classificationCache = new LRUCache({ maxSize: 100, ttlMs: 5 * 60 * 1000 });
|
|
64
|
+
this.selectionCache = new LRUCache({ maxSize: 50, ttlMs: 2 * 60 * 1000 });
|
|
121
65
|
this.buildIndex();
|
|
122
66
|
}
|
|
123
67
|
/**
|