minimal-agent 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -3
- package/package.json +52 -5
- package/plugins/computer_use/.claude-plugin/plugin.json +6 -0
- package/plugins/computer_use/README.md +94 -0
- package/plugins/computer_use/app/_template.md +142 -0
- package/plugins/computer_use/app/chrome.md +165 -0
- package/plugins/computer_use/app/finder.md +76 -0
- package/plugins/computer_use/app/macos-system-settings.md +81 -0
- package/plugins/computer_use/app/microsoft-excel.md +80 -0
- package/plugins/computer_use/app/microsoft-powerpoint.md +80 -0
- package/plugins/computer_use/app/microsoft-word.md +79 -0
- package/plugins/computer_use/app/wechat.md +413 -0
- package/plugins/computer_use/app/weread.md +319 -0
- package/plugins/computer_use/commands/computeruse.md +25 -0
- package/plugins/computer_use/commands/cu.md +9 -0
- package/plugins/computer_use/plugin.js +26 -0
- package/plugins/computer_use/plugin.ts +30 -0
- package/plugins/computer_use/src/capture.js +90 -0
- package/plugins/computer_use/src/capture.ts +104 -0
- package/plugins/computer_use/src/commands.js +91 -0
- package/plugins/computer_use/src/commands.ts +103 -0
- package/plugins/computer_use/src/daemon.js +134 -0
- package/plugins/computer_use/src/daemon.ts +161 -0
- package/plugins/computer_use/src/manuals.js +135 -0
- package/plugins/computer_use/src/manuals.ts +159 -0
- package/plugins/computer_use/src/verify.js +168 -0
- package/plugins/computer_use/src/verify.ts +219 -0
- package/plugins/computer_use/test/commands.test.ts +73 -0
- package/plugins/cron-runner/plugin.ts +38 -0
- package/plugins/cron-runner/src/commands.ts +373 -0
- package/plugins/cron-runner/src/cronTime.ts +155 -0
- package/plugins/cron-runner/src/lock.ts +79 -0
- package/plugins/cron-runner/src/runs.ts +99 -0
- package/plugins/cron-runner/src/scheduleEngine.ts +76 -0
- package/plugins/cron-runner/src/scheduler.ts +376 -0
- package/plugins/cron-runner/src/schema.ts +139 -0
- package/plugins/cron-runner/src/service.ts +193 -0
- package/plugins/cron-runner/src/store.ts +333 -0
- package/plugins/cron-runner/src/types.ts +141 -0
- package/plugins/cron-runner/test/cronTime.test.ts +41 -0
- package/plugins/cron-runner/test/plugin-integration.test.ts +47 -0
- package/plugins/cron-runner/test/scheduleEngine.test.ts +41 -0
- package/plugins/cron-runner/test/service.test.ts +76 -0
- package/plugins/cron-runner/test/store-and-scheduler.test.ts +501 -0
- package/plugins/workflow-runner/plugin.js +56 -2
- package/plugins/workflow-runner/plugin.ts +113 -0
- package/plugins/workflow-runner/src/agentCommon.js +43 -4
- package/plugins/workflow-runner/src/agentCommon.ts +435 -0
- package/plugins/workflow-runner/src/controlFlow/branch.js +40 -0
- package/plugins/workflow-runner/src/controlFlow/branch.ts +46 -0
- package/plugins/workflow-runner/src/controlFlow/loop.js +211 -0
- package/plugins/workflow-runner/src/controlFlow/loop.ts +248 -0
- package/plugins/workflow-runner/src/controlFlow/pause.js +129 -0
- package/plugins/workflow-runner/src/controlFlow/pause.ts +141 -0
- package/plugins/workflow-runner/src/controlFlow/signal.js +21 -0
- package/plugins/workflow-runner/src/controlFlow/signal.ts +29 -0
- package/plugins/workflow-runner/src/controlFlow/types.js +18 -0
- package/plugins/workflow-runner/src/controlFlow/types.ts +63 -0
- package/plugins/workflow-runner/src/dslManifest.ts +127 -0
- package/plugins/workflow-runner/src/expressions.ts +622 -0
- package/plugins/workflow-runner/src/index.js +13 -4
- package/plugins/workflow-runner/src/index.ts +338 -0
- package/plugins/workflow-runner/src/jsonPayload.ts +84 -0
- package/plugins/workflow-runner/src/loader.js +23 -0
- package/plugins/workflow-runner/src/loader.ts +721 -0
- package/plugins/workflow-runner/src/multimodal.js +2 -1
- package/plugins/workflow-runner/src/multimodal.ts +64 -0
- package/plugins/workflow-runner/src/recovery.ts +149 -0
- package/plugins/workflow-runner/src/runner.js +75 -659
- package/plugins/workflow-runner/src/runner.ts +604 -0
- package/plugins/workflow-runner/src/stepExecutors/agent.ts +234 -0
- package/plugins/workflow-runner/src/stepExecutors/assert.ts +50 -0
- package/plugins/workflow-runner/src/stepExecutors/echo.js +31 -0
- package/plugins/workflow-runner/src/stepExecutors/echo.ts +38 -0
- package/plugins/workflow-runner/src/stepExecutors/llm.ts +133 -0
- package/plugins/workflow-runner/src/stepExecutors/set.js +49 -0
- package/plugins/workflow-runner/src/stepExecutors/set.ts +56 -0
- package/plugins/workflow-runner/src/stepExecutors/skill.js +5 -6
- package/plugins/workflow-runner/src/stepExecutors/skill.ts +140 -0
- package/plugins/workflow-runner/src/stepExecutors/tool.js +80 -8
- package/plugins/workflow-runner/src/stepExecutors/tool.ts +256 -0
- package/plugins/workflow-runner/src/stepExecutors/verify.ts +68 -0
- package/plugins/workflow-runner/src/stepPipeline.js +277 -0
- package/plugins/workflow-runner/src/stepPipeline.ts +366 -0
- package/plugins/workflow-runner/src/taskIdentity.ts +405 -0
- package/plugins/workflow-runner/src/toolRuntime.js +48 -0
- package/plugins/workflow-runner/src/toolRuntime.ts +64 -0
- package/plugins/workflow-runner/src/types.js +37 -1
- package/plugins/workflow-runner/src/types.ts +602 -0
- package/plugins/workflow-runner/src/workflowState.ts +733 -0
- package/plugins/workflow-runner/test/agent.test.ts +587 -0
- package/plugins/workflow-runner/test/agentLlmError.test.ts +373 -0
- package/plugins/workflow-runner/test/bashBlacklist.sweep.test.ts +65 -0
- package/plugins/workflow-runner/test/captureMisbind.test.ts +99 -0
- package/plugins/workflow-runner/test/cli.e2e.test.ts +147 -0
- package/plugins/workflow-runner/test/context-files.test.ts +320 -0
- package/plugins/workflow-runner/test/e2e.test.ts +370 -0
- package/plugins/workflow-runner/test/engineRobustness.test.ts +226 -0
- package/plugins/workflow-runner/test/expressions.test.ts +397 -0
- package/plugins/workflow-runner/test/foundation.test.ts +126 -0
- package/plugins/workflow-runner/test/graceful.test.ts +139 -0
- package/plugins/workflow-runner/test/greedyInputs.load.test.ts +47 -0
- package/plugins/workflow-runner/test/hostCapability.test.ts +108 -0
- package/plugins/workflow-runner/test/illustrate.load.test.ts +183 -0
- package/plugins/workflow-runner/test/jsonPayload.test.ts +83 -0
- package/plugins/workflow-runner/test/loader.test.ts +617 -0
- package/plugins/workflow-runner/test/maxTurnsRetryDelay.test.ts +33 -0
- package/plugins/workflow-runner/test/multimodal.test.ts +342 -0
- package/plugins/workflow-runner/test/output-schema.test.ts +213 -0
- package/plugins/workflow-runner/test/parseWorkflowArgs.test.ts +87 -0
- package/plugins/workflow-runner/test/pluginRunner.isolation.test.ts +269 -0
- package/plugins/workflow-runner/test/ppt.load.test.ts +1220 -0
- package/plugins/workflow-runner/test/ralphLoop.e2e.test.ts +220 -0
- package/plugins/workflow-runner/test/recovery.test.ts +265 -0
- package/plugins/workflow-runner/test/runner.test.ts +911 -0
- package/plugins/workflow-runner/test/runnerPreflight.test.ts +379 -0
- package/plugins/workflow-runner/test/skillTurnDone.test.ts +71 -0
- package/plugins/workflow-runner/test/stateLifecycle.test.ts +150 -0
- package/plugins/workflow-runner/test/stepPipeline.test.ts +114 -0
- package/plugins/workflow-runner/test/taskArbitration.test.ts +338 -0
- package/plugins/workflow-runner/test/taskIdentity.test.ts +237 -0
- package/plugins/workflow-runner/test/taskLifecycle.test.ts +270 -0
- package/plugins/workflow-runner/test/toolArtifactOverwrite.test.ts +221 -0
- package/plugins/workflow-runner/test/toolCapabilityBoundary.test.ts +29 -0
- package/plugins/workflow-runner/test/toolRuntimeIntegration.test.ts +168 -0
- package/plugins/workflow-runner/test/webppt.load.test.ts +250 -0
- package/plugins/workflow-runner/test/word.load.test.ts +208 -0
- package/plugins/workflow-runner/test/workflows.smoke.test.ts +76 -0
- package/plugins/workflow-runner/test/yamlGuardConditions.test.ts +207 -0
- package/skills/browser-automation/SKILL.md +9 -19
- package/skills/web-crawler/SKILL.md +9 -11
- package/src/askBridge.ts +249 -0
- package/src/bootstrap/cwdArg.js +10 -3
- package/src/bootstrap/cwdArg.ts +31 -0
- package/src/bootstrap/workingDir.ts +48 -0
- package/src/cli/args.js +70 -7
- package/src/cli/args.ts +146 -0
- package/src/cli/computerUse.js +185 -0
- package/src/cli/computerUse.ts +221 -0
- package/src/cli/configMenu.js +274 -202
- package/src/cli/configMenu.tsx +521 -0
- package/src/cli/memoryAccess.js +45 -0
- package/src/cli/memoryAccess.ts +53 -0
- package/src/cli/print.js +119 -5
- package/src/cli/print.ts +597 -0
- package/src/cli/quickModelSetup.js +60 -0
- package/src/cli/quickModelSetup.tsx +96 -0
- package/src/cli/streamJson.js +80 -4
- package/src/cli/streamJson.ts +433 -0
- package/src/cli/wizards/advisorWizard.js +1 -0
- package/src/cli/wizards/advisorWizard.tsx +195 -0
- package/src/cli/wizards/agentRuntimeWizard.js +84 -0
- package/src/cli/wizards/agentRuntimeWizard.tsx +134 -0
- package/src/cli/wizards/browserWizard.js +55 -0
- package/src/cli/wizards/browserWizard.tsx +77 -0
- package/src/cli/wizards/contextAdvisorWizard.js +84 -0
- package/src/cli/wizards/contextAdvisorWizard.tsx +124 -0
- package/src/cli/wizards/fieldForm.js +7 -4
- package/src/cli/wizards/fieldForm.tsx +250 -0
- package/src/cli/wizards/imageWizard.js +4 -5
- package/src/cli/wizards/imageWizard.tsx +96 -0
- package/src/cli/wizards/openRouterWizard.tsx +48 -0
- package/src/cli/wizards/providerPresets.js +1 -8
- package/src/cli/wizards/providerPresets.ts +106 -0
- package/src/cli/wizards/providerWizard.js +152 -205
- package/src/cli/wizards/providerWizard.tsx +266 -0
- package/src/cli/wizards/searchWizard.tsx +47 -0
- package/src/cli/wizards/ttsWizard.js +4 -3
- package/src/cli/wizards/ttsWizard.tsx +61 -0
- package/src/config/configDefaults.js +25 -0
- package/src/config/configDefaults.ts +33 -0
- package/src/config/configDiagnostics.js +117 -0
- package/src/config/configDiagnostics.ts +166 -0
- package/src/config/configFile.js +52 -32
- package/src/config/configFile.ts +633 -0
- package/src/config/configManagement.js +126 -0
- package/src/config/configManagement.ts +159 -0
- package/src/config/configService.js +66 -11
- package/src/config/configService.ts +554 -0
- package/src/config/numericLimits.js +61 -0
- package/src/config/numericLimits.ts +80 -0
- package/src/config.ts +109 -0
- package/src/context/agentHome.ts +11 -0
- package/src/context/archive.ts +144 -0
- package/src/context/budgetPlan.js +54 -0
- package/src/context/budgetPlan.ts +88 -0
- package/src/context/compact.js +179 -106
- package/src/context/compact.ts +584 -0
- package/src/context/contextBundle.js +119 -0
- package/src/context/contextBundle.ts +176 -0
- package/src/context/contextLock.ts +84 -0
- package/src/context/microCompactLite.ts +218 -0
- package/src/context/persistContext.ts +177 -0
- package/src/context/reactiveCompact.js +19 -4
- package/src/context/reactiveCompact.ts +240 -0
- package/src/context/recentDirs.ts +42 -0
- package/src/context/sessionPath.ts +206 -0
- package/src/context/sessionRegistry.js +8 -2
- package/src/context/sessionRegistry.ts +414 -0
- package/src/context/snipCompact.ts +163 -0
- package/src/context/tokenCounter.ts +106 -0
- package/src/context/workspaceIndex.js +2 -2
- package/src/context/workspaceIndex.ts +751 -0
- package/src/entry.compiled.tsx +20 -0
- package/src/harness/claimValidation.js +9 -0
- package/src/harness/claimValidation.ts +22 -0
- package/src/harness/evidenceLedger.js +92 -0
- package/src/harness/evidenceLedger.ts +96 -0
- package/src/harness/stallGuard.js +60 -0
- package/src/harness/stallGuard.ts +81 -0
- package/src/humanIntervention.js +129 -0
- package/src/humanIntervention.ts +216 -0
- package/src/infra/verification.ts +295 -0
- package/src/irreversible/rules.js +216 -0
- package/src/irreversible/rules.ts +261 -0
- package/src/llm/client.js +40 -3
- package/src/llm/client.ts +621 -0
- package/src/llm/promptCache.js +27 -0
- package/src/llm/promptCache.ts +41 -0
- package/src/llm/providers/anthropicFormat.js +41 -5
- package/src/llm/providers/anthropicFormat.ts +290 -0
- package/src/llm/providers/openaiFormat.js +186 -8
- package/src/llm/providers/openaiFormat.ts +428 -0
- package/src/loop.js +614 -63
- package/src/loop.ts +1454 -0
- package/src/main.js +67 -10
- package/src/main.tsx +344 -0
- package/src/memory/codec.js +204 -0
- package/src/memory/codec.ts +249 -0
- package/src/memory/constants.js +34 -0
- package/src/memory/constants.ts +50 -0
- package/src/memory/episodicMemory.js +684 -0
- package/src/memory/episodicMemory.ts +815 -0
- package/src/memory/fileStore.js +148 -0
- package/src/memory/fileStore.ts +184 -0
- package/src/memory/memoryService.js +570 -0
- package/src/memory/memoryService.ts +859 -0
- package/src/memory/migration.js +128 -0
- package/src/memory/migration.ts +160 -0
- package/src/memory/repository.js +789 -0
- package/src/memory/repository.ts +893 -0
- package/src/memory/retrieval.js +108 -0
- package/src/memory/retrieval.ts +145 -0
- package/src/memory/searchIndex.js +107 -0
- package/src/memory/searchIndex.ts +153 -0
- package/src/memory/topicView.js +82 -0
- package/src/memory/topicView.ts +95 -0
- package/src/memory/turnJournal.js +95 -0
- package/src/memory/turnJournal.ts +120 -0
- package/src/memory/types.js +2 -0
- package/src/memory/types.ts +186 -0
- package/src/messageQueue.ts +123 -0
- package/src/multimodal.js +128 -46
- package/src/multimodal.ts +349 -0
- package/src/plugin-sdk.js +21 -1
- package/src/plugin-sdk.ts +134 -0
- package/src/plugins/builtinPluginIds.js +47 -0
- package/src/plugins/builtinPluginIds.ts +51 -0
- package/src/plugins/builtinRegistry.ts +43 -0
- package/src/plugins/commandRouter.js +40 -26
- package/src/plugins/commandRouter.ts +261 -0
- package/src/plugins/hookEngine.ts +341 -0
- package/src/plugins/pluginApi.ts +133 -0
- package/src/plugins/pluginLifecycle.ts +34 -0
- package/src/plugins/pluginLoader.ts +147 -0
- package/src/plugins/pluginManifest.js +127 -0
- package/src/plugins/pluginManifest.ts +152 -0
- package/src/plugins/pluginRunner.js +47 -2
- package/src/plugins/pluginRunner.ts +207 -0
- package/src/plugins/transcript.ts +208 -0
- package/src/prompts/lint.js +169 -0
- package/src/prompts/lint.ts +220 -0
- package/src/prompts/pluginList.js +32 -16
- package/src/prompts/pluginList.ts +86 -0
- package/src/prompts/policyRegistry.js +318 -0
- package/src/prompts/policyRegistry.ts +358 -0
- package/src/prompts/projectInstructions.js +24 -0
- package/src/prompts/projectInstructions.ts +81 -0
- package/src/prompts/promptAssembly.js +311 -202
- package/src/prompts/promptAssembly.ts +510 -0
- package/src/prompts/skillList.js +106 -45
- package/src/prompts/skillList.ts +198 -0
- package/src/prompts/system.js +31 -6
- package/src/prompts/system.ts +81 -0
- package/src/runtime/agentRuntimeContext.js +53 -0
- package/src/runtime/agentRuntimeContext.ts +112 -0
- package/src/session/runAgentTurn.js +71 -12
- package/src/session/runAgentTurn.ts +181 -0
- package/src/session/runTurn.js +29 -0
- package/src/session/runTurn.ts +160 -0
- package/src/session/runtimeStore.js +149 -0
- package/src/session/runtimeStore.ts +197 -0
- package/src/session/sessionState.js +79 -3
- package/src/session/sessionState.ts +134 -0
- package/src/toolGate.ts +50 -0
- package/src/tools/advisor/advisor.js +76 -15
- package/src/tools/advisor/advisor.ts +512 -0
- package/src/tools/agent/agent.js +245 -29
- package/src/tools/agent/agent.ts +837 -0
- package/src/tools/agent/agentTaskService.js +60 -0
- package/src/tools/agent/agentTaskService.ts +113 -0
- package/src/tools/agent/agentTypes.js +20 -8
- package/src/tools/agent/agentTypes.ts +94 -0
- package/src/tools/agent/coordinator.ts +111 -0
- package/src/tools/agent/worktree.ts +296 -0
- package/src/tools/ask/ask.js +17 -0
- package/src/tools/ask/ask.ts +122 -0
- package/src/tools/bash/attachImage.js +223 -0
- package/src/tools/bash/attachImage.ts +281 -0
- package/src/tools/bash/bash.js +132 -14
- package/src/tools/bash/bash.ts +642 -0
- package/src/tools/bash/semantics.ts +103 -0
- package/src/tools/bash/warnings.js +17 -11
- package/src/tools/bash/warnings.ts +117 -0
- package/src/tools/capabilities.js +42 -0
- package/src/tools/capabilities.ts +69 -0
- package/src/tools/computeruse/accessibilityPermission.js +261 -0
- package/src/tools/computeruse/accessibilityPermission.ts +294 -0
- package/src/tools/computeruse/axSearch.js +96 -0
- package/src/tools/computeruse/axSearch.ts +120 -0
- package/src/tools/computeruse/axSnap.js +272 -0
- package/src/tools/computeruse/axSnap.ts +349 -0
- package/src/tools/computeruse/capabilityLabels.js +95 -0
- package/src/tools/computeruse/capabilityLabels.ts +115 -0
- package/src/tools/computeruse/computerUseService.js +2235 -0
- package/src/tools/computeruse/computerUseService.ts +2686 -0
- package/src/tools/computeruse/computeruse.js +2742 -0
- package/src/tools/computeruse/computeruse.ts +3265 -0
- package/src/tools/computeruse/diagnosticTrace.js +154 -0
- package/src/tools/computeruse/diagnosticTrace.ts +195 -0
- package/src/tools/computeruse/diagnostics.js +369 -0
- package/src/tools/computeruse/diagnostics.ts +453 -0
- package/src/tools/computeruse/grants.js +53 -0
- package/src/tools/computeruse/grants.ts +70 -0
- package/src/tools/computeruse/humanBoundary.js +159 -0
- package/src/tools/computeruse/humanBoundary.ts +199 -0
- package/src/tools/computeruse/intervention.js +77 -0
- package/src/tools/computeruse/intervention.ts +110 -0
- package/src/tools/computeruse/manualSchema.js +709 -0
- package/src/tools/computeruse/manualSchema.ts +916 -0
- package/src/tools/computeruse/observationImages.js +705 -0
- package/src/tools/computeruse/observationImages.ts +889 -0
- package/src/tools/computeruse/protocol.js +60 -0
- package/src/tools/computeruse/protocol.ts +515 -0
- package/src/tools/computeruse/telemetryLog.js +57 -0
- package/src/tools/computeruse/telemetryLog.ts +67 -0
- package/src/tools/computeruse/vision/appGuide.js +325 -0
- package/src/tools/computeruse/vision/appGuide.ts +482 -0
- package/src/tools/computeruse/vision/focus.js +291 -0
- package/src/tools/computeruse/vision/focus.ts +382 -0
- package/src/tools/computeruse/vision/frames.js +248 -0
- package/src/tools/computeruse/vision/frames.ts +329 -0
- package/src/tools/computeruse/vision/geometry.js +131 -0
- package/src/tools/computeruse/vision/geometry.ts +163 -0
- package/src/tools/computeruse/vision/imaging.js +201 -0
- package/src/tools/computeruse/vision/imaging.ts +281 -0
- package/src/tools/computeruse/vision/png.js +274 -0
- package/src/tools/computeruse/vision/png.ts +317 -0
- package/src/tools/computeruse/vision/raster.js +320 -0
- package/src/tools/computeruse/vision/raster.ts +408 -0
- package/src/tools/computeruse/vision/settle.js +131 -0
- package/src/tools/computeruse/vision/settle.ts +220 -0
- package/src/tools/computeruse/vision/telemetry.js +196 -0
- package/src/tools/computeruse/vision/telemetry.ts +241 -0
- package/src/tools/computeruse/visionOcr.js +115 -0
- package/src/tools/computeruse/visionOcr.ts +128 -0
- package/src/tools/computeruse/waitFor.js +237 -0
- package/src/tools/computeruse/waitFor.ts +291 -0
- package/src/tools/download/download.js +10 -6
- package/src/tools/download/download.ts +187 -0
- package/src/tools/edit/edit.js +30 -18
- package/src/tools/edit/edit.ts +349 -0
- package/src/tools/edit/multi-edit.js +25 -14
- package/src/tools/edit/multi-edit.ts +223 -0
- package/src/tools/effectiveToolSet.js +73 -0
- package/src/tools/effectiveToolSet.ts +91 -0
- package/src/tools/genimage/genimage.ts +257 -0
- package/src/tools/genspeech/genspeech.ts +83 -0
- package/src/tools/glob/glob.js +176 -29
- package/src/tools/glob/glob.ts +283 -0
- package/src/tools/grep/grep.js +61 -19
- package/src/tools/grep/grep.ts +239 -0
- package/src/tools/grep/rgPath.ts +206 -0
- package/src/tools/index.js +99 -5
- package/src/tools/index.ts +293 -0
- package/src/tools/lsp/config.js +113 -0
- package/src/tools/lsp/config.ts +113 -0
- package/src/tools/lsp/lsp.js +256 -0
- package/src/tools/lsp/lsp.ts +274 -0
- package/src/tools/lsp/service.js +303 -0
- package/src/tools/lsp/service.ts +296 -0
- package/src/tools/memory/memory.js +520 -0
- package/src/tools/memory/memory.ts +625 -0
- package/src/tools/memory/policy.js +53 -0
- package/src/tools/memory/policy.ts +72 -0
- package/src/tools/notebook/notebookEdit.js +114 -0
- package/src/tools/notebook/notebookEdit.ts +126 -0
- package/src/tools/powershell/powershell.js +210 -0
- package/src/tools/powershell/powershell.ts +239 -0
- package/src/tools/presentresult/presentresult.js +145 -0
- package/src/tools/presentresult/presentresult.ts +156 -0
- package/src/tools/read/read.js +371 -163
- package/src/tools/read/read.ts +558 -0
- package/src/tools/shared/diffSummary.js +44 -0
- package/src/tools/shared/diffSummary.ts +63 -0
- package/src/tools/shared/fileActivity.js +51 -0
- package/src/tools/shared/fileActivity.ts +66 -0
- package/src/tools/shared/fileMutationBus.js +13 -0
- package/src/tools/shared/fileMutationBus.ts +14 -0
- package/src/tools/shared/fileState.js +73 -65
- package/src/tools/shared/fileState.ts +145 -0
- package/src/tools/shared/fileUtils.js +81 -4
- package/src/tools/shared/fileUtils.ts +504 -0
- package/src/tools/shared/killTree.ts +50 -0
- package/src/tools/shared/permanentDeleteGuard.js +26 -0
- package/src/tools/shared/permanentDeleteGuard.ts +34 -0
- package/src/tools/shared/schemas.ts +20 -0
- package/src/tools/shared/toolOutputStore.js +97 -0
- package/src/tools/shared/toolOutputStore.ts +110 -0
- package/src/tools/skill/skill.js +86 -0
- package/src/tools/skill/skill.ts +94 -0
- package/src/tools/snip/snip.js +67 -0
- package/src/tools/snip/snip.ts +80 -0
- package/src/tools/tasks/backgroundTaskService.js +275 -0
- package/src/tools/tasks/backgroundTaskService.ts +369 -0
- package/src/tools/tasks/processEvidence.js +24 -0
- package/src/tools/tasks/processEvidence.ts +34 -0
- package/src/tools/tasks/taskManagement.js +163 -0
- package/src/tools/tasks/taskManagement.ts +180 -0
- package/src/tools/tasks/taskOutput.js +113 -0
- package/src/tools/tasks/taskOutput.ts +145 -0
- package/src/tools/tasks/taskStop.js +74 -0
- package/src/tools/tasks/taskStop.ts +99 -0
- package/src/tools/tasks/taskStore.js +46 -0
- package/src/tools/tasks/taskStore.ts +77 -0
- package/src/tools/terminal/terminalCapture.js +167 -0
- package/src/tools/terminal/terminalCapture.ts +179 -0
- package/src/tools/trash/trash.js +233 -0
- package/src/tools/trash/trash.ts +283 -0
- package/src/tools/types.js +1 -1
- package/src/tools/types.ts +24 -0
- package/src/tools/webbrowser/agentBrowserPath.ts +286 -0
- package/src/tools/webbrowser/browser.js +214 -11
- package/src/tools/webbrowser/browser.ts +293 -0
- package/src/tools/webbrowser/browserControl.js +153 -48
- package/src/tools/webbrowser/browserControl.ts +262 -0
- package/src/tools/webbrowser/browserDaemon.js +147 -39
- package/src/tools/webbrowser/browserDaemon.ts +270 -0
- package/src/tools/webbrowser/browserOutcome.js +273 -0
- package/src/tools/webbrowser/browserOutcome.ts +306 -0
- package/src/tools/webbrowser/browserRunRegistry.js +150 -0
- package/src/tools/webbrowser/browserRunRegistry.ts +179 -0
- package/src/tools/webbrowser/browserService.js +20 -6
- package/src/tools/webbrowser/browserService.ts +56 -0
- package/src/tools/webbrowser/browserStream.js +30 -1
- package/src/tools/webbrowser/browserStream.ts +216 -0
- package/src/tools/webbrowser/browserTabs.js +143 -0
- package/src/tools/webbrowser/browserTabs.ts +210 -0
- package/src/tools/webbrowser/webbrowser.js +831 -150
- package/src/tools/webbrowser/webbrowser.ts +1537 -0
- package/src/tools/webfetch/preapproved.ts +274 -0
- package/src/tools/webfetch/webfetch.js +24 -23
- package/src/tools/webfetch/webfetch.ts +383 -0
- package/src/tools/websearch/websearch.js +12 -11
- package/src/tools/websearch/websearch.ts +292 -0
- package/src/tools/worktree/sessionWorktree.js +128 -0
- package/src/tools/worktree/sessionWorktree.ts +158 -0
- package/src/tools/write/write.js +27 -44
- package/src/tools/write/write.ts +136 -0
- package/src/types.ts +960 -0
- package/src/ui/App.js +216 -12
- package/src/ui/App.tsx +879 -0
- package/src/ui/AskQuestion.tsx +300 -0
- package/src/ui/InputBox.js +77 -1
- package/src/ui/InputBox.tsx +567 -0
- package/src/ui/LiveArea.tsx +344 -0
- package/src/ui/MessageList.js +27 -6
- package/src/ui/MessageList.tsx +559 -0
- package/src/ui/ModelPicker.tsx +107 -0
- package/src/ui/QueuedMessagesBanner.tsx +51 -0
- package/src/ui/Root.js +6 -8
- package/src/ui/Root.tsx +100 -0
- package/src/ui/SessionPicker.tsx +169 -0
- package/src/ui/StatusLine.js +22 -1
- package/src/ui/StatusLine.tsx +145 -0
- package/src/ui/WorkspacePicker.tsx +84 -0
- package/src/ui/advisorCommand.js +54 -0
- package/src/ui/advisorCommand.ts +63 -0
- package/src/ui/computerCommand.js +19 -0
- package/src/ui/computerCommand.ts +35 -0
- package/src/ui/figures.ts +45 -0
- package/src/ui/hooks/askSlot.ts +112 -0
- package/src/ui/hooks/useChat.js +271 -7
- package/src/ui/hooks/useChat.ts +1578 -0
- package/src/ui/hooks/useInputHistory.ts +235 -0
- package/src/ui/hooks/usePasteHandler.ts +220 -0
- package/src/ui/hooks/useTerminalSize.ts +41 -0
- package/src/ui/hooks/useTextBuffer.ts +108 -0
- package/src/ui/hooks/useTokenUsage.ts +44 -0
- package/src/ui/liveViewport.ts +189 -0
- package/src/ui/markdown.ts +230 -0
- package/src/ui/textBuffer.ts +239 -0
- package/src/ui/theme.ts +35 -0
- package/src/utils/clipboardImage.js +358 -0
- package/src/utils/clipboardImage.ts +420 -0
- package/src/utils/debugLog.ts +50 -0
- package/src/utils/greenRoot.ts +35 -0
- package/src/utils/packageRoot.ts +40 -0
- package/src/utils/pathContains.js +28 -0
- package/src/utils/pathContains.ts +31 -0
- package/src/utils/resourcePaths.ts +71 -0
- package/src/utils/stripThink.ts +49 -0
- package/src/utils/zodToJson.ts +73 -0
- package/src/web/browserBridge.js +365 -66
- package/src/web/browserBridge.ts +672 -0
- package/src/web/protocol.ts +1804 -0
- package/vendor/computer-use/arm64-darwin/cu-helperd +0 -0
- package/workflows/ppt.yaml +13 -2
- package/workflows/wechat-reading-shots.yaml +315 -0
- package/workflows/word.yaml +11 -1
- package/skills/docx/scripts/office/helpers/__pycache__/__init__.cpython-314.pyc +0 -0
- package/skills/docx/scripts/office/helpers/__pycache__/merge_runs.cpython-314.pyc +0 -0
- package/skills/docx/scripts/office/helpers/__pycache__/simplify_redlines.cpython-314.pyc +0 -0
- package/skills/docx/scripts/office/validators/__pycache__/__init__.cpython-314.pyc +0 -0
- package/skills/docx/scripts/office/validators/__pycache__/base.cpython-314.pyc +0 -0
- package/skills/docx/scripts/office/validators/__pycache__/docx.cpython-314.pyc +0 -0
- package/skills/docx/scripts/office/validators/__pycache__/pptx.cpython-314.pyc +0 -0
- package/skills/docx/scripts/office/validators/__pycache__/redlining.cpython-314.pyc +0 -0
- package/skills/image-stats/scripts/__pycache__/image_stats.cpython-314.pyc +0 -0
- package/skills/image-stats/scripts/__pycache__/image_stats_test.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/assemble_material.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/assemble_material_test.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/chart_sanity_check.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/check_language_policy.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/check_ui_artifact.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/check_ui_artifact_pptx_test.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/deck_pages_check.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/deck_pages_check_test.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/deck_visual_style_check.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/deck_visual_style_check_test.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/design_budget_check.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/design_budget_check_test.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/design_repair_hints.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/design_repair_hints_test.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/design_style_check.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/design_style_check_test.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/draft_coverage_check.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/extract_source_assets.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/extract_source_assets_test.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/generated_image_review_collect.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/generated_image_review_collect_test.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/image_plan_lint.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/image_plan_lint_test.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/image_prompt_rewrite.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/image_prompt_rewrite_test.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/layout_balance.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/layout_balance_test.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/normalize_design_json.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/normalize_design_json_test.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/plan_source_layout.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/plan_source_layout_test.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/resolve_source_hint.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/resolve_source_hint_test.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/resolve_source_input.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/resolve_source_input_test.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/scaffold_ui_artifact.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/source_usage_check.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/source_usage_check_test.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/text_readability_pixel.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/visual_quality_gate.cpython-314.pyc +0 -0
- package/skills/ui-design/scripts/__pycache__/visual_quality_gate_test.cpython-314.pyc +0 -0
- /package/skills/ai-image/scripts/{generate_test.mjs → generate.selftest.mjs} +0 -0
- /package/skills/ai-image/scripts/{generate_from_plan_test.mjs → generate_from_plan.selftest.mjs} +0 -0
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ============================================================
|
|
3
|
+
* vision/focus.ts —— 焦点通道(纯函数)
|
|
4
|
+
* ------------------------------------------------------------
|
|
5
|
+
* 焦点是与像素并列的**辅助**通道(不是主判据,原因见下):
|
|
6
|
+
*
|
|
7
|
+
* ① L1 绕开了「应用不暴露 UI 结构」这个前提。
|
|
8
|
+
* 前台窗口/进程不归应用管、归 OS 管。微信可以拒绝暴露控件树,它拒绝不了
|
|
9
|
+
* 「我这个窗口是不是前台窗口」。**跨进程打断因此永远可检出**。
|
|
10
|
+
*
|
|
11
|
+
* ② 它比截图便宜 20 倍。
|
|
12
|
+
* 一次焦点采样约 50 token,一张全屏截图约 1100 token。
|
|
13
|
+
*
|
|
14
|
+
* ★ 一条已被推翻的旧论断,留在这里免得有人再推一遍:
|
|
15
|
+
*
|
|
16
|
+
* 曾经的说法是「输入法必须知道 caret 矩形才能画候选词框,所以只要支持中文
|
|
17
|
+
* 输入,自绘应用也堵不上 caret 泄漏」。**这是错的。** IME 通道(TSF/IMM、
|
|
18
|
+
* TSM)与无障碍通道(GetGUIThreadInfo、AX)是两条独立路径,前者不向第三方
|
|
19
|
+
* 进程暴露。已查证:
|
|
20
|
+
* - Windows:`GetGUIThreadInfo.hwndCaret` 对 Chromium/Electron 恒为 NULL
|
|
21
|
+
* - macOS:`AXSelectedTextRange` 在 MAS 版 Electron 被整体 patch 掉
|
|
22
|
+
* 而 Electron / 自绘控件正是目标场景。本仓库实测也确认:Claude 桌面端
|
|
23
|
+
* 只暴露窗口与三个窗口按钮,内容对 AX 完全不可见。
|
|
24
|
+
*
|
|
25
|
+
* **因此 L2 不可假定可用**,settle 的主判据必须是像素(见 settle.ts),
|
|
26
|
+
* 焦点只做增强与跨进程打断检测。
|
|
27
|
+
* ============================================================
|
|
28
|
+
*/
|
|
29
|
+
/** 层级语义。原生侧拿不到高层就降级,但必须**如实报告拿到了哪层**。 */
|
|
30
|
+
export const FOCUS_LEVELS = {
|
|
31
|
+
1: '前台窗口/进程(总是可用)',
|
|
32
|
+
2: '焦点控件 + caret 矩形(自绘/Electron 应用上常常拿不到)',
|
|
33
|
+
3: '输入法合成状态(看应用)',
|
|
34
|
+
};
|
|
35
|
+
/**
|
|
36
|
+
* 焦点变化的三种语义。这是**引擎级通用对策**,不是任何 app 的特判 ——
|
|
37
|
+
* `other-process` 只用 L1 就能算,所以对自绘应用同样成立。
|
|
38
|
+
*
|
|
39
|
+
* same-window → app 自己的交互反馈(弹联想、展开面板、切 tab);需 L2
|
|
40
|
+
* other-process → 外部打断(通知、别的 app 抢前台);L1 即可
|
|
41
|
+
* expected → 就是我刚点的目标,正常
|
|
42
|
+
* none → 没变
|
|
43
|
+
*/
|
|
44
|
+
export const FOCUS_CHANGE = {
|
|
45
|
+
NONE: 'none',
|
|
46
|
+
EXPECTED: 'expected',
|
|
47
|
+
SAME_WINDOW: 'same-window',
|
|
48
|
+
OTHER_PROCESS: 'other-process',
|
|
49
|
+
};
|
|
50
|
+
/** 比较两个焦点快照,判定变化语义。 */
|
|
51
|
+
export function classifyFocusChange(before, after, expectation = {}) {
|
|
52
|
+
if (!before || !after)
|
|
53
|
+
return FOCUS_CHANGE.NONE;
|
|
54
|
+
// 进程换了就是换了 —— 这一项**必须**独立参与判定。
|
|
55
|
+
//
|
|
56
|
+
// 曾经的写法只看窗口键/caret/role,于是当两个应用的窗口键恰好相同(或两边
|
|
57
|
+
// 都拿不到窗口 id、标题也一样)时,一次真实的跨进程抢前台会被判成 `none`。
|
|
58
|
+
// 而 L1 的全部价值就在于「进程身份是最可靠的那一层」,让它取决于更弱的
|
|
59
|
+
// 窗口信息是把依赖关系倒过来了。
|
|
60
|
+
const processChanged = (before.pid ?? '') !== (after.pid ?? '')
|
|
61
|
+
|| before.app !== after.app;
|
|
62
|
+
const windowChanged = keyOf(before) !== keyOf(after);
|
|
63
|
+
const caretMoved = !sameRect(before.caret, after.caret);
|
|
64
|
+
const roleChanged = (before.role ?? '') !== (after.role ?? '');
|
|
65
|
+
if (!processChanged && !windowChanged && !caretMoved && !roleChanged) {
|
|
66
|
+
return FOCUS_CHANGE.NONE;
|
|
67
|
+
}
|
|
68
|
+
// 预期之内:这次动作本来就是要把焦点挪过去
|
|
69
|
+
if (expectation.expectWindowId &&
|
|
70
|
+
after.windowId === expectation.expectWindowId &&
|
|
71
|
+
(!expectation.expectRole || after.role === expectation.expectRole)) {
|
|
72
|
+
return FOCUS_CHANGE.EXPECTED;
|
|
73
|
+
}
|
|
74
|
+
if (expectation.expectAppId && matchesApp(after, expectation.expectAppId)) {
|
|
75
|
+
return FOCUS_CHANGE.EXPECTED;
|
|
76
|
+
}
|
|
77
|
+
if (processChanged)
|
|
78
|
+
return FOCUS_CHANGE.OTHER_PROCESS;
|
|
79
|
+
return FOCUS_CHANGE.SAME_WINDOW;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* 变化语义 → 建议动作。写死在引擎里当默认,`apps/<app>.md` 可以描述例外
|
|
83
|
+
* (但那是**事实**描述,覆盖策略由模型读了 md 之后自己决定,不是这里加分支)。
|
|
84
|
+
*/
|
|
85
|
+
export function recommendedAction(change) {
|
|
86
|
+
switch (change) {
|
|
87
|
+
case FOCUS_CHANGE.SAME_WINDOW:
|
|
88
|
+
return { abortTyping: true, advice: '应用弹出了新界面元素,先观察再继续,不要盲目输入' };
|
|
89
|
+
case FOCUS_CHANGE.OTHER_PROCESS:
|
|
90
|
+
return { abortTyping: true, advice: '被其它进程抢走焦点,需先抢回前台再重试' };
|
|
91
|
+
case FOCUS_CHANGE.EXPECTED:
|
|
92
|
+
return { abortTyping: false, advice: '焦点如期落位' };
|
|
93
|
+
default:
|
|
94
|
+
return { abortTyping: false, advice: '焦点未变' };
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* 从一次观测里派生焦点快照。
|
|
99
|
+
*
|
|
100
|
+
* **零协议改动**:协议里没有专门的焦点快照字段,但需要的两件事都已经在观测里了 ——
|
|
101
|
+
* `activeApp`/`windows` 给 L1,`AccessibilityNode.focused`(原生侧从 `AXFocused` 采集)
|
|
102
|
+
* 给 L2。所以这里是纯派生,不需要多跑一次原生调用。
|
|
103
|
+
*
|
|
104
|
+
* `level` 必须如实:
|
|
105
|
+
* - 找到带 bounds 的焦点控件 → **L2**(能说出焦点落在哪个控件、在哪)
|
|
106
|
+
* - 只有前台应用/窗口 → **L1**
|
|
107
|
+
* 刻意不产出 L3:输入法合成状态在本项目里根本没有采集通道,报 L3 就是撒谎
|
|
108
|
+
* (本文件头注记录了「IME 能拿到 caret」这个已被推翻的论断,别再往回走)。
|
|
109
|
+
*
|
|
110
|
+
* 拿不到前台应用时返回 null,不造一个空壳 —— 「不知道」和「没有焦点」是两件事。
|
|
111
|
+
*/
|
|
112
|
+
/**
|
|
113
|
+
* 这次观测的无障碍树**给不给得出控件级信息**(而不是「有没有一棵树」)。
|
|
114
|
+
*
|
|
115
|
+
* ⚠️ 这是「要不要退回像素判据」的唯一判据,此前在两处各自手写成
|
|
116
|
+
* `accessibility !== undefined && !truncated` —— 那测的是「树存在」,不是「树有用」。
|
|
117
|
+
* 微信的 AX 树有 5 个节点(纯窗口壳)、未截断,于是两处都判「可用」,
|
|
118
|
+
* 像素探测因此从未发生,整条效果反馈链路(生效判定、滚动边界、no-effect 回执、
|
|
119
|
+
* 像素级人工接管)全部失效。
|
|
120
|
+
*
|
|
121
|
+
* 正确的谓词就是 `deriveFocusSnapshot` 已经算好的 `level`:L2 才代表「说得出焦点
|
|
122
|
+
* 落在哪个控件、在哪」。判据只写这一遍,两个调用方不会再各自漂移。
|
|
123
|
+
*/
|
|
124
|
+
export function axGivesControlDetail(state) {
|
|
125
|
+
if (!state)
|
|
126
|
+
return false;
|
|
127
|
+
if (state.accessibility?.truncated === true)
|
|
128
|
+
return false;
|
|
129
|
+
return axFocusIsConcreteControl(state);
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* 焦点落在一个**具体控件**上(有 bounds、且不是纯容器)。
|
|
133
|
+
*
|
|
134
|
+
* 与 `axGivesControlDetail` 只差一个 `truncated` 条件,而这个差是有意的 ——
|
|
135
|
+
* 两个调用方问的不是同一个问题:
|
|
136
|
+
* · 「要不要退回像素探测」→ 树被截断就不能信 AX,`truncated` 必须一票否决
|
|
137
|
+
* · 「ROI 取哪块」 → 只要这个节点本身是真控件,它的矩形就是有效的,
|
|
138
|
+
* 别处截没截断与它无关
|
|
139
|
+
* 把两者硬绑成一个判据,会让「微信的树被截断、但焦点确实在输入框上」这种
|
|
140
|
+
* 常见形态丢掉本来拿得到的 ROI。
|
|
141
|
+
*/
|
|
142
|
+
export function axFocusIsConcreteControl(state) {
|
|
143
|
+
if (!state)
|
|
144
|
+
return false;
|
|
145
|
+
const snapshot = deriveFocusSnapshot(state);
|
|
146
|
+
if ((snapshot?.level ?? 0) < 2)
|
|
147
|
+
return false;
|
|
148
|
+
// ⚠️ **L2 还不够 —— 焦点节点必须是个「说得出事」的控件。**
|
|
149
|
+
//
|
|
150
|
+
// 实测:微信读书(iOS 套壳)报的是 `AXGroup · caret(51,98)` —— 一个位于窗口左上角的
|
|
151
|
+
// 无用包装容器。它满足「有 bounds 的焦点节点」= L2,于是这里判「AX 可用」→ 跳过像素
|
|
152
|
+
// 探测 → 按键之后**变化率恒为「未测」**。
|
|
153
|
+
//
|
|
154
|
+
// 后果不是少个数字:agent 拿不到「这一下有没有生效」的任何证据,只能干等、反复
|
|
155
|
+
// observe、最后去乱点碰运气。而实测证明按键**一直是好使的**(连按 3 次 right,
|
|
156
|
+
// 书从 15/106 翻到 21/106)—— 纯粹是反馈缺席造成的瞎折腾。
|
|
157
|
+
//
|
|
158
|
+
// 容器角色不构成「控件级信息」:它们回答不了「焦点在哪个可交互控件上」这个问题。
|
|
159
|
+
return !CONTAINER_ROLES.has(snapshot.role ?? '');
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* 纯容器角色 —— 有它们当焦点等于没有焦点信息。
|
|
163
|
+
*
|
|
164
|
+
* 自绘 / 套壳应用常把整个画布报成一个 AXGroup,看起来像 L2 实则零信息量。
|
|
165
|
+
* 这里只列**明确的容器**,不列未知角色:拿不准的角色宁可当作有信息(少做一次
|
|
166
|
+
* 像素探测的代价,远小于把一个真有 AX 的应用误判成瞎子)。
|
|
167
|
+
*/
|
|
168
|
+
const CONTAINER_ROLES = new Set([
|
|
169
|
+
'AXGroup',
|
|
170
|
+
'AXWindow',
|
|
171
|
+
'AXScrollArea',
|
|
172
|
+
'AXSplitGroup',
|
|
173
|
+
'AXUnknown',
|
|
174
|
+
'AXLayoutArea',
|
|
175
|
+
]);
|
|
176
|
+
export function deriveFocusSnapshot(state) {
|
|
177
|
+
const app = state.activeApp?.name ?? state.activeApp?.id;
|
|
178
|
+
if (!app)
|
|
179
|
+
return null;
|
|
180
|
+
const windows = state.windows ?? [];
|
|
181
|
+
const window = windows.find((candidate) => candidate.focused) ?? windows[0];
|
|
182
|
+
const node = findFocusedNode(state.accessibility?.root);
|
|
183
|
+
const snapshot = {
|
|
184
|
+
// 有控件级信息才敢报 L2。只有窗口壳(Electron/自绘应用的常态)就是 L1。
|
|
185
|
+
level: node?.bounds ? 2 : 1,
|
|
186
|
+
app,
|
|
187
|
+
...(state.activeApp?.id ? { appId: state.activeApp.id } : {}),
|
|
188
|
+
...(state.activeApp?.pid !== undefined ? { pid: String(state.activeApp.pid) } : {}),
|
|
189
|
+
...(window?.id ? { windowId: window.id } : {}),
|
|
190
|
+
...(window?.title ? { windowTitle: window.title } : {}),
|
|
191
|
+
...(window?.bounds ? { windowRect: window.bounds } : {}),
|
|
192
|
+
...(node?.role ? { role: node.role } : {}),
|
|
193
|
+
// AX 给的是控件矩形,不是插入点。当 caret 的近似值用,够 classifyFocusChange
|
|
194
|
+
// 判「焦点挪没挪」,但别当成真 caret —— 真 caret 这条通道拿不到(见头注)。
|
|
195
|
+
...(node?.bounds ? { caret: node.bounds } : {}),
|
|
196
|
+
};
|
|
197
|
+
return snapshot;
|
|
198
|
+
}
|
|
199
|
+
function findFocusedNode(node) {
|
|
200
|
+
if (!node)
|
|
201
|
+
return undefined;
|
|
202
|
+
if (node.focused === true)
|
|
203
|
+
return node;
|
|
204
|
+
for (const child of node.children ?? []) {
|
|
205
|
+
const hit = findFocusedNode(child);
|
|
206
|
+
if (hit)
|
|
207
|
+
return hit;
|
|
208
|
+
}
|
|
209
|
+
return undefined;
|
|
210
|
+
}
|
|
211
|
+
/**
|
|
212
|
+
* 快照是否就是期待的那个应用。
|
|
213
|
+
*
|
|
214
|
+
* 双向包含匹配,因为调用方手里的标识形态不定:`activate_app` 可能收到
|
|
215
|
+
* bundleId(`com.tencent.xinWeChat`)、进程名(`WeChat`)或显示名(`微信`),
|
|
216
|
+
* 而快照拿到的是另一种。宁可宽一点 —— 判错的方向是「把一次真实抢占当成
|
|
217
|
+
* 预期之内」,但那要求抢占者的名字恰好包含目标名,概率极低;反过来判严了
|
|
218
|
+
* 就是每次切应用都卡住,那是必然发生的。
|
|
219
|
+
*/
|
|
220
|
+
function matchesApp(snapshot, expected) {
|
|
221
|
+
const needle = expected.trim().toLowerCase();
|
|
222
|
+
if (!needle)
|
|
223
|
+
return false;
|
|
224
|
+
return [snapshot.appId, snapshot.app]
|
|
225
|
+
.filter((value) => typeof value === 'string' && value.length > 0)
|
|
226
|
+
.some((value) => {
|
|
227
|
+
const hay = value.toLowerCase();
|
|
228
|
+
return hay === needle || hay.includes(needle) || needle.includes(hay);
|
|
229
|
+
});
|
|
230
|
+
}
|
|
231
|
+
/** 窗口身份键。windowId 缺失时退回 pid+标题 —— 有的平台给不出稳定 id。 */
|
|
232
|
+
function keyOf(snapshot) {
|
|
233
|
+
return snapshot.windowId ?? `${snapshot.pid ?? ''}::${snapshot.windowTitle ?? ''}`;
|
|
234
|
+
}
|
|
235
|
+
function sameRect(a, b) {
|
|
236
|
+
if (!a && !b)
|
|
237
|
+
return true;
|
|
238
|
+
if (!a || !b)
|
|
239
|
+
return false;
|
|
240
|
+
return a.x === b.x && a.y === b.y && a.width === b.width && a.height === b.height;
|
|
241
|
+
}
|
|
242
|
+
/**
|
|
243
|
+
* 人话渲染,供文本通道输出。刻意做得**短** —— 这条通道的全部价值就是便宜,
|
|
244
|
+
* 写成一大段就失去了相对截图 20 倍的优势。
|
|
245
|
+
*/
|
|
246
|
+
export function describeFocus(snapshot) {
|
|
247
|
+
if (!snapshot)
|
|
248
|
+
return '焦点: 不可用';
|
|
249
|
+
const parts = [`前台: ${snapshot.app}`];
|
|
250
|
+
if (snapshot.windowTitle)
|
|
251
|
+
parts.push(`窗口「${snapshot.windowTitle}」`);
|
|
252
|
+
if (snapshot.role)
|
|
253
|
+
parts.push(`控件 ${snapshot.role}`);
|
|
254
|
+
if (snapshot.caret)
|
|
255
|
+
parts.push(`caret(${snapshot.caret.x},${snapshot.caret.y})`);
|
|
256
|
+
if (snapshot.composing)
|
|
257
|
+
parts.push('输入法合成中');
|
|
258
|
+
parts.push(`[L${snapshot.level}]`);
|
|
259
|
+
return parts.join(' · ');
|
|
260
|
+
}
|
|
261
|
+
export function describeFocusChange(before, after) {
|
|
262
|
+
if (!before || !after)
|
|
263
|
+
return '';
|
|
264
|
+
const changes = [];
|
|
265
|
+
if (before.app !== after.app) {
|
|
266
|
+
changes.push(`前台: ${before.app} → ${after.app}`);
|
|
267
|
+
}
|
|
268
|
+
const beforeTitle = before.windowTitle ?? '(无标题)';
|
|
269
|
+
const afterTitle = after.windowTitle ?? '(无标题)';
|
|
270
|
+
if (beforeTitle !== afterTitle) {
|
|
271
|
+
changes.push(`窗口标题: ${beforeTitle} → ${afterTitle}`);
|
|
272
|
+
}
|
|
273
|
+
const beforeRole = before.role ?? '(无焦点控件)';
|
|
274
|
+
const afterRole = after.role ?? '(无焦点控件)';
|
|
275
|
+
if (beforeRole !== afterRole) {
|
|
276
|
+
changes.push(`焦点控件: ${beforeRole} → ${afterRole}`);
|
|
277
|
+
}
|
|
278
|
+
if (changes.length === 0)
|
|
279
|
+
return '焦点状态未变化';
|
|
280
|
+
return `状态变化:${changes.join(';')}`;
|
|
281
|
+
}
|
|
282
|
+
/**
|
|
283
|
+
* 焦点是否稳定。
|
|
284
|
+
*
|
|
285
|
+
* ⚠️ 它**不是** settle 的判据之一 —— 那是 R1 之前的设计,已被推翻(见本文件
|
|
286
|
+
* 头注与 settle.ts)。保留它是给需要单独问「焦点动没动」的调用方用的。
|
|
287
|
+
*/
|
|
288
|
+
export function focusStable(before, after, expectation) {
|
|
289
|
+
const change = classifyFocusChange(before, after, expectation);
|
|
290
|
+
return change === FOCUS_CHANGE.NONE || change === FOCUS_CHANGE.EXPECTED;
|
|
291
|
+
}
|
|
@@ -0,0 +1,382 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ============================================================
|
|
3
|
+
* vision/focus.ts —— 焦点通道(纯函数)
|
|
4
|
+
* ------------------------------------------------------------
|
|
5
|
+
* 焦点是与像素并列的**辅助**通道(不是主判据,原因见下):
|
|
6
|
+
*
|
|
7
|
+
* ① L1 绕开了「应用不暴露 UI 结构」这个前提。
|
|
8
|
+
* 前台窗口/进程不归应用管、归 OS 管。微信可以拒绝暴露控件树,它拒绝不了
|
|
9
|
+
* 「我这个窗口是不是前台窗口」。**跨进程打断因此永远可检出**。
|
|
10
|
+
*
|
|
11
|
+
* ② 它比截图便宜 20 倍。
|
|
12
|
+
* 一次焦点采样约 50 token,一张全屏截图约 1100 token。
|
|
13
|
+
*
|
|
14
|
+
* ★ 一条已被推翻的旧论断,留在这里免得有人再推一遍:
|
|
15
|
+
*
|
|
16
|
+
* 曾经的说法是「输入法必须知道 caret 矩形才能画候选词框,所以只要支持中文
|
|
17
|
+
* 输入,自绘应用也堵不上 caret 泄漏」。**这是错的。** IME 通道(TSF/IMM、
|
|
18
|
+
* TSM)与无障碍通道(GetGUIThreadInfo、AX)是两条独立路径,前者不向第三方
|
|
19
|
+
* 进程暴露。已查证:
|
|
20
|
+
* - Windows:`GetGUIThreadInfo.hwndCaret` 对 Chromium/Electron 恒为 NULL
|
|
21
|
+
* - macOS:`AXSelectedTextRange` 在 MAS 版 Electron 被整体 patch 掉
|
|
22
|
+
* 而 Electron / 自绘控件正是目标场景。本仓库实测也确认:Claude 桌面端
|
|
23
|
+
* 只暴露窗口与三个窗口按钮,内容对 AX 完全不可见。
|
|
24
|
+
*
|
|
25
|
+
* **因此 L2 不可假定可用**,settle 的主判据必须是像素(见 settle.ts),
|
|
26
|
+
* 焦点只做增强与跨进程打断检测。
|
|
27
|
+
* ============================================================
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
import type { Rect } from './raster.js';
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* 归一化后的焦点快照。各平台原生实现须产出这个形状。
|
|
34
|
+
*
|
|
35
|
+
* `level` 是**实际拿到的层级**,必须如实报告,不许假装。
|
|
36
|
+
*/
|
|
37
|
+
export interface FocusSnapshot {
|
|
38
|
+
/** 1 = 前台窗口/进程;2 = 焦点控件 + caret;3 = 输入法合成状态 */
|
|
39
|
+
level: 1 | 2 | 3;
|
|
40
|
+
/** 前台应用名(L1) */
|
|
41
|
+
app: string;
|
|
42
|
+
/** 前台应用的稳定标识(macOS bundleId / Windows 可执行名)。显示名会随系统语言变。 */
|
|
43
|
+
appId?: string;
|
|
44
|
+
pid?: string;
|
|
45
|
+
windowId?: string;
|
|
46
|
+
windowTitle?: string;
|
|
47
|
+
/** 逻辑点 */
|
|
48
|
+
windowRect?: Rect;
|
|
49
|
+
/** 焦点控件角色,如 AXTextField(L2) */
|
|
50
|
+
role?: string;
|
|
51
|
+
/** 插入点矩形,逻辑点(L2) */
|
|
52
|
+
caret?: Rect;
|
|
53
|
+
/** 输入法正在合成(L3) */
|
|
54
|
+
composing?: boolean;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** 层级语义。原生侧拿不到高层就降级,但必须**如实报告拿到了哪层**。 */
|
|
58
|
+
export const FOCUS_LEVELS: Record<number, string> = {
|
|
59
|
+
1: '前台窗口/进程(总是可用)',
|
|
60
|
+
2: '焦点控件 + caret 矩形(自绘/Electron 应用上常常拿不到)',
|
|
61
|
+
3: '输入法合成状态(看应用)',
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* 焦点变化的三种语义。这是**引擎级通用对策**,不是任何 app 的特判 ——
|
|
66
|
+
* `other-process` 只用 L1 就能算,所以对自绘应用同样成立。
|
|
67
|
+
*
|
|
68
|
+
* same-window → app 自己的交互反馈(弹联想、展开面板、切 tab);需 L2
|
|
69
|
+
* other-process → 外部打断(通知、别的 app 抢前台);L1 即可
|
|
70
|
+
* expected → 就是我刚点的目标,正常
|
|
71
|
+
* none → 没变
|
|
72
|
+
*/
|
|
73
|
+
export const FOCUS_CHANGE = {
|
|
74
|
+
NONE: 'none',
|
|
75
|
+
EXPECTED: 'expected',
|
|
76
|
+
SAME_WINDOW: 'same-window',
|
|
77
|
+
OTHER_PROCESS: 'other-process',
|
|
78
|
+
} as const;
|
|
79
|
+
|
|
80
|
+
export type FocusChange = (typeof FOCUS_CHANGE)[keyof typeof FOCUS_CHANGE];
|
|
81
|
+
|
|
82
|
+
export interface FocusExpectation {
|
|
83
|
+
/** 动作发起方对「焦点应该落到哪」的预期。 */
|
|
84
|
+
expectWindowId?: string;
|
|
85
|
+
expectRole?: string;
|
|
86
|
+
/**
|
|
87
|
+
* 预期落到的应用(bundleId、进程名或显示名皆可,做包含匹配)。
|
|
88
|
+
*
|
|
89
|
+
* 没有这一项时,agent **自己**执行 `activate_app` 切过去的应用会在下一次
|
|
90
|
+
* 观测里被判成「别的进程抢了前台」,进而触发人工介入 —— 一次正常的、
|
|
91
|
+
* 由 agent 主动发起的切换变成了需要人来处理的事故。这不是理论问题:
|
|
92
|
+
* 只要工作流里有一步切应用,接下来就必然卡住。
|
|
93
|
+
*/
|
|
94
|
+
expectAppId?: string;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** 比较两个焦点快照,判定变化语义。 */
|
|
98
|
+
export function classifyFocusChange(
|
|
99
|
+
before: FocusSnapshot | null | undefined,
|
|
100
|
+
after: FocusSnapshot | null | undefined,
|
|
101
|
+
expectation: FocusExpectation = {},
|
|
102
|
+
): FocusChange {
|
|
103
|
+
if (!before || !after) return FOCUS_CHANGE.NONE;
|
|
104
|
+
|
|
105
|
+
// 进程换了就是换了 —— 这一项**必须**独立参与判定。
|
|
106
|
+
//
|
|
107
|
+
// 曾经的写法只看窗口键/caret/role,于是当两个应用的窗口键恰好相同(或两边
|
|
108
|
+
// 都拿不到窗口 id、标题也一样)时,一次真实的跨进程抢前台会被判成 `none`。
|
|
109
|
+
// 而 L1 的全部价值就在于「进程身份是最可靠的那一层」,让它取决于更弱的
|
|
110
|
+
// 窗口信息是把依赖关系倒过来了。
|
|
111
|
+
const processChanged = (before.pid ?? '') !== (after.pid ?? '')
|
|
112
|
+
|| before.app !== after.app;
|
|
113
|
+
const windowChanged = keyOf(before) !== keyOf(after);
|
|
114
|
+
const caretMoved = !sameRect(before.caret, after.caret);
|
|
115
|
+
const roleChanged = (before.role ?? '') !== (after.role ?? '');
|
|
116
|
+
|
|
117
|
+
if (!processChanged && !windowChanged && !caretMoved && !roleChanged) {
|
|
118
|
+
return FOCUS_CHANGE.NONE;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// 预期之内:这次动作本来就是要把焦点挪过去
|
|
122
|
+
if (
|
|
123
|
+
expectation.expectWindowId &&
|
|
124
|
+
after.windowId === expectation.expectWindowId &&
|
|
125
|
+
(!expectation.expectRole || after.role === expectation.expectRole)
|
|
126
|
+
) {
|
|
127
|
+
return FOCUS_CHANGE.EXPECTED;
|
|
128
|
+
}
|
|
129
|
+
if (expectation.expectAppId && matchesApp(after, expectation.expectAppId)) {
|
|
130
|
+
return FOCUS_CHANGE.EXPECTED;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
if (processChanged) return FOCUS_CHANGE.OTHER_PROCESS;
|
|
134
|
+
return FOCUS_CHANGE.SAME_WINDOW;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
export interface FocusAdvice {
|
|
138
|
+
abortTyping: boolean;
|
|
139
|
+
advice: string;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* 变化语义 → 建议动作。写死在引擎里当默认,`apps/<app>.md` 可以描述例外
|
|
144
|
+
* (但那是**事实**描述,覆盖策略由模型读了 md 之后自己决定,不是这里加分支)。
|
|
145
|
+
*/
|
|
146
|
+
export function recommendedAction(change: FocusChange): FocusAdvice {
|
|
147
|
+
switch (change) {
|
|
148
|
+
case FOCUS_CHANGE.SAME_WINDOW:
|
|
149
|
+
return { abortTyping: true, advice: '应用弹出了新界面元素,先观察再继续,不要盲目输入' };
|
|
150
|
+
case FOCUS_CHANGE.OTHER_PROCESS:
|
|
151
|
+
return { abortTyping: true, advice: '被其它进程抢走焦点,需先抢回前台再重试' };
|
|
152
|
+
case FOCUS_CHANGE.EXPECTED:
|
|
153
|
+
return { abortTyping: false, advice: '焦点如期落位' };
|
|
154
|
+
default:
|
|
155
|
+
return { abortTyping: false, advice: '焦点未变' };
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* 从一次观测里派生焦点快照。
|
|
161
|
+
*
|
|
162
|
+
* **零协议改动**:协议里没有专门的焦点快照字段,但需要的两件事都已经在观测里了 ——
|
|
163
|
+
* `activeApp`/`windows` 给 L1,`AccessibilityNode.focused`(原生侧从 `AXFocused` 采集)
|
|
164
|
+
* 给 L2。所以这里是纯派生,不需要多跑一次原生调用。
|
|
165
|
+
*
|
|
166
|
+
* `level` 必须如实:
|
|
167
|
+
* - 找到带 bounds 的焦点控件 → **L2**(能说出焦点落在哪个控件、在哪)
|
|
168
|
+
* - 只有前台应用/窗口 → **L1**
|
|
169
|
+
* 刻意不产出 L3:输入法合成状态在本项目里根本没有采集通道,报 L3 就是撒谎
|
|
170
|
+
* (本文件头注记录了「IME 能拿到 caret」这个已被推翻的论断,别再往回走)。
|
|
171
|
+
*
|
|
172
|
+
* 拿不到前台应用时返回 null,不造一个空壳 —— 「不知道」和「没有焦点」是两件事。
|
|
173
|
+
*/
|
|
174
|
+
/**
|
|
175
|
+
* 这次观测的无障碍树**给不给得出控件级信息**(而不是「有没有一棵树」)。
|
|
176
|
+
*
|
|
177
|
+
* ⚠️ 这是「要不要退回像素判据」的唯一判据,此前在两处各自手写成
|
|
178
|
+
* `accessibility !== undefined && !truncated` —— 那测的是「树存在」,不是「树有用」。
|
|
179
|
+
* 微信的 AX 树有 5 个节点(纯窗口壳)、未截断,于是两处都判「可用」,
|
|
180
|
+
* 像素探测因此从未发生,整条效果反馈链路(生效判定、滚动边界、no-effect 回执、
|
|
181
|
+
* 像素级人工接管)全部失效。
|
|
182
|
+
*
|
|
183
|
+
* 正确的谓词就是 `deriveFocusSnapshot` 已经算好的 `level`:L2 才代表「说得出焦点
|
|
184
|
+
* 落在哪个控件、在哪」。判据只写这一遍,两个调用方不会再各自漂移。
|
|
185
|
+
*/
|
|
186
|
+
export function axGivesControlDetail(
|
|
187
|
+
state:
|
|
188
|
+
| {
|
|
189
|
+
activeApp?: { id?: string; name?: string; pid?: number };
|
|
190
|
+
windows?: { id: string; title?: string; bounds?: Rect; focused?: boolean }[];
|
|
191
|
+
accessibility?: { root?: FocusableNode; truncated?: boolean };
|
|
192
|
+
}
|
|
193
|
+
| undefined,
|
|
194
|
+
): boolean {
|
|
195
|
+
if (!state) return false;
|
|
196
|
+
if (state.accessibility?.truncated === true) return false;
|
|
197
|
+
return axFocusIsConcreteControl(state);
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* 焦点落在一个**具体控件**上(有 bounds、且不是纯容器)。
|
|
202
|
+
*
|
|
203
|
+
* 与 `axGivesControlDetail` 只差一个 `truncated` 条件,而这个差是有意的 ——
|
|
204
|
+
* 两个调用方问的不是同一个问题:
|
|
205
|
+
* · 「要不要退回像素探测」→ 树被截断就不能信 AX,`truncated` 必须一票否决
|
|
206
|
+
* · 「ROI 取哪块」 → 只要这个节点本身是真控件,它的矩形就是有效的,
|
|
207
|
+
* 别处截没截断与它无关
|
|
208
|
+
* 把两者硬绑成一个判据,会让「微信的树被截断、但焦点确实在输入框上」这种
|
|
209
|
+
* 常见形态丢掉本来拿得到的 ROI。
|
|
210
|
+
*/
|
|
211
|
+
export function axFocusIsConcreteControl(
|
|
212
|
+
state:
|
|
213
|
+
| {
|
|
214
|
+
activeApp?: { id?: string; name?: string; pid?: number };
|
|
215
|
+
windows?: { id: string; title?: string; bounds?: Rect; focused?: boolean }[];
|
|
216
|
+
accessibility?: { root?: FocusableNode; truncated?: boolean };
|
|
217
|
+
}
|
|
218
|
+
| undefined,
|
|
219
|
+
): boolean {
|
|
220
|
+
if (!state) return false;
|
|
221
|
+
const snapshot = deriveFocusSnapshot(state);
|
|
222
|
+
if ((snapshot?.level ?? 0) < 2) return false;
|
|
223
|
+
// ⚠️ **L2 还不够 —— 焦点节点必须是个「说得出事」的控件。**
|
|
224
|
+
//
|
|
225
|
+
// 实测:微信读书(iOS 套壳)报的是 `AXGroup · caret(51,98)` —— 一个位于窗口左上角的
|
|
226
|
+
// 无用包装容器。它满足「有 bounds 的焦点节点」= L2,于是这里判「AX 可用」→ 跳过像素
|
|
227
|
+
// 探测 → 按键之后**变化率恒为「未测」**。
|
|
228
|
+
//
|
|
229
|
+
// 后果不是少个数字:agent 拿不到「这一下有没有生效」的任何证据,只能干等、反复
|
|
230
|
+
// observe、最后去乱点碰运气。而实测证明按键**一直是好使的**(连按 3 次 right,
|
|
231
|
+
// 书从 15/106 翻到 21/106)—— 纯粹是反馈缺席造成的瞎折腾。
|
|
232
|
+
//
|
|
233
|
+
// 容器角色不构成「控件级信息」:它们回答不了「焦点在哪个可交互控件上」这个问题。
|
|
234
|
+
return !CONTAINER_ROLES.has(snapshot!.role ?? '');
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* 纯容器角色 —— 有它们当焦点等于没有焦点信息。
|
|
239
|
+
*
|
|
240
|
+
* 自绘 / 套壳应用常把整个画布报成一个 AXGroup,看起来像 L2 实则零信息量。
|
|
241
|
+
* 这里只列**明确的容器**,不列未知角色:拿不准的角色宁可当作有信息(少做一次
|
|
242
|
+
* 像素探测的代价,远小于把一个真有 AX 的应用误判成瞎子)。
|
|
243
|
+
*/
|
|
244
|
+
const CONTAINER_ROLES = new Set([
|
|
245
|
+
'AXGroup',
|
|
246
|
+
'AXWindow',
|
|
247
|
+
'AXScrollArea',
|
|
248
|
+
'AXSplitGroup',
|
|
249
|
+
'AXUnknown',
|
|
250
|
+
'AXLayoutArea',
|
|
251
|
+
]);
|
|
252
|
+
|
|
253
|
+
export function deriveFocusSnapshot(state: {
|
|
254
|
+
activeApp?: { id?: string; name?: string; pid?: number };
|
|
255
|
+
windows?: { id: string; title?: string; bounds?: Rect; focused?: boolean }[];
|
|
256
|
+
accessibility?: { root?: FocusableNode };
|
|
257
|
+
}): FocusSnapshot | null {
|
|
258
|
+
const app = state.activeApp?.name ?? state.activeApp?.id;
|
|
259
|
+
if (!app) return null;
|
|
260
|
+
|
|
261
|
+
const windows = state.windows ?? [];
|
|
262
|
+
const window = windows.find((candidate) => candidate.focused) ?? windows[0];
|
|
263
|
+
const node = findFocusedNode(state.accessibility?.root);
|
|
264
|
+
|
|
265
|
+
const snapshot: FocusSnapshot = {
|
|
266
|
+
// 有控件级信息才敢报 L2。只有窗口壳(Electron/自绘应用的常态)就是 L1。
|
|
267
|
+
level: node?.bounds ? 2 : 1,
|
|
268
|
+
app,
|
|
269
|
+
...(state.activeApp?.id ? { appId: state.activeApp.id } : {}),
|
|
270
|
+
...(state.activeApp?.pid !== undefined ? { pid: String(state.activeApp.pid) } : {}),
|
|
271
|
+
...(window?.id ? { windowId: window.id } : {}),
|
|
272
|
+
...(window?.title ? { windowTitle: window.title } : {}),
|
|
273
|
+
...(window?.bounds ? { windowRect: window.bounds } : {}),
|
|
274
|
+
...(node?.role ? { role: node.role } : {}),
|
|
275
|
+
// AX 给的是控件矩形,不是插入点。当 caret 的近似值用,够 classifyFocusChange
|
|
276
|
+
// 判「焦点挪没挪」,但别当成真 caret —— 真 caret 这条通道拿不到(见头注)。
|
|
277
|
+
...(node?.bounds ? { caret: node.bounds } : {}),
|
|
278
|
+
};
|
|
279
|
+
return snapshot;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
/** 只关心 focused/role/bounds/children 四件事,不绑具体协议类型。 */
|
|
283
|
+
export interface FocusableNode {
|
|
284
|
+
role?: string;
|
|
285
|
+
focused?: boolean;
|
|
286
|
+
bounds?: Rect;
|
|
287
|
+
children?: FocusableNode[];
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
function findFocusedNode(node: FocusableNode | undefined): FocusableNode | undefined {
|
|
291
|
+
if (!node) return undefined;
|
|
292
|
+
if (node.focused === true) return node;
|
|
293
|
+
for (const child of node.children ?? []) {
|
|
294
|
+
const hit = findFocusedNode(child);
|
|
295
|
+
if (hit) return hit;
|
|
296
|
+
}
|
|
297
|
+
return undefined;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/**
|
|
301
|
+
* 快照是否就是期待的那个应用。
|
|
302
|
+
*
|
|
303
|
+
* 双向包含匹配,因为调用方手里的标识形态不定:`activate_app` 可能收到
|
|
304
|
+
* bundleId(`com.tencent.xinWeChat`)、进程名(`WeChat`)或显示名(`微信`),
|
|
305
|
+
* 而快照拿到的是另一种。宁可宽一点 —— 判错的方向是「把一次真实抢占当成
|
|
306
|
+
* 预期之内」,但那要求抢占者的名字恰好包含目标名,概率极低;反过来判严了
|
|
307
|
+
* 就是每次切应用都卡住,那是必然发生的。
|
|
308
|
+
*/
|
|
309
|
+
function matchesApp(snapshot: FocusSnapshot, expected: string): boolean {
|
|
310
|
+
const needle = expected.trim().toLowerCase();
|
|
311
|
+
if (!needle) return false;
|
|
312
|
+
return [snapshot.appId, snapshot.app]
|
|
313
|
+
.filter((value): value is string => typeof value === 'string' && value.length > 0)
|
|
314
|
+
.some((value) => {
|
|
315
|
+
const hay = value.toLowerCase();
|
|
316
|
+
return hay === needle || hay.includes(needle) || needle.includes(hay);
|
|
317
|
+
});
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
/** 窗口身份键。windowId 缺失时退回 pid+标题 —— 有的平台给不出稳定 id。 */
|
|
321
|
+
function keyOf(snapshot: FocusSnapshot): string {
|
|
322
|
+
return snapshot.windowId ?? `${snapshot.pid ?? ''}::${snapshot.windowTitle ?? ''}`;
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
function sameRect(a?: Rect, b?: Rect): boolean {
|
|
326
|
+
if (!a && !b) return true;
|
|
327
|
+
if (!a || !b) return false;
|
|
328
|
+
return a.x === b.x && a.y === b.y && a.width === b.width && a.height === b.height;
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/**
|
|
332
|
+
* 人话渲染,供文本通道输出。刻意做得**短** —— 这条通道的全部价值就是便宜,
|
|
333
|
+
* 写成一大段就失去了相对截图 20 倍的优势。
|
|
334
|
+
*/
|
|
335
|
+
export function describeFocus(snapshot: FocusSnapshot | null | undefined): string {
|
|
336
|
+
if (!snapshot) return '焦点: 不可用';
|
|
337
|
+
const parts = [`前台: ${snapshot.app}`];
|
|
338
|
+
if (snapshot.windowTitle) parts.push(`窗口「${snapshot.windowTitle}」`);
|
|
339
|
+
if (snapshot.role) parts.push(`控件 ${snapshot.role}`);
|
|
340
|
+
if (snapshot.caret) parts.push(`caret(${snapshot.caret.x},${snapshot.caret.y})`);
|
|
341
|
+
if (snapshot.composing) parts.push('输入法合成中');
|
|
342
|
+
parts.push(`[L${snapshot.level}]`);
|
|
343
|
+
return parts.join(' · ');
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
export function describeFocusChange(
|
|
347
|
+
before: FocusSnapshot | null | undefined,
|
|
348
|
+
after: FocusSnapshot | null | undefined,
|
|
349
|
+
): string {
|
|
350
|
+
if (!before || !after) return '';
|
|
351
|
+
const changes: string[] = [];
|
|
352
|
+
if (before.app !== after.app) {
|
|
353
|
+
changes.push(`前台: ${before.app} → ${after.app}`);
|
|
354
|
+
}
|
|
355
|
+
const beforeTitle = before.windowTitle ?? '(无标题)';
|
|
356
|
+
const afterTitle = after.windowTitle ?? '(无标题)';
|
|
357
|
+
if (beforeTitle !== afterTitle) {
|
|
358
|
+
changes.push(`窗口标题: ${beforeTitle} → ${afterTitle}`);
|
|
359
|
+
}
|
|
360
|
+
const beforeRole = before.role ?? '(无焦点控件)';
|
|
361
|
+
const afterRole = after.role ?? '(无焦点控件)';
|
|
362
|
+
if (beforeRole !== afterRole) {
|
|
363
|
+
changes.push(`焦点控件: ${beforeRole} → ${afterRole}`);
|
|
364
|
+
}
|
|
365
|
+
if (changes.length === 0) return '焦点状态未变化';
|
|
366
|
+
return `状态变化:${changes.join(';')}`;
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
/**
|
|
370
|
+
* 焦点是否稳定。
|
|
371
|
+
*
|
|
372
|
+
* ⚠️ 它**不是** settle 的判据之一 —— 那是 R1 之前的设计,已被推翻(见本文件
|
|
373
|
+
* 头注与 settle.ts)。保留它是给需要单独问「焦点动没动」的调用方用的。
|
|
374
|
+
*/
|
|
375
|
+
export function focusStable(
|
|
376
|
+
before: FocusSnapshot | null | undefined,
|
|
377
|
+
after: FocusSnapshot | null | undefined,
|
|
378
|
+
expectation?: FocusExpectation,
|
|
379
|
+
): boolean {
|
|
380
|
+
const change = classifyFocusChange(before, after, expectation);
|
|
381
|
+
return change === FOCUS_CHANGE.NONE || change === FOCUS_CHANGE.EXPECTED;
|
|
382
|
+
}
|