@teevee98/synapse-hub 0.0.0-stage → 3.8.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (369) hide show
  1. package/.agents/agents/api-engineer.md +67 -0
  2. package/.agents/agents/failure-analyst.md +75 -0
  3. package/.agents/agents/live-explorer.md +70 -0
  4. package/.agents/agents/orchestrator.md +104 -0
  5. package/.agents/agents/perf-engineer.md +74 -0
  6. package/.agents/agents/pom-crafter.md +112 -0
  7. package/.agents/agents/quality-gatekeeper.md +87 -0
  8. package/.agents/agents/test-designer.md +73 -0
  9. package/.agents/agents/test-runner.md +67 -0
  10. package/.agents/rules/author-integrity-guard.md +33 -0
  11. package/.agents/rules/communication-style.md +18 -0
  12. package/.agents/rules/context-engineering.md +37 -0
  13. package/.agents/rules/git-push-protocol.md +33 -0
  14. package/.agents/rules/graphify.md +14 -0
  15. package/.agents/rules/lessons-learned.md +25 -0
  16. package/.agents/rules/mandatory-playwright-live.md +25 -0
  17. package/.agents/rules/mobile_appium_rules.md +43 -0
  18. package/.agents/rules/performance_testing_rules.md +35 -0
  19. package/.agents/rules/skill-precedence.md +29 -0
  20. package/.agents/rules/systematic-debugging.md +36 -0
  21. package/.agents/skills/actor-qa-swarm/SKILL.md +77 -0
  22. package/.agents/skills/actor-qa-swarm/examples/broker_sharding_sample.json +12 -0
  23. package/.agents/skills/agent-orchestration-multi-agent-optimize/SKILL.md +88 -0
  24. package/.agents/skills/agent-orchestration-multi-agent-optimize/examples/swarm_workload_config.json +15 -0
  25. package/.agents/skills/agentless-qa-repair/SKILL.md +69 -0
  26. package/.agents/skills/agentless-qa-repair/examples/healing_report_sample.json +17 -0
  27. package/.agents/skills/agentless-qa-repair/examples/three_phase_repair_workflow.md +40 -0
  28. package/.agents/skills/antigravity-test-sentinel/SKILL.md +63 -0
  29. package/.agents/skills/antigravity-test-sentinel/examples/dod_gate_policy.json +11 -0
  30. package/.agents/skills/antigravity-test-sentinel/examples/sprint_dod_enforcement.md +47 -0
  31. package/.agents/skills/api-network-mocking-interception/SKILL.md +142 -0
  32. package/.agents/skills/api-network-mocking-interception/examples/network_route_stub.json +9 -0
  33. package/.agents/skills/api-spec-to-e2e-matrix/SKILL.md +80 -0
  34. package/.agents/skills/api-spec-to-e2e-matrix/examples/sample_openapi.json +19 -0
  35. package/.agents/skills/architecture-decision-records/SKILL.md +64 -0
  36. package/.agents/skills/architecture-decision-records/examples/0001-record-architecture-decisions.md +12 -0
  37. package/.agents/skills/auth-session-and-cookie-manager/SKILL.md +75 -0
  38. package/.agents/skills/auth-session-and-cookie-manager/examples/multi_role_auth_matrix.md +70 -0
  39. package/.agents/skills/auth-session-and-cookie-manager/examples/storage_state_sample.json +11 -0
  40. package/.agents/skills/backend-load-and-stress-patterns/SKILL.md +85 -0
  41. package/.agents/skills/backend-load-and-stress-patterns/examples/k6_load_test_sample.js +13 -0
  42. package/.agents/skills/backend-load-and-stress-patterns/references/k6_load_template.js +82 -0
  43. package/.agents/skills/bats-testing-patterns/SKILL.md +100 -0
  44. package/.agents/skills/bats-testing-patterns/examples/cli_runner_test.bats +6 -0
  45. package/.agents/skills/bats-testing-patterns/resources/implementation-playbook.md +614 -0
  46. package/.agents/skills/clean-pom-crafter/SKILL.md +110 -0
  47. package/.agents/skills/clean-pom-crafter/examples/standard_test_case.groovy +28 -0
  48. package/.agents/skills/clean-pom-crafter/references/pom_patterns.groovy +138 -0
  49. package/.agents/skills/code-refactoring-refactor-clean/SKILL.md +64 -0
  50. package/.agents/skills/code-refactoring-refactor-clean/examples/refactored_page_sample.groovy +9 -0
  51. package/.agents/skills/code-refactoring-refactor-clean/examples/refactored_playwright_pom.ts +52 -0
  52. package/.agents/skills/code-refactoring-refactor-clean/resources/implementation-playbook.md +881 -0
  53. package/.agents/skills/consumer-driven-contract-testing/SKILL.md +147 -0
  54. package/.agents/skills/consumer-driven-contract-testing/examples/order_service_pact.json +45 -0
  55. package/.agents/skills/db-state-verifier-and-seeder/SKILL.md +123 -0
  56. package/.agents/skills/db-state-verifier-and-seeder/examples/db_fixture_seed.json +12 -0
  57. package/.agents/skills/e2e-testing-patterns/SKILL.md +103 -0
  58. package/.agents/skills/e2e-testing-patterns/examples/playwright_sharded_config.ts +12 -0
  59. package/.agents/skills/e2e-testing-patterns/resources/implementation-playbook.md +522 -0
  60. package/.agents/skills/event-stream-qa-orchestrator/SKILL.md +79 -0
  61. package/.agents/skills/event-stream-qa-orchestrator/examples/timeline_sample.jsonl +2 -0
  62. package/.agents/skills/failure-autopsy-and-healing/SKILL.md +66 -0
  63. package/.agents/skills/failure-autopsy-and-healing/examples/closed_loop_healing_walkthrough.md +48 -0
  64. package/.agents/skills/failure-autopsy-and-healing/examples/sample_junit_failure.xml +8 -0
  65. package/.agents/skills/failure-autopsy-and-healing/references/healing_matrix.md +64 -0
  66. package/.agents/skills/failure-autopsy-and-healing/references/qa_investigation_protocol.md +56 -0
  67. package/.agents/skills/failure-autopsy-and-healing/scripts/triage_failures.sh +83 -0
  68. package/.agents/skills/file-and-report-validator/SKILL.md +90 -0
  69. package/.agents/skills/file-and-report-validator/examples/sample_export_schema.json +14 -0
  70. package/.agents/skills/flaky-test-sentinel-and-quarantine/SKILL.md +114 -0
  71. package/.agents/skills/flaky-test-sentinel-and-quarantine/examples/quarantine_registry.json +11 -0
  72. package/.agents/skills/gherkin-to-suite/SKILL.md +103 -0
  73. package/.agents/skills/gherkin-to-suite/examples/checkout.feature +39 -0
  74. package/.agents/skills/gherkin-to-suite/examples/suite_report_sample.json +334 -0
  75. package/.agents/skills/github-actions-templates/SKILL.md +77 -0
  76. package/.agents/skills/github-actions-templates/examples/matrix_test_pipeline.yml +8 -0
  77. package/.agents/skills/graphify-impact-analyzer/SKILL.md +76 -0
  78. package/.agents/skills/graphify-impact-analyzer/examples/dependency_blast_radius.json +8 -0
  79. package/.agents/skills/katalon-cli-runner/SKILL.md +76 -0
  80. package/.agents/skills/katalon-cli-runner/examples/headless_kre_exec.sh +3 -0
  81. package/.agents/skills/katalon-cli-runner/references/groovy-patterns.md +209 -0
  82. package/.agents/skills/katalon-cli-runner/references/katalon-api.md +155 -0
  83. package/.agents/skills/katalon-cli-runner/references/runner_commands.md +67 -0
  84. package/.agents/skills/lam-script-generator/SKILL.md +123 -0
  85. package/.agents/skills/lam-script-generator/examples/user_story_sample.md +8 -0
  86. package/.agents/skills/lean-code-ladder/SKILL.md +94 -0
  87. package/.agents/skills/lean-code-ladder/examples/ladder_walkthrough.md +37 -0
  88. package/.agents/skills/lean-impact-scoreboard/SKILL.md +48 -0
  89. package/.agents/skills/lean-impact-scoreboard/examples/sample_benchmark_gain.md +20 -0
  90. package/.agents/skills/lean-impact-scoreboard/examples/sample_scoreboard.md +15 -0
  91. package/.agents/skills/meta-prompt-crafter/SKILL.md +107 -0
  92. package/.agents/skills/meta-prompt-crafter/examples/qa_prompt_template.md +35 -0
  93. package/.agents/skills/mipro-prompt-compiler/SKILL.md +59 -0
  94. package/.agents/skills/mipro-prompt-compiler/examples/compiled_module_sample.json +22 -0
  95. package/.agents/skills/mipro-prompt-compiler/examples/dspy_assertion_optimizer_case.md +46 -0
  96. package/.agents/skills/mobile-appium-som/SKILL.md +88 -0
  97. package/.agents/skills/mobile-appium-som/examples/BaseScreen.java +88 -0
  98. package/.agents/skills/mobile-appium-som/examples/LoginScreen.java +46 -0
  99. package/.agents/skills/mobile-appium-som/examples/TC_MobileLogin.java +43 -0
  100. package/.agents/skills/persistent-planning-files/SKILL.md +108 -0
  101. package/.agents/skills/persistent-planning-files/examples/task_plan_example.md +56 -0
  102. package/.agents/skills/playwright-live-interaction/SKILL.md +131 -0
  103. package/.agents/skills/playwright-live-interaction/examples/live_probe_script.js +5 -0
  104. package/.agents/skills/playwright-live-interaction/references/advanced_superpowers.md +160 -0
  105. package/.agents/skills/playwright-live-interaction/references/compact_accessibility_snapshots.md +75 -0
  106. package/.agents/skills/playwright-live-interaction/references/recipes.md +388 -0
  107. package/.agents/skills/playwright-live-interaction/references/swarm_orchestration.md +134 -0
  108. package/.agents/skills/playwright-webapp-testing/SKILL.md +93 -0
  109. package/.agents/skills/playwright-webapp-testing/examples/server_lifecycle_playwright.py +63 -0
  110. package/.agents/skills/qa-aci-interface/SKILL.md +67 -0
  111. package/.agents/skills/qa-aci-interface/examples/surgical_code_window_walkthrough.md +40 -0
  112. package/.agents/skills/qa-aci-interface/examples/surgical_edit_sample.json +12 -0
  113. package/.agents/skills/qa-memory-os/SKILL.md +80 -0
  114. package/.agents/skills/qa-memory-os/examples/archival_memory_sample.json +21 -0
  115. package/.agents/skills/rbt-manual-testing/SKILL.md +137 -0
  116. package/.agents/skills/rbt-manual-testing/examples/checkout_rbt_manual_suite.md +25 -0
  117. package/.agents/skills/rbt-manual-testing/references/field_validation_checklist.md +29 -0
  118. package/.agents/skills/repo-quality-audit/SKILL.md +68 -0
  119. package/.agents/skills/repo-quality-audit/examples/sample_audit_walkthrough.md +15 -0
  120. package/.agents/skills/repo-quality-audit/examples/sample_repo_audit.md +33 -0
  121. package/.agents/skills/review-spec/SKILL.md +60 -0
  122. package/.agents/skills/review-spec/examples/pocock_grilling_decision_rubric.md +40 -0
  123. package/.agents/skills/review-spec/examples/sample_spec_review.md +33 -0
  124. package/.agents/skills/security-dast-sentinel/SKILL.md +119 -0
  125. package/.agents/skills/security-dast-sentinel/examples/dast_scan_config.json +10 -0
  126. package/.agents/skills/security-dast-sentinel/references/AI-AND-LLM.md +113 -0
  127. package/.agents/skills/security-dast-sentinel/references/ATTACK-CLASSES.md +89 -0
  128. package/.agents/skills/security-dast-sentinel/references/dast_recipes.ts +33 -0
  129. package/.agents/skills/self-healing-locator/SKILL.md +86 -0
  130. package/.agents/skills/self-healing-locator/examples/aom_diff_sample.json +16 -0
  131. package/.agents/skills/senior-code-review/SKILL.md +108 -0
  132. package/.agents/skills/senior-code-review/examples/sample_review_walkthrough.md +29 -0
  133. package/.agents/skills/skillspector-security-guard/SKILL.md +57 -0
  134. package/.agents/skills/skillspector-security-guard/examples/audit_threat_vectors_report.json +35 -0
  135. package/.agents/skills/skillspector-security-guard/examples/safe_mcp_config.json +12 -0
  136. package/.agents/skills/surgical-coding-principles/SKILL.md +89 -0
  137. package/.agents/skills/surgical-coding-principles/examples/surgical_vs_scope_creep.md +46 -0
  138. package/.agents/skills/synthetic-test-data-generator/SKILL.md +135 -0
  139. package/.agents/skills/synthetic-test-data-generator/examples/idempotent_data_recipe.json +9 -0
  140. package/.agents/skills/systematic-debugging/SKILL.md +100 -0
  141. package/.agents/skills/systematic-debugging/examples/four_phase_case.md +44 -0
  142. package/.agents/skills/technical-debt-harvester/SKILL.md +70 -0
  143. package/.agents/skills/technical-debt-harvester/examples/sample_debt_ledger.md +25 -0
  144. package/.agents/skills/technical-debt-harvester/examples/sample_ledger_output.md +20 -0
  145. package/.agents/skills/test-driven-development/SKILL.md +114 -0
  146. package/.agents/skills/test-driven-development/examples/red_green_refactor_qa.md +78 -0
  147. package/.agents/skills/token-and-context-optimizer/SKILL.md +70 -0
  148. package/.agents/skills/token-and-context-optimizer/examples/distilled_log_sample.txt +3 -0
  149. package/.agents/skills/token-and-context-optimizer/references/context_engineering_guide.md +49 -0
  150. package/.agents/skills/tot-exploratory-testing/SKILL.md +59 -0
  151. package/.agents/skills/tot-exploratory-testing/examples/payment_gateway_tot_tree.json +62 -0
  152. package/.agents/skills/tot-exploratory-testing/examples/tot_graph_sample.json +35 -0
  153. package/.agents/skills/verification-before-completion/SKILL.md +100 -0
  154. package/.agents/skills/verification-before-completion/examples/evidence_log_example.md +46 -0
  155. package/.agents/skills/visual-regression-sentinel/SKILL.md +82 -0
  156. package/.agents/skills/visual-regression-sentinel/examples/visual_mask_rules.json +9 -0
  157. package/.agents/skills/wcag-audit-patterns/SKILL.md +77 -0
  158. package/.agents/skills/wcag-audit-patterns/examples/wcag_compliance_checklist.json +10 -0
  159. package/.agents/skills/wcag-audit-patterns/resources/implementation-playbook.md +541 -0
  160. package/.agents/skills/web-agent-gym-benchmarking/SKILL.md +62 -0
  161. package/.agents/skills/web-agent-gym-benchmarking/examples/benchmark_report_sample.json +22 -0
  162. package/.agents/skills/web-agent-gym-benchmarking/examples/enterprise_tasks_suite.json +38 -0
  163. package/.agents/skills/web-vitals-and-performance-profiler/SKILL.md +65 -0
  164. package/.agents/skills/web-vitals-and-performance-profiler/examples/cwv_budget_policy.json +6 -0
  165. package/.agents/skills/web-vitals-and-performance-profiler/examples/playwright_cwv_profiler_fixture.ts +118 -0
  166. package/.agents/templates/completion_report.md +57 -0
  167. package/.agents/workflows/analyze_requirement_document.md +18 -0
  168. package/.agents/workflows/export_testcases_excel.md +32 -0
  169. package/.agents/workflows/generate_combinatorial_test_data.md +28 -0
  170. package/.agents/workflows/generate_cross_module_test_plan.md +37 -0
  171. package/.agents/workflows/generate_manual_testcases_rbt.md +56 -0
  172. package/.agents/workflows/generate_requirements_from_website.md +25 -0
  173. package/.agents/workflows/generate_testcases_from_requirements.md +21 -0
  174. package/.agents/workflows/graphify.md +10 -0
  175. package/.agents/workflows/test-impact.md +15 -0
  176. package/.agents/workflows/verify.md +34 -0
  177. package/.cursor/rules/agents.mdc +16 -0
  178. package/.cursor/rules/language.mdc +7 -0
  179. package/.cursor/rules/synapse-qa.mdc +24 -0
  180. package/.cursorrules +27 -0
  181. package/.dockerignore +53 -0
  182. package/.pre-commit-hooks.yaml +14 -0
  183. package/.windsurfrules +26 -0
  184. package/AGENTS.md +86 -0
  185. package/CLAUDE.md +54 -0
  186. package/Dockerfile +81 -0
  187. package/LICENSE +52 -0
  188. package/NOTICE +264 -0
  189. package/README.en.md +171 -0
  190. package/README.md +170 -2
  191. package/action.yml +51 -0
  192. package/agentskills.json +16 -0
  193. package/bin/agent-qa.js +138 -0
  194. package/docker-compose.yml +74 -0
  195. package/docs/ARCHITECTURE.en.md +198 -0
  196. package/docs/ARCHITECTURE.md +199 -0
  197. package/docs/CI_CD_INTEGRATION.en.md +170 -0
  198. package/docs/CI_CD_INTEGRATION.md +170 -0
  199. package/docs/CLI_REFERENCE.en.md +174 -0
  200. package/docs/CLI_REFERENCE.md +176 -0
  201. package/docs/CONTRIBUTING.en.md +49 -0
  202. package/docs/CONTRIBUTING.md +49 -0
  203. package/docs/PERFORMANCE_AUDIT_REPORT.md +783 -0
  204. package/docs/PLUGIN_DEVELOPMENT_GUIDE.md +127 -0
  205. package/docs/QUALITY_GATES.en.md +118 -0
  206. package/docs/QUALITY_GATES.md +118 -0
  207. package/docs/TOKEN_QUOTA_PLAYBOOK.md +41 -0
  208. package/docs/TOOL_CATALOG.md +129 -0
  209. package/docs/adopted-ecosystem-tools.md +96 -0
  210. package/docs/research/EXTERNAL_AGENT_SKILLS_SURVEY.md +1062 -0
  211. package/docs/research/agent_skill_training_analysis.md +1358 -0
  212. package/docs/schemas/synapse.config.schema.json +86 -0
  213. package/package.json +94 -4
  214. package/plugin.json +31 -0
  215. package/scripts/generate_skill_examples.py +244 -0
  216. package/scripts/install.ps1 +112 -0
  217. package/scripts/install_global_plugin.ps1 +94 -0
  218. package/scripts/install_global_plugin.sh +49 -0
  219. package/scripts/lint_skills.ps1 +57 -0
  220. package/scripts/package_dist.py +337 -0
  221. package/scripts/setup.sh +59 -0
  222. package/scripts/update_hub.ps1 +19 -0
  223. package/scripts/update_hub.sh +15 -0
  224. package/skills_manifest.json +854 -0
  225. package/synapse.config.json +31 -0
  226. package/templates/PROJECT_AI_GUIDE_TEMPLATE.md +111 -0
  227. package/templates/PROJECT_STATE_TEMPLATE.md +20 -0
  228. package/templates/ai_bridges/CLAUDE.md +54 -0
  229. package/templates/ai_bridges/copilot-instructions.md +20 -0
  230. package/templates/ai_bridges/cursor_synapse_qa.mdc +24 -0
  231. package/templates/ai_bridges/windsurfrules +26 -0
  232. package/templates/ci/github-actions-synapse.yml +95 -0
  233. package/templates/ci/gitlab-ci-synapse.yml +16 -0
  234. package/templates/ui/index.html +1069 -0
  235. package/templates/welcome.html.tmpl +2074 -0
  236. package/tools/.infer_baseline.json +30 -0
  237. package/tools/TOOLS_RUNBOOK.md +1553 -0
  238. package/tools/TOOL_CATALOG.json +1769 -0
  239. package/tools/__init__.py +43 -0
  240. package/tools/_openapi_parser.py +158 -0
  241. package/tools/agent_canary.py +525 -0
  242. package/tools/agentless_patcher.py +417 -0
  243. package/tools/analyze_project.py +262 -0
  244. package/tools/api_drift_sentinel.py +208 -0
  245. package/tools/api_fuzzer.py +762 -0
  246. package/tools/author_integrity.py +190 -0
  247. package/tools/auto_heal.py +271 -0
  248. package/tools/auto_heal_reflexion.py +532 -0
  249. package/tools/autogen_qa_broker.py +844 -0
  250. package/tools/benchmark_swarm_context.py +394 -0
  251. package/tools/browser_helpers.js +746 -0
  252. package/tools/browsergym_eval.py +236 -0
  253. package/tools/bug_reporter.py +252 -0
  254. package/tools/bump_version.py +423 -0
  255. package/tools/cache_resolver.py +1214 -0
  256. package/tools/check_env.py +426 -0
  257. package/tools/check_version_sync.py +256 -0
  258. package/tools/ci_healing_hook.py +126 -0
  259. package/tools/cli.py +1598 -0
  260. package/tools/config_decoupler.py +253 -0
  261. package/tools/contract_validator.py +141 -0
  262. package/tools/core/__init__.py +70 -0
  263. package/tools/core/adapters.py +431 -0
  264. package/tools/core/circuit_breaker.py +263 -0
  265. package/tools/core/cli_runtime.py +66 -0
  266. package/tools/core/file_lock.py +159 -0
  267. package/tools/core/i18n.py +394 -0
  268. package/tools/core/memory_bus.py +905 -0
  269. package/tools/core/process_supervisor.py +332 -0
  270. package/tools/core/telemetry.py +329 -0
  271. package/tools/count_tokens.py +1267 -0
  272. package/tools/create_mock_docx.py +104 -0
  273. package/tools/dashboard_generator.py +2403 -0
  274. package/tools/db_sandbox.py +675 -0
  275. package/tools/debt_harvester.py +237 -0
  276. package/tools/distill_log.py +161 -0
  277. package/tools/dspy_qa_signatures.py +356 -0
  278. package/tools/e2e_flow_prober.py +132 -0
  279. package/tools/export_postman_collection.py +945 -0
  280. package/tools/flaky_detector.py +389 -0
  281. package/tools/fsm_enforcer.py +450 -0
  282. package/tools/fsm_middleware.py +401 -0
  283. package/tools/fuzz_payloads.py +197 -0
  284. package/tools/generate_allure_report.py +88 -0
  285. package/tools/generate_api_matrix.py +193 -0
  286. package/tools/generate_jmx_plan.py +1322 -0
  287. package/tools/generate_sbom.py +252 -0
  288. package/tools/generate_tc_from_story.py +203 -0
  289. package/tools/generate_test_reports.py +133 -0
  290. package/tools/generate_tool_catalog.py +482 -0
  291. package/tools/gherkin_suite.py +758 -0
  292. package/tools/git_pre_commit_guard.py +298 -0
  293. package/tools/goldfish_review.py +366 -0
  294. package/tools/heal/__init__.py +28 -0
  295. package/tools/heal_pr_automator.py +220 -0
  296. package/tools/heuristic_locator_matcher.py +351 -0
  297. package/tools/hub_publisher.py +623 -0
  298. package/tools/humanize_linter.py +438 -0
  299. package/tools/infer_linter.py +977 -0
  300. package/tools/init_config.py +306 -0
  301. package/tools/interview_story.py +116 -0
  302. package/tools/jira_sync.py +566 -0
  303. package/tools/jtl_analyzer.py +474 -0
  304. package/tools/lint/__init__.py +30 -0
  305. package/tools/lint_ai_docs.py +374 -0
  306. package/tools/lookup_tc.py +165 -0
  307. package/tools/lookup_xpath.py +167 -0
  308. package/tools/matrix_sharder.py +148 -0
  309. package/tools/mcp/__init__.py +18 -0
  310. package/tools/mcp_docx_server.ps1 +13 -0
  311. package/tools/mcp_docx_server.py +851 -0
  312. package/tools/mcp_excel_server.ps1 +13 -0
  313. package/tools/mcp_excel_server.py +441 -0
  314. package/tools/mcp_server.py +725 -0
  315. package/tools/md_to_xlsx.py +327 -0
  316. package/tools/micro_mutator.py +887 -0
  317. package/tools/mipro_assertion_tuner.py +210 -0
  318. package/tools/network_mock_shield.py +311 -0
  319. package/tools/newman_analyzer.py +384 -0
  320. package/tools/omni_emitter.py +579 -0
  321. package/tools/omni_parser.py +450 -0
  322. package/tools/omni_transpiler.py +168 -0
  323. package/tools/openxml_guard.py +438 -0
  324. package/tools/output_formatter.py +182 -0
  325. package/tools/package_release.py +325 -0
  326. package/tools/perf/__init__.py +24 -0
  327. package/tools/pinpoint_failure.py +190 -0
  328. package/tools/project_state.py +171 -0
  329. package/tools/qa_event_stream.py +358 -0
  330. package/tools/repo_map.py +346 -0
  331. package/tools/report/__init__.py +32 -0
  332. package/tools/reversible_sandbox.py +344 -0
  333. package/tools/safety_firewall.py +437 -0
  334. package/tools/security_coverage_ledger.py +538 -0
  335. package/tools/security_scanner.py +229 -0
  336. package/tools/senior_code_review.py +393 -0
  337. package/tools/setup_project.py +770 -0
  338. package/tools/skill_collision_guard.py +304 -0
  339. package/tools/skill_profiles.py +217 -0
  340. package/tools/skill_vault.py +481 -0
  341. package/tools/skillspector.py +1278 -0
  342. package/tools/smart_visual_masker.py +132 -0
  343. package/tools/spec_drift_sentinel.py +438 -0
  344. package/tools/swe_qa_aci.py +369 -0
  345. package/tools/synapse_observer.py +485 -0
  346. package/tools/synapse_qa_bench.py +190 -0
  347. package/tools/sync_agent_rules.py +274 -0
  348. package/tools/telemetry_correlator.py +397 -0
  349. package/tools/template_excel_exporter.py +514 -0
  350. package/tools/templates/jmeter-distributed/README.md +54 -0
  351. package/tools/templates/jmeter-distributed/docker-compose.yml +83 -0
  352. package/tools/test_impact.py +419 -0
  353. package/tools/test_memory_os.py +462 -0
  354. package/tools/textgrad_mutation_optimizer.py +237 -0
  355. package/tools/tms_sync_bridge.py +834 -0
  356. package/tools/tot_exploratory_tester.py +239 -0
  357. package/tools/trace_data_flow.py +240 -0
  358. package/tools/traffic_session_converter.py +256 -0
  359. package/tools/trend_analytics.py +314 -0
  360. package/tools/ui_server.py +401 -0
  361. package/tools/universal_ir.py +488 -0
  362. package/tools/update_sentinel.py +383 -0
  363. package/tools/validate_groovy_scripts.py +173 -0
  364. package/tools/verify_all.py +1271 -0
  365. package/tools/visual_diff_canvas.py +143 -0
  366. package/tools/visual_som_helper.py +550 -0
  367. package/tools/watch_mode.py +308 -0
  368. package/tools/wcag_audit.py +239 -0
  369. package/tools/welcome_page.py +290 -0
@@ -0,0 +1,143 @@
1
+ #!/usr/bin/env python3
2
+ # -*- coding: utf-8 -*-
3
+ """
4
+ Enterprise Canvas Visual Diff & Regression Checker
5
+ ===================================================
6
+ Provides visual regression AI testing capabilities for HTML5 Canvas, SVG,
7
+ and DOM-less charts (e.g. DataMiningPage / GIS / Analytics).
8
+
9
+ Features:
10
+ 1. compare: Pixel-by-pixel Canvas comparison against Golden Baseline.
11
+ Computes diff ratio and generates highlight overlay diff artifacts.
12
+ 2. check-blank: Detects white-screen or empty non-rendered Canvas regressions.
13
+ 3. update-baseline: Captures new Golden Baseline upon design updates.
14
+
15
+ Usage:
16
+ python3 tools/visual_diff_canvas.py compare baseline.png current.png --diff diff.png --threshold 0.02
17
+ python3 tools/visual_diff_canvas.py check-blank current.png
18
+ """
19
+ from __future__ import annotations
20
+
21
+ import os
22
+ import sys
23
+ import argparse
24
+
25
+ try:
26
+ from PIL import Image, ImageChops, ImageStat
27
+ except ImportError:
28
+ print("❌ Error: Pillow library not installed. Please install: pip install Pillow")
29
+ sys.exit(1)
30
+
31
+
32
+ def is_canvas_blank(image_path: str, std_dev_threshold: float = 2.0) -> tuple[bool, float]:
33
+ """
34
+ Checks if a canvas screenshot is blank, empty, or single-color non-rendered.
35
+ Returns (is_blank, std_dev).
36
+ """
37
+ with Image.open(image_path) as img:
38
+ gray = img.convert("L")
39
+ stat = ImageStat.Stat(gray)
40
+ std_dev = stat.stddev[0]
41
+ # Low standard deviation indicates uniform solid/blank canvas
42
+ return (std_dev < std_dev_threshold, std_dev)
43
+
44
+
45
+ def compare_canvas_images(
46
+ baseline_path: str,
47
+ current_path: str,
48
+ diff_output_path: str | None = None,
49
+ threshold: float = 0.01
50
+ ) -> tuple[bool, float, int]:
51
+ """
52
+ Compares two canvas screenshots pixel-by-pixel.
53
+ Returns (is_match, diff_ratio, diff_pixels).
54
+ threshold: Allowed diff ratio (e.g. 0.01 = 1% tolerance)
55
+ """
56
+ with Image.open(baseline_path) as img_base, Image.open(current_path) as img_curr:
57
+ # Normalize dimensions and color mode
58
+ curr_image: Image.Image = img_curr
59
+ if img_base.size != img_curr.size:
60
+ curr_image = img_curr.resize(img_base.size, Image.Resampling.LANCZOS)
61
+
62
+ base_rgb = img_base.convert("RGB")
63
+ curr_rgb = curr_image.convert("RGB")
64
+
65
+ # C-level accelerated pixel difference via Pillow vectorization
66
+ diff = ImageChops.difference(base_rgb, curr_rgb)
67
+ gray_diff = diff.convert("L")
68
+ total_pixels = img_base.width * img_base.height
69
+
70
+ # Histogram-based count of divergent pixels (delta > 10, < 0.03s for 1920x1080)
71
+ hist = gray_diff.histogram()
72
+ diff_count = sum(hist[11:])
73
+
74
+ diff_ratio = diff_count / total_pixels
75
+ is_match = diff_ratio <= threshold
76
+
77
+ if diff_output_path and not is_match:
78
+ os.makedirs(os.path.dirname(os.path.abspath(diff_output_path)), exist_ok=True)
79
+ red_layer = Image.new("RGBA", img_base.size, (255, 0, 0, 200))
80
+ alpha_mask = gray_diff.point(lambda p: 200 if p > 10 else 0)
81
+ red_layer.putalpha(alpha_mask)
82
+ annotated = curr_rgb.convert("RGBA")
83
+ annotated.paste(red_layer, (0, 0), red_layer)
84
+ annotated.save(diff_output_path)
85
+
86
+ return (is_match, diff_ratio, diff_count)
87
+
88
+
89
+ def main():
90
+ parser = argparse.ArgumentParser(description="Enterprise Canvas Visual Diff & Regression Checker")
91
+ subparsers = parser.add_subparsers(dest="command", required=True)
92
+
93
+ # Command: compare
94
+ compare_p = subparsers.add_parser("compare", help="Compare current canvas screenshot against Golden Baseline")
95
+ compare_p.add_argument("baseline", help="Path to Golden Baseline image")
96
+ compare_p.add_argument("current", help="Path to current runtime screenshot")
97
+ compare_p.add_argument("--diff", default=None, help="Output path for annotated diff highlight image")
98
+ compare_p.add_argument("--threshold", type=float, default=0.01, help="Max allowed divergence ratio (default: 0.01 = 1%%)")
99
+
100
+ # Command: check-blank
101
+ blank_p = subparsers.add_parser("check-blank", help="Check whether canvas is blank or unrendered")
102
+ blank_p.add_argument("current", help="Path to canvas screenshot to verify")
103
+ blank_p.add_argument("--threshold", type=float, default=2.0, help="Min standard deviation for valid render (default: 2.0)")
104
+
105
+ args = parser.parse_args()
106
+
107
+ if args.command == "compare":
108
+ if not os.path.exists(args.baseline):
109
+ print(f"❌ Baseline file not found: {args.baseline}")
110
+ sys.exit(2)
111
+ if not os.path.exists(args.current):
112
+ print(f"❌ Current screenshot file not found: {args.current}")
113
+ sys.exit(2)
114
+
115
+ is_match, diff_ratio, diff_count = compare_canvas_images(
116
+ args.baseline, args.current, args.diff, args.threshold
117
+ )
118
+ pct = diff_ratio * 100
119
+ if is_match:
120
+ print(f"✅ PASS: Visual canvas matches Baseline (Diff: {pct:.2f}%, {diff_count} px <= {args.threshold*100:.1f}%)")
121
+ sys.exit(0)
122
+ else:
123
+ print(f"❌ FAIL: Visual canvas exceeded divergence threshold (Diff: {pct:.2f}%, {diff_count} px > {args.threshold*100:.1f}%)")
124
+ if args.diff:
125
+ print(f" 📸 Annotated diff artifact saved to: {args.diff}")
126
+ sys.exit(1)
127
+
128
+ elif args.command == "check-blank":
129
+ if not os.path.exists(args.current):
130
+ print(f"❌ Screenshot file not found: {args.current}")
131
+ sys.exit(2)
132
+
133
+ blank, std_dev = is_canvas_blank(args.current, args.threshold)
134
+ if blank:
135
+ print(f"❌ WARNING: Canvas is blank / unrendered (Standard Deviation: {std_dev:.2f} < {args.threshold})")
136
+ sys.exit(1)
137
+ else:
138
+ print(f"✅ PASS: Canvas renders valid graphical content (Standard Deviation: {std_dev:.2f} >= {args.threshold})")
139
+ sys.exit(0)
140
+
141
+
142
+ if __name__ == "__main__":
143
+ main()
@@ -0,0 +1,550 @@
1
+ #!/usr/bin/env python3
2
+ """
3
+ Enterprise Vision-Grounded Fallback & Set-of-Marks (SoM) Engine
4
+ --------------------------------------------------------------
5
+ Provides multimodal coordinate grounding when standard DOM locators fail
6
+ (HTML5 Canvas, WebGL financial charts, deeply nested Shadow DOM, cross-origin iframes).
7
+
8
+ Features:
9
+ 1. Set-of-Marks (SoM) Overlay Generation:
10
+ - Detects interactive visual regions (or accepts DOM bounding boxes).
11
+ - Renders high-contrast colored bounding boxes with numeric IDs (1, 2, 3...) on screenshots.
12
+ 2. Coordinate Resolution & Physical Mouse Dispatch:
13
+ - Maps Mark IDs and visual targets to exact center coordinates (x, y).
14
+ - Generates direct Playwright physical mouse actions: await page.mouse.click(x, y).
15
+ 3. Visual Perceptual Caching & Token Economy:
16
+ - Generates difference hash (dHash) fingerprints for visual components.
17
+ - Caches coordinates in SQLite (memory_cache.db). If visual similarity > 98%,
18
+ instantly retrieves cached coordinates without expensive Vision LLM calls (Zero-Token).
19
+ 4. Compact Vision Prompt Formatter:
20
+ - Formats token-efficient briefs for Multimodal Vision Models (Gemini / Claude / GPT-4o).
21
+ """
22
+ from __future__ import annotations
23
+
24
+ import os
25
+ import sys
26
+ import json
27
+ import sqlite3
28
+ import argparse
29
+ from typing import Dict, Any, List, Optional, Tuple
30
+ try:
31
+ from PIL import Image, ImageDraw, ImageFont, ImageFilter
32
+ HAS_PIL = True
33
+ except ImportError:
34
+ Image = None # type: ignore[assignment]
35
+ ImageDraw = None # type: ignore[assignment]
36
+ ImageFont = None # type: ignore[assignment]
37
+ ImageFilter = None # type: ignore[assignment]
38
+ HAS_PIL = False
39
+
40
+ ImageSourceType = Any
41
+
42
+ # Enforce UTF-8 stdout encoding for Windows console compatibility
43
+ if hasattr(sys.stdout, 'reconfigure'):
44
+ try:
45
+ sys.stdout.reconfigure(encoding='utf-8')
46
+ sys.stderr.reconfigure(encoding='utf-8')
47
+ except Exception:
48
+ pass
49
+
50
+ SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
51
+ QA_HUB_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, ".."))
52
+ DEFAULT_CACHE_DB = os.path.expanduser("~/.agent_memory/global_cache.db")
53
+
54
+ MARK_COLORS = [
55
+ "#FF0055", # Vibrant Red/Pink
56
+ "#00CCFF", # Cyan
57
+ "#00FF66", # Neon Green
58
+ "#FFCC00", # Bright Yellow
59
+ "#9933FF", # Purple
60
+ "#FF6600", # Orange
61
+ ]
62
+
63
+
64
+ def compute_visual_hash(image_source: ImageSourceType, bbox: Optional[List[int]] = None, hash_size: int = 8) -> str:
65
+ """
66
+ Computes a 64-bit difference hash (dHash) for an image or cropped component.
67
+ Fast, deterministic, and 100% offline.
68
+ """
69
+ if not HAS_PIL or Image is None:
70
+ raise ImportError("Pillow is required for visual perceptual operations (pip install Pillow).")
71
+ if isinstance(image_source, str):
72
+ if not os.path.exists(image_source):
73
+ raise FileNotFoundError(f"Image not found: {image_source}")
74
+ img: Any = Image.open(image_source)
75
+ else:
76
+ img = image_source
77
+
78
+ if bbox and len(bbox) == 4:
79
+ img = img.crop(tuple(bbox))
80
+
81
+ # Resize to (hash_size + 1, hash_size) in grayscale
82
+ resized = img.convert("L").resize((hash_size + 1, hash_size), Image.Resampling.LANCZOS)
83
+ pixels = list(resized.tobytes())
84
+
85
+ diff = []
86
+ for row in range(hash_size):
87
+ row_start = row * (hash_size + 1)
88
+ for col in range(hash_size):
89
+ left = pixels[row_start + col]
90
+ right = pixels[row_start + col + 1]
91
+ diff.append("1" if left > right else "0")
92
+
93
+ return "".join(diff)
94
+
95
+
96
+ def calculate_visual_similarity(hash1: str, hash2: str) -> float:
97
+ """Calculates normalized Hamming similarity between two binary hashes (0.0 to 1.0)."""
98
+ if not hash1 or not hash2 or len(hash1) != len(hash2):
99
+ return 0.0
100
+ hamming_dist = sum(c1 != c2 for c1, c2 in zip(hash1, hash2))
101
+ return round(1.0 - (hamming_dist / len(hash1)), 4)
102
+
103
+
104
+ class VisualSoMHelper:
105
+ """Manages visual grounding, bounding box marking, visual caching, and coordinate dispatch."""
106
+
107
+ def __init__(self, viewport_width: int = 1920, viewport_height: int = 1080, db_path: Optional[str] = None):
108
+ self.viewport_width = viewport_width
109
+ self.viewport_height = viewport_height
110
+ self.db_path = db_path or DEFAULT_CACHE_DB
111
+ self.init_cache_table()
112
+
113
+ def init_cache_table(self):
114
+ """Initializes the visual_som_cache SQLite table if not already present."""
115
+ try:
116
+ conn = sqlite3.connect(self.db_path, timeout=10.0)
117
+ try:
118
+ conn.execute("""
119
+ CREATE TABLE IF NOT EXISTS visual_som_cache (
120
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
121
+ component_name TEXT NOT NULL,
122
+ visual_hash TEXT NOT NULL,
123
+ center_x INTEGER NOT NULL,
124
+ center_y INTEGER NOT NULL,
125
+ rel_x REAL DEFAULT 0.0,
126
+ rel_y REAL DEFAULT 0.0,
127
+ hit_count INTEGER DEFAULT 0,
128
+ created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
129
+ updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
130
+ UNIQUE(component_name, visual_hash)
131
+ );
132
+ """)
133
+ conn.execute("CREATE INDEX IF NOT EXISTS idx_som_component ON visual_som_cache (component_name);")
134
+ conn.commit()
135
+ finally:
136
+ conn.close()
137
+ except (sqlite3.Error, OSError):
138
+ self.db_path = ":memory:"
139
+
140
+ def lookup_cached_coordinates(
141
+ self,
142
+ component_name: str,
143
+ image_source: ImageSourceType,
144
+ bbox: Optional[List[int]] = None,
145
+ threshold: float = 0.98,
146
+ db_path: Optional[str] = None
147
+ ) -> Optional[Dict[str, Any]]:
148
+ """
149
+ Looks up cached coordinates for a component if visual similarity exceeds threshold (default 98%).
150
+ Returns resolved coordinate data if cache hit, None if cache miss.
151
+ """
152
+ current_hash = compute_visual_hash(image_source, bbox=bbox)
153
+ db = db_path or self.db_path
154
+
155
+ try:
156
+ conn = sqlite3.connect(db, timeout=10.0)
157
+ try:
158
+ conn.row_factory = sqlite3.Row
159
+ cursor = conn.cursor()
160
+ cursor.execute(
161
+ "SELECT id, component_name, visual_hash, center_x, center_y, hit_count FROM visual_som_cache WHERE component_name = ?",
162
+ (component_name,)
163
+ )
164
+ rows = cursor.fetchall()
165
+
166
+ best_match = None
167
+ best_sim = -1.0
168
+
169
+ for row in rows:
170
+ cached_hash = row["visual_hash"]
171
+ sim = calculate_visual_similarity(current_hash, cached_hash)
172
+ if sim >= threshold and sim > best_sim:
173
+ best_sim = sim
174
+ best_match = row
175
+
176
+ if best_match:
177
+ match_id = best_match["id"]
178
+ cursor.execute(
179
+ "UPDATE visual_som_cache SET hit_count = hit_count + 1, updated_at = CURRENT_TIMESTAMP WHERE id = ?",
180
+ (match_id,)
181
+ )
182
+ conn.commit()
183
+
184
+ cx = best_match["center_x"]
185
+ cy = best_match["center_y"]
186
+ return {
187
+ "component_name": component_name,
188
+ "center_x": cx,
189
+ "center_y": cy,
190
+ "visual_similarity": best_sim,
191
+ "visual_hash": current_hash,
192
+ "cached": True,
193
+ "hit_count": best_match["hit_count"] + 1,
194
+ "playwright_action": f"await page.mouse.click({cx}, {cy});"
195
+ }
196
+ finally:
197
+ conn.close()
198
+ except (sqlite3.Error, OSError):
199
+ self.db_path = ":memory:"
200
+
201
+ return None
202
+
203
+ def save_cached_coordinates(
204
+ self,
205
+ component_name: str,
206
+ image_source: ImageSourceType,
207
+ center_x: int,
208
+ center_y: int,
209
+ bbox: Optional[List[int]] = None,
210
+ db_path: Optional[str] = None
211
+ ) -> str:
212
+ """
213
+ Saves resolved coordinates and visual hash into SQLite cache.
214
+ """
215
+ visual_hash = compute_visual_hash(image_source, bbox=bbox)
216
+ db = db_path or self.db_path
217
+
218
+ try:
219
+ conn = sqlite3.connect(db, timeout=10.0)
220
+ try:
221
+ conn.execute("""
222
+ INSERT INTO visual_som_cache (component_name, visual_hash, center_x, center_y, hit_count)
223
+ VALUES (?, ?, ?, ?, 1)
224
+ ON CONFLICT(component_name, visual_hash) DO UPDATE SET
225
+ center_x = excluded.center_x,
226
+ center_y = excluded.center_y,
227
+ hit_count = visual_som_cache.hit_count + 1,
228
+ updated_at = CURRENT_TIMESTAMP;
229
+ """, (component_name, visual_hash, center_x, center_y))
230
+ conn.commit()
231
+ finally:
232
+ conn.close()
233
+ except (sqlite3.Error, OSError):
234
+ self.db_path = ":memory:"
235
+
236
+ return visual_hash
237
+
238
+ def detect_canvas_interactive_regions(
239
+ self,
240
+ image_path: str,
241
+ stride: int = 16,
242
+ edge_threshold: int = 28,
243
+ min_pixels_per_cell: int = 6,
244
+ min_w: int = 16,
245
+ min_h: int = 16,
246
+ max_ratio: float = 0.88
247
+ ) -> List[Dict[str, Any]]:
248
+ """
249
+ Standalone Computer Vision Contour / Active Blob Detector for Canvas/GIS.
250
+ Detects interactive visual components (buttons, polygon nodes, markers, sliders, chart icons)
251
+ without requiring DOM elements or accessibility nodes.
252
+ Uses pure PIL edge filtering and connected-grid component clustering (Zero-Dependency).
253
+ """
254
+ if not os.path.exists(image_path) or not HAS_PIL:
255
+ return []
256
+
257
+ img = Image.open(image_path).convert("L")
258
+ w, h = img.size
259
+ max_allowed_w = int(w * max_ratio)
260
+ max_allowed_h = int(h * max_ratio)
261
+
262
+ # Detect high-contrast edges & contours
263
+ edges = img.filter(ImageFilter.FIND_EDGES)
264
+ edges = edges.point(lambda p: 255 if p > edge_threshold else 0)
265
+ pixels = edges.load()
266
+ if pixels is None:
267
+ return []
268
+
269
+ # Identify active cells with high edge density
270
+ active_cells = set()
271
+ for y in range(0, h, stride):
272
+ for x in range(0, w, stride):
273
+ count = 0
274
+ for cy in range(y, min(h, y + stride)):
275
+ for cx in range(x, min(w, x + stride)):
276
+ px_val = pixels[cx, cy]
277
+ if isinstance(px_val, (int, float)) and px_val > 0:
278
+ count += 1
279
+ elif isinstance(px_val, tuple) and px_val and px_val[0] > 0:
280
+ count += 1
281
+ if count >= min_pixels_per_cell:
282
+ active_cells.add((x // stride, y // stride))
283
+
284
+ # 8-direction BFS clustering to group connected components
285
+ visited = set()
286
+ detected_elements = []
287
+ cluster_id = 1
288
+
289
+ for cell in sorted(active_cells):
290
+ if cell in visited:
291
+ continue
292
+ queue = [cell]
293
+ visited.add(cell)
294
+ cluster = [cell]
295
+
296
+ while queue:
297
+ curr = queue.pop(0)
298
+ cx, cy = curr
299
+ for dx in (-1, 0, 1):
300
+ for dy in (-1, 0, 1):
301
+ nb = (cx + dx, cy + dy)
302
+ if nb in active_cells and nb not in visited:
303
+ visited.add(nb)
304
+ queue.append(nb)
305
+ cluster.append(nb)
306
+
307
+ min_x = max(0, min(c[0] for c in cluster) * stride - 4)
308
+ min_y = max(0, min(c[1] for c in cluster) * stride - 4)
309
+ box_w = min(w - min_x, (max(c[0] for c in cluster) + 1) * stride - min_x + 8)
310
+ box_h = min(h - min_y, (max(c[1] for c in cluster) + 1) * stride - min_y + 8)
311
+
312
+ # Filter out whole-canvas background borders or tiny noise
313
+ if box_w >= min_w and box_h >= min_h and (box_w < max_allowed_w or box_h < max_allowed_h):
314
+ detected_elements.append({
315
+ "x": min_x,
316
+ "y": min_y,
317
+ "width": box_w,
318
+ "height": box_h,
319
+ "label": f"Canvas_Target_{cluster_id}",
320
+ "tag": "canvas_interactive_node"
321
+ })
322
+ cluster_id += 1
323
+
324
+ return detected_elements
325
+
326
+ def generate_som_overlay(
327
+ self,
328
+ image_path: str,
329
+ elements: Optional[List[Dict[str, Any]]] = None,
330
+ output_path: Optional[str] = None
331
+ ) -> Tuple[str, Dict[int, Dict[str, Any]]]:
332
+ """
333
+ Draws high-contrast numbered badges (Set-of-Marks) over each element bounding box.
334
+ If elements is omitted, autonomously runs detect_canvas_interactive_regions.
335
+ Returns (marked_image_path, marks_map).
336
+ """
337
+ if not HAS_PIL or Image is None:
338
+ raise ImportError("Pillow is required for Set-of-Marks overlay generation (pip install Pillow).")
339
+
340
+ if not os.path.exists(image_path):
341
+ img = Image.new("RGB", (self.viewport_width, self.viewport_height), color="#1E1E2E")
342
+ draw = ImageDraw.Draw(img)
343
+ draw.text((50, 50), "Mock Financial Dashboard (HTML5 Canvas)", fill="#FFFFFF")
344
+ draw.rectangle([100, 150, 220, 190], outline="#00CCFF", width=2)
345
+ draw.rectangle([240, 150, 360, 190], outline="#FF0055", width=2)
346
+ draw.rectangle([500, 300, 700, 350], outline="#00FF66", width=2)
347
+ img.save(image_path)
348
+
349
+ if not elements:
350
+ detected = self.detect_canvas_interactive_regions(image_path)
351
+ elements = detected if detected else [
352
+ {"x": 100, "y": 150, "width": 120, "height": 40, "label": "TradingView Zoom In", "tag": "canvas_control"},
353
+ {"x": 240, "y": 150, "width": 120, "height": 40, "label": "TradingView Zoom Out", "tag": "canvas_control"},
354
+ {"x": 500, "y": 300, "width": 200, "height": 50, "label": "Execute Trade Order", "tag": "shadow_button"},
355
+ {"x": 720, "y": 300, "width": 150, "height": 50, "label": "Cancel Order", "tag": "shadow_button"}
356
+ ]
357
+
358
+ img = Image.open(image_path).convert("RGBA")
359
+ overlay = Image.new("RGBA", img.size, (255, 255, 255, 0))
360
+ draw = ImageDraw.Draw(overlay)
361
+
362
+ marks_map = {}
363
+ for idx, el in enumerate(elements, start=1):
364
+ x = el.get("x", 0)
365
+ y = el.get("y", 0)
366
+ w = el.get("width", 80)
367
+ h = el.get("height", 35)
368
+
369
+ center_x = round(x + w / 2)
370
+ center_y = round(y + h / 2)
371
+ color = MARK_COLORS[(idx - 1) % len(MARK_COLORS)]
372
+
373
+ # Draw bounding box
374
+ draw.rectangle([x, y, x + w, y + h], outline=color, width=3)
375
+
376
+ # Draw badge label (e.g. [1])
377
+ badge_w, badge_h = 24, 20
378
+ badge_x = max(0, x)
379
+ badge_y = max(0, y - badge_h)
380
+ draw.rectangle([badge_x, badge_y, badge_x + badge_w, badge_y + badge_h], fill=color)
381
+ draw.text((badge_x + 6, badge_y + 3), str(idx), fill="#000000")
382
+
383
+ marks_map[idx] = {
384
+ "mark_id": idx,
385
+ "label": el.get("label") or el.get("text") or el.get("role") or f"Element_{idx}",
386
+ "tag": el.get("tag", "element"),
387
+ "bounds": [x, y, x + w, y + h],
388
+ "center_x": center_x,
389
+ "center_y": center_y,
390
+ "playwright_action": f"await page.mouse.click({center_x}, {center_y});"
391
+ }
392
+
393
+ combined = Image.alpha_composite(img, overlay).convert("RGB")
394
+ out_file = output_path or image_path.replace(".png", "_som_marked.png")
395
+ combined.save(out_file)
396
+
397
+ return out_file, marks_map
398
+
399
+ def resolve_mark_coordinates(
400
+ self,
401
+ marks_map: Dict[int, Dict[str, Any]],
402
+ target_mark_id: Optional[int] = None,
403
+ target_label: Optional[str] = None
404
+ ) -> Optional[Dict[str, Any]]:
405
+ """
406
+ Resolves the exact (x, y) coordinates for a given Mark ID or fuzzy label.
407
+ """
408
+ if target_mark_id and target_mark_id in marks_map:
409
+ return marks_map[target_mark_id]
410
+
411
+ if target_label:
412
+ target_clean = target_label.strip().lower()
413
+ for mark in marks_map.values():
414
+ if target_clean in mark.get("label", "").lower():
415
+ return mark
416
+
417
+ return None
418
+
419
+ def format_vision_prompt(self, marks_map: Dict[int, Dict[str, Any]], goal: str) -> str:
420
+ """
421
+ Formats a compact Vision brief for multimodal models (Gemini / Claude).
422
+ """
423
+ lines = [
424
+ f"GOAL: {goal}",
425
+ f"SET-OF-MARKS (SoM) CANDIDATE REGIONS DETECTED: {len(marks_map)}",
426
+ "",
427
+ "Inspect the attached marked screenshot and identify the target Mark ID:",
428
+ ]
429
+ for m_id, data in marks_map.items():
430
+ lines.append(f" [{m_id}] '{data['label']}' -> Center ({data['center_x']}, {data['center_y']})")
431
+ lines.append("")
432
+ lines.append("RESPOND WITH FORMAT: CLICK_MARK: <ID>")
433
+ return "\n".join(lines)
434
+
435
+
436
+ def main():
437
+ parser = argparse.ArgumentParser(description="Multimodal Vision Grounding & Set-of-Marks Engine")
438
+ parser.add_argument("--image", default="screenshot.png", help="Path to input screenshot image")
439
+ parser.add_argument("--output", default="", help="Path to output marked image")
440
+ parser.add_argument("--elements-json", default="", help="JSON string or file of candidate bounding boxes")
441
+ parser.add_argument("--mark-id", type=int, default=None, help="Target Mark ID to resolve")
442
+ parser.add_argument("--label", default="", help="Target semantic label to resolve")
443
+ parser.add_argument("--component", default="", help="Target component name for visual cache lookup/save")
444
+ parser.add_argument("--cache-lookup", action="store_true", help="Attempt zero-token visual cache resolution first")
445
+ parser.add_argument("--cache-save", action="store_true", help="Save resolved coordinates to visual cache")
446
+ parser.add_argument("--threshold", type=float, default=0.98, help="Visual similarity threshold for cache hit (default: 0.98)")
447
+ parser.add_argument("--detect-only", action="store_true", help="Run pure contour/blob detector and print detected bounding boxes")
448
+ parser.add_argument("--json", action="store_true", help="Output results in JSON format")
449
+ args = parser.parse_args()
450
+
451
+ helper = VisualSoMHelper()
452
+
453
+ # Step 1: Zero-Token Visual Cache Lookup
454
+ if (args.cache_lookup or args.component) and args.component and os.path.exists(args.image):
455
+ cached_result = helper.lookup_cached_coordinates(
456
+ component_name=args.component,
457
+ image_source=args.image,
458
+ threshold=args.threshold
459
+ )
460
+ if cached_result:
461
+ if args.json:
462
+ print(json.dumps({"status": "CACHE_HIT", "data": cached_result}, indent=2))
463
+ else:
464
+ print(f"\n⚡ [CACHE HIT - ZERO VISION TOKEN] Component '{args.component}' matched ({cached_result['visual_similarity']*100:.1f}% Similarity)")
465
+ print(f" Coordinates : ({cached_result['center_x']}, {cached_result['center_y']})")
466
+ print(f" Hit Count : {cached_result['hit_count']}")
467
+ print(f" Action : {cached_result['playwright_action']}\n")
468
+ return 0
469
+
470
+ # Auto-generate mock canvas if image doesn't exist
471
+ if not os.path.exists(args.image):
472
+ img = Image.new("RGB", (helper.viewport_width, helper.viewport_height), color="#1E1E2E")
473
+ draw = ImageDraw.Draw(img)
474
+ draw.text((50, 50), "Mock GIS & Financial Dashboard (HTML5 Canvas)", fill="#FFFFFF")
475
+ draw.rectangle([100, 150, 220, 190], fill="#00CCFF", outline="#FFFFFF", width=2)
476
+ draw.rectangle([240, 150, 360, 190], fill="#FF0055", outline="#FFFFFF", width=2)
477
+ draw.rectangle([500, 300, 700, 350], fill="#00FF66", outline="#FFFFFF", width=2)
478
+ img.save(args.image)
479
+
480
+ # Detect Only
481
+ if args.detect_only:
482
+ blobs = helper.detect_canvas_interactive_regions(args.image)
483
+ if args.json:
484
+ print(json.dumps(blobs, indent=2))
485
+ else:
486
+ print(f"\n🔍 [CANVAS CONTOURS DETECTED] Found {len(blobs)} interactive regions:")
487
+ for b in blobs:
488
+ print(f" • {b['label']}: Bounds {b['x']},{b['y']} ({b['width']}x{b['height']} px)")
489
+ return 0
490
+
491
+ # Load elements
492
+ elements = []
493
+ if args.elements_json:
494
+ if os.path.exists(args.elements_json):
495
+ with open(args.elements_json, "r", encoding="utf-8") as f:
496
+ elements = json.load(f)
497
+ else:
498
+ try:
499
+ elements = json.loads(args.elements_json)
500
+ except json.JSONDecodeError:
501
+ elements = []
502
+
503
+ if not elements:
504
+ detected = helper.detect_canvas_interactive_regions(args.image)
505
+ elements = detected if detected else [
506
+ {"x": 100, "y": 150, "width": 120, "height": 40, "label": "TradingView Zoom In", "tag": "canvas_control"},
507
+ {"x": 240, "y": 150, "width": 120, "height": 40, "label": "TradingView Zoom Out", "tag": "canvas_control"},
508
+ {"x": 500, "y": 300, "width": 200, "height": 50, "label": "Execute Trade Order", "tag": "shadow_button"},
509
+ {"x": 720, "y": 300, "width": 150, "height": 50, "label": "Cancel Order", "tag": "shadow_button"}
510
+ ]
511
+
512
+ marked_img, marks_map = helper.generate_som_overlay(args.image, elements, output_path=args.output)
513
+
514
+ resolved = None
515
+ target_query = args.label or args.component
516
+ if args.mark_id is not None or target_query:
517
+ resolved = helper.resolve_mark_coordinates(marks_map, target_mark_id=args.mark_id, target_label=target_query)
518
+
519
+ # Cache Save if requested or if component is specified and resolved
520
+ if resolved and (args.cache_save or args.component):
521
+ comp_name = args.component or resolved["label"]
522
+ helper.save_cached_coordinates(
523
+ component_name=comp_name,
524
+ image_source=args.image,
525
+ center_x=resolved["center_x"],
526
+ center_y=resolved["center_y"]
527
+ )
528
+
529
+ if args.json:
530
+ result = {
531
+ "marked_image": marked_img,
532
+ "total_marks": len(marks_map),
533
+ "marks": marks_map,
534
+ "resolved_target": resolved
535
+ }
536
+ print(json.dumps(result, indent=2))
537
+ else:
538
+ print(f"\n🎨 [SET-OF-MARKS GENERATED] Marked image: {marked_img}")
539
+ print(f" Candidate Visual Regions: {len(marks_map)}")
540
+ for mid, data in marks_map.items():
541
+ print(f" [{mid}] {data['label']} at ({data['center_x']}, {data['center_y']})")
542
+ if resolved:
543
+ print(f"\n🎯 [COORDINATE GROUNDED] Target '{resolved['label']}' -> Mark #{resolved['mark_id']}")
544
+ print(f" Playwright Mouse Action: {resolved['playwright_action']}\n")
545
+
546
+ return 0
547
+
548
+
549
+ if __name__ == "__main__":
550
+ sys.exit(main())