@teevee98/synapse-hub 0.0.0-stage → 3.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/agents/api-engineer.md +67 -0
- package/.agents/agents/failure-analyst.md +75 -0
- package/.agents/agents/live-explorer.md +70 -0
- package/.agents/agents/orchestrator.md +104 -0
- package/.agents/agents/perf-engineer.md +74 -0
- package/.agents/agents/pom-crafter.md +112 -0
- package/.agents/agents/quality-gatekeeper.md +87 -0
- package/.agents/agents/test-designer.md +73 -0
- package/.agents/agents/test-runner.md +67 -0
- package/.agents/rules/author-integrity-guard.md +33 -0
- package/.agents/rules/communication-style.md +18 -0
- package/.agents/rules/context-engineering.md +37 -0
- package/.agents/rules/git-push-protocol.md +33 -0
- package/.agents/rules/graphify.md +14 -0
- package/.agents/rules/lessons-learned.md +25 -0
- package/.agents/rules/mandatory-playwright-live.md +25 -0
- package/.agents/rules/mobile_appium_rules.md +43 -0
- package/.agents/rules/performance_testing_rules.md +35 -0
- package/.agents/rules/skill-precedence.md +29 -0
- package/.agents/rules/systematic-debugging.md +36 -0
- package/.agents/skills/actor-qa-swarm/SKILL.md +77 -0
- package/.agents/skills/actor-qa-swarm/examples/broker_sharding_sample.json +12 -0
- package/.agents/skills/agent-orchestration-multi-agent-optimize/SKILL.md +88 -0
- package/.agents/skills/agent-orchestration-multi-agent-optimize/examples/swarm_workload_config.json +15 -0
- package/.agents/skills/agentless-qa-repair/SKILL.md +69 -0
- package/.agents/skills/agentless-qa-repair/examples/healing_report_sample.json +17 -0
- package/.agents/skills/agentless-qa-repair/examples/three_phase_repair_workflow.md +40 -0
- package/.agents/skills/antigravity-test-sentinel/SKILL.md +63 -0
- package/.agents/skills/antigravity-test-sentinel/examples/dod_gate_policy.json +11 -0
- package/.agents/skills/antigravity-test-sentinel/examples/sprint_dod_enforcement.md +47 -0
- package/.agents/skills/api-network-mocking-interception/SKILL.md +142 -0
- package/.agents/skills/api-network-mocking-interception/examples/network_route_stub.json +9 -0
- package/.agents/skills/api-spec-to-e2e-matrix/SKILL.md +80 -0
- package/.agents/skills/api-spec-to-e2e-matrix/examples/sample_openapi.json +19 -0
- package/.agents/skills/architecture-decision-records/SKILL.md +64 -0
- package/.agents/skills/architecture-decision-records/examples/0001-record-architecture-decisions.md +12 -0
- package/.agents/skills/auth-session-and-cookie-manager/SKILL.md +75 -0
- package/.agents/skills/auth-session-and-cookie-manager/examples/multi_role_auth_matrix.md +70 -0
- package/.agents/skills/auth-session-and-cookie-manager/examples/storage_state_sample.json +11 -0
- package/.agents/skills/backend-load-and-stress-patterns/SKILL.md +85 -0
- package/.agents/skills/backend-load-and-stress-patterns/examples/k6_load_test_sample.js +13 -0
- package/.agents/skills/backend-load-and-stress-patterns/references/k6_load_template.js +82 -0
- package/.agents/skills/bats-testing-patterns/SKILL.md +100 -0
- package/.agents/skills/bats-testing-patterns/examples/cli_runner_test.bats +6 -0
- package/.agents/skills/bats-testing-patterns/resources/implementation-playbook.md +614 -0
- package/.agents/skills/clean-pom-crafter/SKILL.md +110 -0
- package/.agents/skills/clean-pom-crafter/examples/standard_test_case.groovy +28 -0
- package/.agents/skills/clean-pom-crafter/references/pom_patterns.groovy +138 -0
- package/.agents/skills/code-refactoring-refactor-clean/SKILL.md +64 -0
- package/.agents/skills/code-refactoring-refactor-clean/examples/refactored_page_sample.groovy +9 -0
- package/.agents/skills/code-refactoring-refactor-clean/examples/refactored_playwright_pom.ts +52 -0
- package/.agents/skills/code-refactoring-refactor-clean/resources/implementation-playbook.md +881 -0
- package/.agents/skills/consumer-driven-contract-testing/SKILL.md +147 -0
- package/.agents/skills/consumer-driven-contract-testing/examples/order_service_pact.json +45 -0
- package/.agents/skills/db-state-verifier-and-seeder/SKILL.md +123 -0
- package/.agents/skills/db-state-verifier-and-seeder/examples/db_fixture_seed.json +12 -0
- package/.agents/skills/e2e-testing-patterns/SKILL.md +103 -0
- package/.agents/skills/e2e-testing-patterns/examples/playwright_sharded_config.ts +12 -0
- package/.agents/skills/e2e-testing-patterns/resources/implementation-playbook.md +522 -0
- package/.agents/skills/event-stream-qa-orchestrator/SKILL.md +79 -0
- package/.agents/skills/event-stream-qa-orchestrator/examples/timeline_sample.jsonl +2 -0
- package/.agents/skills/failure-autopsy-and-healing/SKILL.md +66 -0
- package/.agents/skills/failure-autopsy-and-healing/examples/closed_loop_healing_walkthrough.md +48 -0
- package/.agents/skills/failure-autopsy-and-healing/examples/sample_junit_failure.xml +8 -0
- package/.agents/skills/failure-autopsy-and-healing/references/healing_matrix.md +64 -0
- package/.agents/skills/failure-autopsy-and-healing/references/qa_investigation_protocol.md +56 -0
- package/.agents/skills/failure-autopsy-and-healing/scripts/triage_failures.sh +83 -0
- package/.agents/skills/file-and-report-validator/SKILL.md +90 -0
- package/.agents/skills/file-and-report-validator/examples/sample_export_schema.json +14 -0
- package/.agents/skills/flaky-test-sentinel-and-quarantine/SKILL.md +114 -0
- package/.agents/skills/flaky-test-sentinel-and-quarantine/examples/quarantine_registry.json +11 -0
- package/.agents/skills/gherkin-to-suite/SKILL.md +103 -0
- package/.agents/skills/gherkin-to-suite/examples/checkout.feature +39 -0
- package/.agents/skills/gherkin-to-suite/examples/suite_report_sample.json +334 -0
- package/.agents/skills/github-actions-templates/SKILL.md +77 -0
- package/.agents/skills/github-actions-templates/examples/matrix_test_pipeline.yml +8 -0
- package/.agents/skills/graphify-impact-analyzer/SKILL.md +76 -0
- package/.agents/skills/graphify-impact-analyzer/examples/dependency_blast_radius.json +8 -0
- package/.agents/skills/katalon-cli-runner/SKILL.md +76 -0
- package/.agents/skills/katalon-cli-runner/examples/headless_kre_exec.sh +3 -0
- package/.agents/skills/katalon-cli-runner/references/groovy-patterns.md +209 -0
- package/.agents/skills/katalon-cli-runner/references/katalon-api.md +155 -0
- package/.agents/skills/katalon-cli-runner/references/runner_commands.md +67 -0
- package/.agents/skills/lam-script-generator/SKILL.md +123 -0
- package/.agents/skills/lam-script-generator/examples/user_story_sample.md +8 -0
- package/.agents/skills/lean-code-ladder/SKILL.md +94 -0
- package/.agents/skills/lean-code-ladder/examples/ladder_walkthrough.md +37 -0
- package/.agents/skills/lean-impact-scoreboard/SKILL.md +48 -0
- package/.agents/skills/lean-impact-scoreboard/examples/sample_benchmark_gain.md +20 -0
- package/.agents/skills/lean-impact-scoreboard/examples/sample_scoreboard.md +15 -0
- package/.agents/skills/meta-prompt-crafter/SKILL.md +107 -0
- package/.agents/skills/meta-prompt-crafter/examples/qa_prompt_template.md +35 -0
- package/.agents/skills/mipro-prompt-compiler/SKILL.md +59 -0
- package/.agents/skills/mipro-prompt-compiler/examples/compiled_module_sample.json +22 -0
- package/.agents/skills/mipro-prompt-compiler/examples/dspy_assertion_optimizer_case.md +46 -0
- package/.agents/skills/mobile-appium-som/SKILL.md +88 -0
- package/.agents/skills/mobile-appium-som/examples/BaseScreen.java +88 -0
- package/.agents/skills/mobile-appium-som/examples/LoginScreen.java +46 -0
- package/.agents/skills/mobile-appium-som/examples/TC_MobileLogin.java +43 -0
- package/.agents/skills/persistent-planning-files/SKILL.md +108 -0
- package/.agents/skills/persistent-planning-files/examples/task_plan_example.md +56 -0
- package/.agents/skills/playwright-live-interaction/SKILL.md +131 -0
- package/.agents/skills/playwright-live-interaction/examples/live_probe_script.js +5 -0
- package/.agents/skills/playwright-live-interaction/references/advanced_superpowers.md +160 -0
- package/.agents/skills/playwright-live-interaction/references/compact_accessibility_snapshots.md +75 -0
- package/.agents/skills/playwright-live-interaction/references/recipes.md +388 -0
- package/.agents/skills/playwright-live-interaction/references/swarm_orchestration.md +134 -0
- package/.agents/skills/playwright-webapp-testing/SKILL.md +93 -0
- package/.agents/skills/playwright-webapp-testing/examples/server_lifecycle_playwright.py +63 -0
- package/.agents/skills/qa-aci-interface/SKILL.md +67 -0
- package/.agents/skills/qa-aci-interface/examples/surgical_code_window_walkthrough.md +40 -0
- package/.agents/skills/qa-aci-interface/examples/surgical_edit_sample.json +12 -0
- package/.agents/skills/qa-memory-os/SKILL.md +80 -0
- package/.agents/skills/qa-memory-os/examples/archival_memory_sample.json +21 -0
- package/.agents/skills/rbt-manual-testing/SKILL.md +137 -0
- package/.agents/skills/rbt-manual-testing/examples/checkout_rbt_manual_suite.md +25 -0
- package/.agents/skills/rbt-manual-testing/references/field_validation_checklist.md +29 -0
- package/.agents/skills/repo-quality-audit/SKILL.md +68 -0
- package/.agents/skills/repo-quality-audit/examples/sample_audit_walkthrough.md +15 -0
- package/.agents/skills/repo-quality-audit/examples/sample_repo_audit.md +33 -0
- package/.agents/skills/review-spec/SKILL.md +60 -0
- package/.agents/skills/review-spec/examples/pocock_grilling_decision_rubric.md +40 -0
- package/.agents/skills/review-spec/examples/sample_spec_review.md +33 -0
- package/.agents/skills/security-dast-sentinel/SKILL.md +119 -0
- package/.agents/skills/security-dast-sentinel/examples/dast_scan_config.json +10 -0
- package/.agents/skills/security-dast-sentinel/references/AI-AND-LLM.md +113 -0
- package/.agents/skills/security-dast-sentinel/references/ATTACK-CLASSES.md +89 -0
- package/.agents/skills/security-dast-sentinel/references/dast_recipes.ts +33 -0
- package/.agents/skills/self-healing-locator/SKILL.md +86 -0
- package/.agents/skills/self-healing-locator/examples/aom_diff_sample.json +16 -0
- package/.agents/skills/senior-code-review/SKILL.md +108 -0
- package/.agents/skills/senior-code-review/examples/sample_review_walkthrough.md +29 -0
- package/.agents/skills/skillspector-security-guard/SKILL.md +57 -0
- package/.agents/skills/skillspector-security-guard/examples/audit_threat_vectors_report.json +35 -0
- package/.agents/skills/skillspector-security-guard/examples/safe_mcp_config.json +12 -0
- package/.agents/skills/surgical-coding-principles/SKILL.md +89 -0
- package/.agents/skills/surgical-coding-principles/examples/surgical_vs_scope_creep.md +46 -0
- package/.agents/skills/synthetic-test-data-generator/SKILL.md +135 -0
- package/.agents/skills/synthetic-test-data-generator/examples/idempotent_data_recipe.json +9 -0
- package/.agents/skills/systematic-debugging/SKILL.md +100 -0
- package/.agents/skills/systematic-debugging/examples/four_phase_case.md +44 -0
- package/.agents/skills/technical-debt-harvester/SKILL.md +70 -0
- package/.agents/skills/technical-debt-harvester/examples/sample_debt_ledger.md +25 -0
- package/.agents/skills/technical-debt-harvester/examples/sample_ledger_output.md +20 -0
- package/.agents/skills/test-driven-development/SKILL.md +114 -0
- package/.agents/skills/test-driven-development/examples/red_green_refactor_qa.md +78 -0
- package/.agents/skills/token-and-context-optimizer/SKILL.md +70 -0
- package/.agents/skills/token-and-context-optimizer/examples/distilled_log_sample.txt +3 -0
- package/.agents/skills/token-and-context-optimizer/references/context_engineering_guide.md +49 -0
- package/.agents/skills/tot-exploratory-testing/SKILL.md +59 -0
- package/.agents/skills/tot-exploratory-testing/examples/payment_gateway_tot_tree.json +62 -0
- package/.agents/skills/tot-exploratory-testing/examples/tot_graph_sample.json +35 -0
- package/.agents/skills/verification-before-completion/SKILL.md +100 -0
- package/.agents/skills/verification-before-completion/examples/evidence_log_example.md +46 -0
- package/.agents/skills/visual-regression-sentinel/SKILL.md +82 -0
- package/.agents/skills/visual-regression-sentinel/examples/visual_mask_rules.json +9 -0
- package/.agents/skills/wcag-audit-patterns/SKILL.md +77 -0
- package/.agents/skills/wcag-audit-patterns/examples/wcag_compliance_checklist.json +10 -0
- package/.agents/skills/wcag-audit-patterns/resources/implementation-playbook.md +541 -0
- package/.agents/skills/web-agent-gym-benchmarking/SKILL.md +62 -0
- package/.agents/skills/web-agent-gym-benchmarking/examples/benchmark_report_sample.json +22 -0
- package/.agents/skills/web-agent-gym-benchmarking/examples/enterprise_tasks_suite.json +38 -0
- package/.agents/skills/web-vitals-and-performance-profiler/SKILL.md +65 -0
- package/.agents/skills/web-vitals-and-performance-profiler/examples/cwv_budget_policy.json +6 -0
- package/.agents/skills/web-vitals-and-performance-profiler/examples/playwright_cwv_profiler_fixture.ts +118 -0
- package/.agents/templates/completion_report.md +57 -0
- package/.agents/workflows/analyze_requirement_document.md +18 -0
- package/.agents/workflows/export_testcases_excel.md +32 -0
- package/.agents/workflows/generate_combinatorial_test_data.md +28 -0
- package/.agents/workflows/generate_cross_module_test_plan.md +37 -0
- package/.agents/workflows/generate_manual_testcases_rbt.md +56 -0
- package/.agents/workflows/generate_requirements_from_website.md +25 -0
- package/.agents/workflows/generate_testcases_from_requirements.md +21 -0
- package/.agents/workflows/graphify.md +10 -0
- package/.agents/workflows/test-impact.md +15 -0
- package/.agents/workflows/verify.md +34 -0
- package/.cursor/rules/agents.mdc +16 -0
- package/.cursor/rules/language.mdc +7 -0
- package/.cursor/rules/synapse-qa.mdc +24 -0
- package/.cursorrules +27 -0
- package/.dockerignore +53 -0
- package/.pre-commit-hooks.yaml +14 -0
- package/.windsurfrules +26 -0
- package/AGENTS.md +86 -0
- package/CLAUDE.md +54 -0
- package/Dockerfile +81 -0
- package/LICENSE +52 -0
- package/NOTICE +264 -0
- package/README.en.md +172 -0
- package/README.md +171 -2
- package/action.yml +51 -0
- package/agentskills.json +16 -0
- package/bin/agent-qa.js +138 -0
- package/docker-compose.yml +74 -0
- package/docs/ARCHITECTURE.en.md +197 -0
- package/docs/ARCHITECTURE.md +198 -0
- package/docs/CI_CD_INTEGRATION.en.md +170 -0
- package/docs/CI_CD_INTEGRATION.md +170 -0
- package/docs/CLI_REFERENCE.en.md +175 -0
- package/docs/CLI_REFERENCE.md +177 -0
- package/docs/CONTRIBUTING.en.md +49 -0
- package/docs/CONTRIBUTING.md +49 -0
- package/docs/PERFORMANCE_AUDIT_REPORT.md +784 -0
- package/docs/PLUGIN_DEVELOPMENT_GUIDE.md +127 -0
- package/docs/QUALITY_GATES.en.md +121 -0
- package/docs/QUALITY_GATES.md +121 -0
- package/docs/TOKEN_QUOTA_PLAYBOOK.md +41 -0
- package/docs/TOOL_CATALOG.md +129 -0
- package/docs/adopted-ecosystem-tools.md +96 -0
- package/docs/research/EXTERNAL_AGENT_SKILLS_SURVEY.md +1062 -0
- package/docs/research/agent_skill_training_analysis.md +1359 -0
- package/docs/schemas/synapse.config.schema.json +87 -0
- package/package.json +94 -4
- package/plugin.json +31 -0
- package/scripts/generate_skill_examples.py +244 -0
- package/scripts/install.ps1 +112 -0
- package/scripts/install_global_plugin.ps1 +94 -0
- package/scripts/install_global_plugin.sh +49 -0
- package/scripts/lint_skills.ps1 +57 -0
- package/scripts/package_dist.py +337 -0
- package/scripts/setup.sh +59 -0
- package/scripts/update_hub.ps1 +19 -0
- package/scripts/update_hub.sh +15 -0
- package/skills_manifest.json +854 -0
- package/synapse.config.json +32 -0
- package/templates/PROJECT_AI_GUIDE_TEMPLATE.md +111 -0
- package/templates/PROJECT_STATE_TEMPLATE.md +20 -0
- package/templates/ai_bridges/CLAUDE.md +54 -0
- package/templates/ai_bridges/copilot-instructions.md +20 -0
- package/templates/ai_bridges/cursor_synapse_qa.mdc +24 -0
- package/templates/ai_bridges/windsurfrules +26 -0
- package/templates/ci/github-actions-synapse.yml +95 -0
- package/templates/ci/gitlab-ci-synapse.yml +16 -0
- package/templates/ui/index.html +1069 -0
- package/templates/welcome.html.tmpl +2079 -0
- package/tools/.infer_baseline.json +30 -0
- package/tools/TOOLS_RUNBOOK.md +1553 -0
- package/tools/TOOL_CATALOG.json +1769 -0
- package/tools/__init__.py +43 -0
- package/tools/_openapi_parser.py +158 -0
- package/tools/agent_canary.py +525 -0
- package/tools/agentless_patcher.py +417 -0
- package/tools/analyze_project.py +258 -0
- package/tools/api_drift_sentinel.py +208 -0
- package/tools/api_fuzzer.py +762 -0
- package/tools/author_integrity.py +184 -0
- package/tools/auto_heal.py +271 -0
- package/tools/auto_heal_reflexion.py +532 -0
- package/tools/autogen_qa_broker.py +844 -0
- package/tools/benchmark_swarm_context.py +394 -0
- package/tools/browser_helpers.js +746 -0
- package/tools/browsergym_eval.py +236 -0
- package/tools/bug_reporter.py +252 -0
- package/tools/bump_version.py +423 -0
- package/tools/cache_resolver.py +1214 -0
- package/tools/check_env.py +426 -0
- package/tools/check_version_sync.py +256 -0
- package/tools/ci_healing_hook.py +126 -0
- package/tools/cli.py +1598 -0
- package/tools/config_decoupler.py +253 -0
- package/tools/contract_validator.py +141 -0
- package/tools/core/__init__.py +70 -0
- package/tools/core/adapters.py +431 -0
- package/tools/core/circuit_breaker.py +263 -0
- package/tools/core/cli_runtime.py +66 -0
- package/tools/core/file_lock.py +159 -0
- package/tools/core/i18n.py +394 -0
- package/tools/core/memory_bus.py +905 -0
- package/tools/core/process_supervisor.py +332 -0
- package/tools/core/telemetry.py +329 -0
- package/tools/count_tokens.py +1267 -0
- package/tools/create_mock_docx.py +104 -0
- package/tools/dashboard_generator.py +2419 -0
- package/tools/db_sandbox.py +675 -0
- package/tools/debt_harvester.py +237 -0
- package/tools/distill_log.py +161 -0
- package/tools/dspy_qa_signatures.py +356 -0
- package/tools/e2e_flow_prober.py +132 -0
- package/tools/export_postman_collection.py +945 -0
- package/tools/flaky_detector.py +389 -0
- package/tools/fsm_enforcer.py +450 -0
- package/tools/fsm_middleware.py +401 -0
- package/tools/fuzz_payloads.py +197 -0
- package/tools/generate_allure_report.py +88 -0
- package/tools/generate_api_matrix.py +193 -0
- package/tools/generate_jmx_plan.py +1322 -0
- package/tools/generate_sbom.py +252 -0
- package/tools/generate_tc_from_story.py +203 -0
- package/tools/generate_test_reports.py +133 -0
- package/tools/generate_tool_catalog.py +482 -0
- package/tools/gherkin_suite.py +758 -0
- package/tools/git_pre_commit_guard.py +298 -0
- package/tools/goldfish_review.py +366 -0
- package/tools/heal/__init__.py +28 -0
- package/tools/heal_pr_automator.py +220 -0
- package/tools/heuristic_locator_matcher.py +351 -0
- package/tools/hub_publisher.py +623 -0
- package/tools/humanize_linter.py +438 -0
- package/tools/infer_linter.py +977 -0
- package/tools/init_config.py +307 -0
- package/tools/interview_story.py +116 -0
- package/tools/jira_sync.py +566 -0
- package/tools/jtl_analyzer.py +474 -0
- package/tools/lint/__init__.py +30 -0
- package/tools/lint_ai_docs.py +374 -0
- package/tools/lookup_tc.py +165 -0
- package/tools/lookup_xpath.py +167 -0
- package/tools/matrix_sharder.py +148 -0
- package/tools/mcp/__init__.py +18 -0
- package/tools/mcp_docx_server.ps1 +13 -0
- package/tools/mcp_docx_server.py +851 -0
- package/tools/mcp_excel_server.ps1 +13 -0
- package/tools/mcp_excel_server.py +441 -0
- package/tools/mcp_server.py +724 -0
- package/tools/md_to_xlsx.py +327 -0
- package/tools/micro_mutator.py +887 -0
- package/tools/mipro_assertion_tuner.py +210 -0
- package/tools/network_mock_shield.py +311 -0
- package/tools/newman_analyzer.py +384 -0
- package/tools/omni_emitter.py +579 -0
- package/tools/omni_parser.py +450 -0
- package/tools/omni_transpiler.py +168 -0
- package/tools/openxml_guard.py +438 -0
- package/tools/output_formatter.py +182 -0
- package/tools/package_release.py +325 -0
- package/tools/perf/__init__.py +24 -0
- package/tools/pinpoint_failure.py +190 -0
- package/tools/project_state.py +171 -0
- package/tools/qa_event_stream.py +358 -0
- package/tools/repo_map.py +346 -0
- package/tools/report/__init__.py +32 -0
- package/tools/reversible_sandbox.py +344 -0
- package/tools/safety_firewall.py +437 -0
- package/tools/security_coverage_ledger.py +538 -0
- package/tools/security_scanner.py +229 -0
- package/tools/senior_code_review.py +393 -0
- package/tools/setup_project.py +770 -0
- package/tools/skill_collision_guard.py +304 -0
- package/tools/skill_profiles.py +217 -0
- package/tools/skill_vault.py +481 -0
- package/tools/skillspector.py +1278 -0
- package/tools/smart_visual_masker.py +132 -0
- package/tools/spec_drift_sentinel.py +438 -0
- package/tools/swe_qa_aci.py +369 -0
- package/tools/synapse_observer.py +485 -0
- package/tools/synapse_qa_bench.py +190 -0
- package/tools/sync_agent_rules.py +274 -0
- package/tools/telemetry_correlator.py +397 -0
- package/tools/template_excel_exporter.py +514 -0
- package/tools/templates/jmeter-distributed/README.md +54 -0
- package/tools/templates/jmeter-distributed/docker-compose.yml +83 -0
- package/tools/test_impact.py +419 -0
- package/tools/test_memory_os.py +462 -0
- package/tools/textgrad_mutation_optimizer.py +237 -0
- package/tools/tms_sync_bridge.py +834 -0
- package/tools/tot_exploratory_tester.py +239 -0
- package/tools/trace_data_flow.py +240 -0
- package/tools/traffic_session_converter.py +256 -0
- package/tools/trend_analytics.py +314 -0
- package/tools/ui_server.py +401 -0
- package/tools/universal_ir.py +488 -0
- package/tools/update_sentinel.py +383 -0
- package/tools/validate_groovy_scripts.py +173 -0
- package/tools/verify_all.py +1259 -0
- package/tools/visual_diff_canvas.py +143 -0
- package/tools/visual_som_helper.py +550 -0
- package/tools/watch_mode.py +308 -0
- package/tools/wcag_audit.py +239 -0
- package/tools/welcome_page.py +290 -0
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
"""
|
|
4
|
+
Enterprise Canvas Visual Diff & Regression Checker
|
|
5
|
+
===================================================
|
|
6
|
+
Provides visual regression AI testing capabilities for HTML5 Canvas, SVG,
|
|
7
|
+
and DOM-less charts (e.g. DataMiningPage / GIS / Analytics).
|
|
8
|
+
|
|
9
|
+
Features:
|
|
10
|
+
1. compare: Pixel-by-pixel Canvas comparison against Golden Baseline.
|
|
11
|
+
Computes diff ratio and generates highlight overlay diff artifacts.
|
|
12
|
+
2. check-blank: Detects white-screen or empty non-rendered Canvas regressions.
|
|
13
|
+
3. update-baseline: Captures new Golden Baseline upon design updates.
|
|
14
|
+
|
|
15
|
+
Usage:
|
|
16
|
+
python3 tools/visual_diff_canvas.py compare baseline.png current.png --diff diff.png --threshold 0.02
|
|
17
|
+
python3 tools/visual_diff_canvas.py check-blank current.png
|
|
18
|
+
"""
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import os
|
|
22
|
+
import sys
|
|
23
|
+
import argparse
|
|
24
|
+
|
|
25
|
+
try:
|
|
26
|
+
from PIL import Image, ImageChops, ImageStat
|
|
27
|
+
except ImportError:
|
|
28
|
+
print("❌ Error: Pillow library not installed. Please install: pip install Pillow")
|
|
29
|
+
sys.exit(1)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def is_canvas_blank(image_path: str, std_dev_threshold: float = 2.0) -> tuple[bool, float]:
|
|
33
|
+
"""
|
|
34
|
+
Checks if a canvas screenshot is blank, empty, or single-color non-rendered.
|
|
35
|
+
Returns (is_blank, std_dev).
|
|
36
|
+
"""
|
|
37
|
+
with Image.open(image_path) as img:
|
|
38
|
+
gray = img.convert("L")
|
|
39
|
+
stat = ImageStat.Stat(gray)
|
|
40
|
+
std_dev = stat.stddev[0]
|
|
41
|
+
# Low standard deviation indicates uniform solid/blank canvas
|
|
42
|
+
return (std_dev < std_dev_threshold, std_dev)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def compare_canvas_images(
|
|
46
|
+
baseline_path: str,
|
|
47
|
+
current_path: str,
|
|
48
|
+
diff_output_path: str | None = None,
|
|
49
|
+
threshold: float = 0.01
|
|
50
|
+
) -> tuple[bool, float, int]:
|
|
51
|
+
"""
|
|
52
|
+
Compares two canvas screenshots pixel-by-pixel.
|
|
53
|
+
Returns (is_match, diff_ratio, diff_pixels).
|
|
54
|
+
threshold: Allowed diff ratio (e.g. 0.01 = 1% tolerance)
|
|
55
|
+
"""
|
|
56
|
+
with Image.open(baseline_path) as img_base, Image.open(current_path) as img_curr:
|
|
57
|
+
# Normalize dimensions and color mode
|
|
58
|
+
curr_image: Image.Image = img_curr
|
|
59
|
+
if img_base.size != img_curr.size:
|
|
60
|
+
curr_image = img_curr.resize(img_base.size, Image.Resampling.LANCZOS)
|
|
61
|
+
|
|
62
|
+
base_rgb = img_base.convert("RGB")
|
|
63
|
+
curr_rgb = curr_image.convert("RGB")
|
|
64
|
+
|
|
65
|
+
# C-level accelerated pixel difference via Pillow vectorization
|
|
66
|
+
diff = ImageChops.difference(base_rgb, curr_rgb)
|
|
67
|
+
gray_diff = diff.convert("L")
|
|
68
|
+
total_pixels = img_base.width * img_base.height
|
|
69
|
+
|
|
70
|
+
# Histogram-based count of divergent pixels (delta > 10, < 0.03s for 1920x1080)
|
|
71
|
+
hist = gray_diff.histogram()
|
|
72
|
+
diff_count = sum(hist[11:])
|
|
73
|
+
|
|
74
|
+
diff_ratio = diff_count / total_pixels
|
|
75
|
+
is_match = diff_ratio <= threshold
|
|
76
|
+
|
|
77
|
+
if diff_output_path and not is_match:
|
|
78
|
+
os.makedirs(os.path.dirname(os.path.abspath(diff_output_path)), exist_ok=True)
|
|
79
|
+
red_layer = Image.new("RGBA", img_base.size, (255, 0, 0, 200))
|
|
80
|
+
alpha_mask = gray_diff.point(lambda p: 200 if p > 10 else 0)
|
|
81
|
+
red_layer.putalpha(alpha_mask)
|
|
82
|
+
annotated = curr_rgb.convert("RGBA")
|
|
83
|
+
annotated.paste(red_layer, (0, 0), red_layer)
|
|
84
|
+
annotated.save(diff_output_path)
|
|
85
|
+
|
|
86
|
+
return (is_match, diff_ratio, diff_count)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def main():
|
|
90
|
+
parser = argparse.ArgumentParser(description="Enterprise Canvas Visual Diff & Regression Checker")
|
|
91
|
+
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
92
|
+
|
|
93
|
+
# Command: compare
|
|
94
|
+
compare_p = subparsers.add_parser("compare", help="Compare current canvas screenshot against Golden Baseline")
|
|
95
|
+
compare_p.add_argument("baseline", help="Path to Golden Baseline image")
|
|
96
|
+
compare_p.add_argument("current", help="Path to current runtime screenshot")
|
|
97
|
+
compare_p.add_argument("--diff", default=None, help="Output path for annotated diff highlight image")
|
|
98
|
+
compare_p.add_argument("--threshold", type=float, default=0.01, help="Max allowed divergence ratio (default: 0.01 = 1%%)")
|
|
99
|
+
|
|
100
|
+
# Command: check-blank
|
|
101
|
+
blank_p = subparsers.add_parser("check-blank", help="Check whether canvas is blank or unrendered")
|
|
102
|
+
blank_p.add_argument("current", help="Path to canvas screenshot to verify")
|
|
103
|
+
blank_p.add_argument("--threshold", type=float, default=2.0, help="Min standard deviation for valid render (default: 2.0)")
|
|
104
|
+
|
|
105
|
+
args = parser.parse_args()
|
|
106
|
+
|
|
107
|
+
if args.command == "compare":
|
|
108
|
+
if not os.path.exists(args.baseline):
|
|
109
|
+
print(f"❌ Baseline file not found: {args.baseline}")
|
|
110
|
+
sys.exit(2)
|
|
111
|
+
if not os.path.exists(args.current):
|
|
112
|
+
print(f"❌ Current screenshot file not found: {args.current}")
|
|
113
|
+
sys.exit(2)
|
|
114
|
+
|
|
115
|
+
is_match, diff_ratio, diff_count = compare_canvas_images(
|
|
116
|
+
args.baseline, args.current, args.diff, args.threshold
|
|
117
|
+
)
|
|
118
|
+
pct = diff_ratio * 100
|
|
119
|
+
if is_match:
|
|
120
|
+
print(f"✅ PASS: Visual canvas matches Baseline (Diff: {pct:.2f}%, {diff_count} px <= {args.threshold*100:.1f}%)")
|
|
121
|
+
sys.exit(0)
|
|
122
|
+
else:
|
|
123
|
+
print(f"❌ FAIL: Visual canvas exceeded divergence threshold (Diff: {pct:.2f}%, {diff_count} px > {args.threshold*100:.1f}%)")
|
|
124
|
+
if args.diff:
|
|
125
|
+
print(f" 📸 Annotated diff artifact saved to: {args.diff}")
|
|
126
|
+
sys.exit(1)
|
|
127
|
+
|
|
128
|
+
elif args.command == "check-blank":
|
|
129
|
+
if not os.path.exists(args.current):
|
|
130
|
+
print(f"❌ Screenshot file not found: {args.current}")
|
|
131
|
+
sys.exit(2)
|
|
132
|
+
|
|
133
|
+
blank, std_dev = is_canvas_blank(args.current, args.threshold)
|
|
134
|
+
if blank:
|
|
135
|
+
print(f"❌ WARNING: Canvas is blank / unrendered (Standard Deviation: {std_dev:.2f} < {args.threshold})")
|
|
136
|
+
sys.exit(1)
|
|
137
|
+
else:
|
|
138
|
+
print(f"✅ PASS: Canvas renders valid graphical content (Standard Deviation: {std_dev:.2f} >= {args.threshold})")
|
|
139
|
+
sys.exit(0)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
if __name__ == "__main__":
|
|
143
|
+
main()
|
|
@@ -0,0 +1,550 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
Enterprise Vision-Grounded Fallback & Set-of-Marks (SoM) Engine
|
|
4
|
+
--------------------------------------------------------------
|
|
5
|
+
Provides multimodal coordinate grounding when standard DOM locators fail
|
|
6
|
+
(HTML5 Canvas, WebGL financial charts, deeply nested Shadow DOM, cross-origin iframes).
|
|
7
|
+
|
|
8
|
+
Features:
|
|
9
|
+
1. Set-of-Marks (SoM) Overlay Generation:
|
|
10
|
+
- Detects interactive visual regions (or accepts DOM bounding boxes).
|
|
11
|
+
- Renders high-contrast colored bounding boxes with numeric IDs (1, 2, 3...) on screenshots.
|
|
12
|
+
2. Coordinate Resolution & Physical Mouse Dispatch:
|
|
13
|
+
- Maps Mark IDs and visual targets to exact center coordinates (x, y).
|
|
14
|
+
- Generates direct Playwright physical mouse actions: await page.mouse.click(x, y).
|
|
15
|
+
3. Visual Perceptual Caching & Token Economy:
|
|
16
|
+
- Generates difference hash (dHash) fingerprints for visual components.
|
|
17
|
+
- Caches coordinates in SQLite (memory_cache.db). If visual similarity > 98%,
|
|
18
|
+
instantly retrieves cached coordinates without expensive Vision LLM calls (Zero-Token).
|
|
19
|
+
4. Compact Vision Prompt Formatter:
|
|
20
|
+
- Formats token-efficient briefs for Multimodal Vision Models (Gemini / Claude / GPT-4o).
|
|
21
|
+
"""
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import os
|
|
25
|
+
import sys
|
|
26
|
+
import json
|
|
27
|
+
import sqlite3
|
|
28
|
+
import argparse
|
|
29
|
+
from typing import Dict, Any, List, Optional, Tuple
|
|
30
|
+
try:
|
|
31
|
+
from PIL import Image, ImageDraw, ImageFont, ImageFilter
|
|
32
|
+
HAS_PIL = True
|
|
33
|
+
except ImportError:
|
|
34
|
+
Image = None # type: ignore[assignment]
|
|
35
|
+
ImageDraw = None # type: ignore[assignment]
|
|
36
|
+
ImageFont = None # type: ignore[assignment]
|
|
37
|
+
ImageFilter = None # type: ignore[assignment]
|
|
38
|
+
HAS_PIL = False
|
|
39
|
+
|
|
40
|
+
ImageSourceType = Any
|
|
41
|
+
|
|
42
|
+
# Enforce UTF-8 stdout encoding for Windows console compatibility
|
|
43
|
+
if hasattr(sys.stdout, 'reconfigure'):
|
|
44
|
+
try:
|
|
45
|
+
sys.stdout.reconfigure(encoding='utf-8')
|
|
46
|
+
sys.stderr.reconfigure(encoding='utf-8')
|
|
47
|
+
except Exception:
|
|
48
|
+
pass
|
|
49
|
+
|
|
50
|
+
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
51
|
+
QA_HUB_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, ".."))
|
|
52
|
+
DEFAULT_CACHE_DB = os.path.expanduser("~/.agent_memory/global_cache.db")
|
|
53
|
+
|
|
54
|
+
MARK_COLORS = [
|
|
55
|
+
"#FF0055", # Vibrant Red/Pink
|
|
56
|
+
"#00CCFF", # Cyan
|
|
57
|
+
"#00FF66", # Neon Green
|
|
58
|
+
"#FFCC00", # Bright Yellow
|
|
59
|
+
"#9933FF", # Purple
|
|
60
|
+
"#FF6600", # Orange
|
|
61
|
+
]
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def compute_visual_hash(image_source: ImageSourceType, bbox: Optional[List[int]] = None, hash_size: int = 8) -> str:
|
|
65
|
+
"""
|
|
66
|
+
Computes a 64-bit difference hash (dHash) for an image or cropped component.
|
|
67
|
+
Fast, deterministic, and 100% offline.
|
|
68
|
+
"""
|
|
69
|
+
if not HAS_PIL or Image is None:
|
|
70
|
+
raise ImportError("Pillow is required for visual perceptual operations (pip install Pillow).")
|
|
71
|
+
if isinstance(image_source, str):
|
|
72
|
+
if not os.path.exists(image_source):
|
|
73
|
+
raise FileNotFoundError(f"Image not found: {image_source}")
|
|
74
|
+
img: Any = Image.open(image_source)
|
|
75
|
+
else:
|
|
76
|
+
img = image_source
|
|
77
|
+
|
|
78
|
+
if bbox and len(bbox) == 4:
|
|
79
|
+
img = img.crop(tuple(bbox))
|
|
80
|
+
|
|
81
|
+
# Resize to (hash_size + 1, hash_size) in grayscale
|
|
82
|
+
resized = img.convert("L").resize((hash_size + 1, hash_size), Image.Resampling.LANCZOS)
|
|
83
|
+
pixels = list(resized.tobytes())
|
|
84
|
+
|
|
85
|
+
diff = []
|
|
86
|
+
for row in range(hash_size):
|
|
87
|
+
row_start = row * (hash_size + 1)
|
|
88
|
+
for col in range(hash_size):
|
|
89
|
+
left = pixels[row_start + col]
|
|
90
|
+
right = pixels[row_start + col + 1]
|
|
91
|
+
diff.append("1" if left > right else "0")
|
|
92
|
+
|
|
93
|
+
return "".join(diff)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def calculate_visual_similarity(hash1: str, hash2: str) -> float:
|
|
97
|
+
"""Calculates normalized Hamming similarity between two binary hashes (0.0 to 1.0)."""
|
|
98
|
+
if not hash1 or not hash2 or len(hash1) != len(hash2):
|
|
99
|
+
return 0.0
|
|
100
|
+
hamming_dist = sum(c1 != c2 for c1, c2 in zip(hash1, hash2))
|
|
101
|
+
return round(1.0 - (hamming_dist / len(hash1)), 4)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class VisualSoMHelper:
|
|
105
|
+
"""Manages visual grounding, bounding box marking, visual caching, and coordinate dispatch."""
|
|
106
|
+
|
|
107
|
+
def __init__(self, viewport_width: int = 1920, viewport_height: int = 1080, db_path: Optional[str] = None):
|
|
108
|
+
self.viewport_width = viewport_width
|
|
109
|
+
self.viewport_height = viewport_height
|
|
110
|
+
self.db_path = db_path or DEFAULT_CACHE_DB
|
|
111
|
+
self.init_cache_table()
|
|
112
|
+
|
|
113
|
+
def init_cache_table(self):
|
|
114
|
+
"""Initializes the visual_som_cache SQLite table if not already present."""
|
|
115
|
+
try:
|
|
116
|
+
conn = sqlite3.connect(self.db_path, timeout=10.0)
|
|
117
|
+
try:
|
|
118
|
+
conn.execute("""
|
|
119
|
+
CREATE TABLE IF NOT EXISTS visual_som_cache (
|
|
120
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
121
|
+
component_name TEXT NOT NULL,
|
|
122
|
+
visual_hash TEXT NOT NULL,
|
|
123
|
+
center_x INTEGER NOT NULL,
|
|
124
|
+
center_y INTEGER NOT NULL,
|
|
125
|
+
rel_x REAL DEFAULT 0.0,
|
|
126
|
+
rel_y REAL DEFAULT 0.0,
|
|
127
|
+
hit_count INTEGER DEFAULT 0,
|
|
128
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
129
|
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
130
|
+
UNIQUE(component_name, visual_hash)
|
|
131
|
+
);
|
|
132
|
+
""")
|
|
133
|
+
conn.execute("CREATE INDEX IF NOT EXISTS idx_som_component ON visual_som_cache (component_name);")
|
|
134
|
+
conn.commit()
|
|
135
|
+
finally:
|
|
136
|
+
conn.close()
|
|
137
|
+
except (sqlite3.Error, OSError):
|
|
138
|
+
self.db_path = ":memory:"
|
|
139
|
+
|
|
140
|
+
def lookup_cached_coordinates(
|
|
141
|
+
self,
|
|
142
|
+
component_name: str,
|
|
143
|
+
image_source: ImageSourceType,
|
|
144
|
+
bbox: Optional[List[int]] = None,
|
|
145
|
+
threshold: float = 0.98,
|
|
146
|
+
db_path: Optional[str] = None
|
|
147
|
+
) -> Optional[Dict[str, Any]]:
|
|
148
|
+
"""
|
|
149
|
+
Looks up cached coordinates for a component if visual similarity exceeds threshold (default 98%).
|
|
150
|
+
Returns resolved coordinate data if cache hit, None if cache miss.
|
|
151
|
+
"""
|
|
152
|
+
current_hash = compute_visual_hash(image_source, bbox=bbox)
|
|
153
|
+
db = db_path or self.db_path
|
|
154
|
+
|
|
155
|
+
try:
|
|
156
|
+
conn = sqlite3.connect(db, timeout=10.0)
|
|
157
|
+
try:
|
|
158
|
+
conn.row_factory = sqlite3.Row
|
|
159
|
+
cursor = conn.cursor()
|
|
160
|
+
cursor.execute(
|
|
161
|
+
"SELECT id, component_name, visual_hash, center_x, center_y, hit_count FROM visual_som_cache WHERE component_name = ?",
|
|
162
|
+
(component_name,)
|
|
163
|
+
)
|
|
164
|
+
rows = cursor.fetchall()
|
|
165
|
+
|
|
166
|
+
best_match = None
|
|
167
|
+
best_sim = -1.0
|
|
168
|
+
|
|
169
|
+
for row in rows:
|
|
170
|
+
cached_hash = row["visual_hash"]
|
|
171
|
+
sim = calculate_visual_similarity(current_hash, cached_hash)
|
|
172
|
+
if sim >= threshold and sim > best_sim:
|
|
173
|
+
best_sim = sim
|
|
174
|
+
best_match = row
|
|
175
|
+
|
|
176
|
+
if best_match:
|
|
177
|
+
match_id = best_match["id"]
|
|
178
|
+
cursor.execute(
|
|
179
|
+
"UPDATE visual_som_cache SET hit_count = hit_count + 1, updated_at = CURRENT_TIMESTAMP WHERE id = ?",
|
|
180
|
+
(match_id,)
|
|
181
|
+
)
|
|
182
|
+
conn.commit()
|
|
183
|
+
|
|
184
|
+
cx = best_match["center_x"]
|
|
185
|
+
cy = best_match["center_y"]
|
|
186
|
+
return {
|
|
187
|
+
"component_name": component_name,
|
|
188
|
+
"center_x": cx,
|
|
189
|
+
"center_y": cy,
|
|
190
|
+
"visual_similarity": best_sim,
|
|
191
|
+
"visual_hash": current_hash,
|
|
192
|
+
"cached": True,
|
|
193
|
+
"hit_count": best_match["hit_count"] + 1,
|
|
194
|
+
"playwright_action": f"await page.mouse.click({cx}, {cy});"
|
|
195
|
+
}
|
|
196
|
+
finally:
|
|
197
|
+
conn.close()
|
|
198
|
+
except (sqlite3.Error, OSError):
|
|
199
|
+
self.db_path = ":memory:"
|
|
200
|
+
|
|
201
|
+
return None
|
|
202
|
+
|
|
203
|
+
def save_cached_coordinates(
|
|
204
|
+
self,
|
|
205
|
+
component_name: str,
|
|
206
|
+
image_source: ImageSourceType,
|
|
207
|
+
center_x: int,
|
|
208
|
+
center_y: int,
|
|
209
|
+
bbox: Optional[List[int]] = None,
|
|
210
|
+
db_path: Optional[str] = None
|
|
211
|
+
) -> str:
|
|
212
|
+
"""
|
|
213
|
+
Saves resolved coordinates and visual hash into SQLite cache.
|
|
214
|
+
"""
|
|
215
|
+
visual_hash = compute_visual_hash(image_source, bbox=bbox)
|
|
216
|
+
db = db_path or self.db_path
|
|
217
|
+
|
|
218
|
+
try:
|
|
219
|
+
conn = sqlite3.connect(db, timeout=10.0)
|
|
220
|
+
try:
|
|
221
|
+
conn.execute("""
|
|
222
|
+
INSERT INTO visual_som_cache (component_name, visual_hash, center_x, center_y, hit_count)
|
|
223
|
+
VALUES (?, ?, ?, ?, 1)
|
|
224
|
+
ON CONFLICT(component_name, visual_hash) DO UPDATE SET
|
|
225
|
+
center_x = excluded.center_x,
|
|
226
|
+
center_y = excluded.center_y,
|
|
227
|
+
hit_count = visual_som_cache.hit_count + 1,
|
|
228
|
+
updated_at = CURRENT_TIMESTAMP;
|
|
229
|
+
""", (component_name, visual_hash, center_x, center_y))
|
|
230
|
+
conn.commit()
|
|
231
|
+
finally:
|
|
232
|
+
conn.close()
|
|
233
|
+
except (sqlite3.Error, OSError):
|
|
234
|
+
self.db_path = ":memory:"
|
|
235
|
+
|
|
236
|
+
return visual_hash
|
|
237
|
+
|
|
238
|
+
def detect_canvas_interactive_regions(
|
|
239
|
+
self,
|
|
240
|
+
image_path: str,
|
|
241
|
+
stride: int = 16,
|
|
242
|
+
edge_threshold: int = 28,
|
|
243
|
+
min_pixels_per_cell: int = 6,
|
|
244
|
+
min_w: int = 16,
|
|
245
|
+
min_h: int = 16,
|
|
246
|
+
max_ratio: float = 0.88
|
|
247
|
+
) -> List[Dict[str, Any]]:
|
|
248
|
+
"""
|
|
249
|
+
Standalone Computer Vision Contour / Active Blob Detector for Canvas/GIS.
|
|
250
|
+
Detects interactive visual components (buttons, polygon nodes, markers, sliders, chart icons)
|
|
251
|
+
without requiring DOM elements or accessibility nodes.
|
|
252
|
+
Uses pure PIL edge filtering and connected-grid component clustering (Zero-Dependency).
|
|
253
|
+
"""
|
|
254
|
+
if not os.path.exists(image_path) or not HAS_PIL:
|
|
255
|
+
return []
|
|
256
|
+
|
|
257
|
+
img = Image.open(image_path).convert("L")
|
|
258
|
+
w, h = img.size
|
|
259
|
+
max_allowed_w = int(w * max_ratio)
|
|
260
|
+
max_allowed_h = int(h * max_ratio)
|
|
261
|
+
|
|
262
|
+
# Detect high-contrast edges & contours
|
|
263
|
+
edges = img.filter(ImageFilter.FIND_EDGES)
|
|
264
|
+
edges = edges.point(lambda p: 255 if p > edge_threshold else 0)
|
|
265
|
+
pixels = edges.load()
|
|
266
|
+
if pixels is None:
|
|
267
|
+
return []
|
|
268
|
+
|
|
269
|
+
# Identify active cells with high edge density
|
|
270
|
+
active_cells = set()
|
|
271
|
+
for y in range(0, h, stride):
|
|
272
|
+
for x in range(0, w, stride):
|
|
273
|
+
count = 0
|
|
274
|
+
for cy in range(y, min(h, y + stride)):
|
|
275
|
+
for cx in range(x, min(w, x + stride)):
|
|
276
|
+
px_val = pixels[cx, cy]
|
|
277
|
+
if isinstance(px_val, (int, float)) and px_val > 0:
|
|
278
|
+
count += 1
|
|
279
|
+
elif isinstance(px_val, tuple) and px_val and px_val[0] > 0:
|
|
280
|
+
count += 1
|
|
281
|
+
if count >= min_pixels_per_cell:
|
|
282
|
+
active_cells.add((x // stride, y // stride))
|
|
283
|
+
|
|
284
|
+
# 8-direction BFS clustering to group connected components
|
|
285
|
+
visited = set()
|
|
286
|
+
detected_elements = []
|
|
287
|
+
cluster_id = 1
|
|
288
|
+
|
|
289
|
+
for cell in sorted(active_cells):
|
|
290
|
+
if cell in visited:
|
|
291
|
+
continue
|
|
292
|
+
queue = [cell]
|
|
293
|
+
visited.add(cell)
|
|
294
|
+
cluster = [cell]
|
|
295
|
+
|
|
296
|
+
while queue:
|
|
297
|
+
curr = queue.pop(0)
|
|
298
|
+
cx, cy = curr
|
|
299
|
+
for dx in (-1, 0, 1):
|
|
300
|
+
for dy in (-1, 0, 1):
|
|
301
|
+
nb = (cx + dx, cy + dy)
|
|
302
|
+
if nb in active_cells and nb not in visited:
|
|
303
|
+
visited.add(nb)
|
|
304
|
+
queue.append(nb)
|
|
305
|
+
cluster.append(nb)
|
|
306
|
+
|
|
307
|
+
min_x = max(0, min(c[0] for c in cluster) * stride - 4)
|
|
308
|
+
min_y = max(0, min(c[1] for c in cluster) * stride - 4)
|
|
309
|
+
box_w = min(w - min_x, (max(c[0] for c in cluster) + 1) * stride - min_x + 8)
|
|
310
|
+
box_h = min(h - min_y, (max(c[1] for c in cluster) + 1) * stride - min_y + 8)
|
|
311
|
+
|
|
312
|
+
# Filter out whole-canvas background borders or tiny noise
|
|
313
|
+
if box_w >= min_w and box_h >= min_h and (box_w < max_allowed_w or box_h < max_allowed_h):
|
|
314
|
+
detected_elements.append({
|
|
315
|
+
"x": min_x,
|
|
316
|
+
"y": min_y,
|
|
317
|
+
"width": box_w,
|
|
318
|
+
"height": box_h,
|
|
319
|
+
"label": f"Canvas_Target_{cluster_id}",
|
|
320
|
+
"tag": "canvas_interactive_node"
|
|
321
|
+
})
|
|
322
|
+
cluster_id += 1
|
|
323
|
+
|
|
324
|
+
return detected_elements
|
|
325
|
+
|
|
326
|
+
def generate_som_overlay(
|
|
327
|
+
self,
|
|
328
|
+
image_path: str,
|
|
329
|
+
elements: Optional[List[Dict[str, Any]]] = None,
|
|
330
|
+
output_path: Optional[str] = None
|
|
331
|
+
) -> Tuple[str, Dict[int, Dict[str, Any]]]:
|
|
332
|
+
"""
|
|
333
|
+
Draws high-contrast numbered badges (Set-of-Marks) over each element bounding box.
|
|
334
|
+
If elements is omitted, autonomously runs detect_canvas_interactive_regions.
|
|
335
|
+
Returns (marked_image_path, marks_map).
|
|
336
|
+
"""
|
|
337
|
+
if not HAS_PIL or Image is None:
|
|
338
|
+
raise ImportError("Pillow is required for Set-of-Marks overlay generation (pip install Pillow).")
|
|
339
|
+
|
|
340
|
+
if not os.path.exists(image_path):
|
|
341
|
+
img = Image.new("RGB", (self.viewport_width, self.viewport_height), color="#1E1E2E")
|
|
342
|
+
draw = ImageDraw.Draw(img)
|
|
343
|
+
draw.text((50, 50), "Mock Financial Dashboard (HTML5 Canvas)", fill="#FFFFFF")
|
|
344
|
+
draw.rectangle([100, 150, 220, 190], outline="#00CCFF", width=2)
|
|
345
|
+
draw.rectangle([240, 150, 360, 190], outline="#FF0055", width=2)
|
|
346
|
+
draw.rectangle([500, 300, 700, 350], outline="#00FF66", width=2)
|
|
347
|
+
img.save(image_path)
|
|
348
|
+
|
|
349
|
+
if not elements:
|
|
350
|
+
detected = self.detect_canvas_interactive_regions(image_path)
|
|
351
|
+
elements = detected if detected else [
|
|
352
|
+
{"x": 100, "y": 150, "width": 120, "height": 40, "label": "TradingView Zoom In", "tag": "canvas_control"},
|
|
353
|
+
{"x": 240, "y": 150, "width": 120, "height": 40, "label": "TradingView Zoom Out", "tag": "canvas_control"},
|
|
354
|
+
{"x": 500, "y": 300, "width": 200, "height": 50, "label": "Execute Trade Order", "tag": "shadow_button"},
|
|
355
|
+
{"x": 720, "y": 300, "width": 150, "height": 50, "label": "Cancel Order", "tag": "shadow_button"}
|
|
356
|
+
]
|
|
357
|
+
|
|
358
|
+
img = Image.open(image_path).convert("RGBA")
|
|
359
|
+
overlay = Image.new("RGBA", img.size, (255, 255, 255, 0))
|
|
360
|
+
draw = ImageDraw.Draw(overlay)
|
|
361
|
+
|
|
362
|
+
marks_map = {}
|
|
363
|
+
for idx, el in enumerate(elements, start=1):
|
|
364
|
+
x = el.get("x", 0)
|
|
365
|
+
y = el.get("y", 0)
|
|
366
|
+
w = el.get("width", 80)
|
|
367
|
+
h = el.get("height", 35)
|
|
368
|
+
|
|
369
|
+
center_x = round(x + w / 2)
|
|
370
|
+
center_y = round(y + h / 2)
|
|
371
|
+
color = MARK_COLORS[(idx - 1) % len(MARK_COLORS)]
|
|
372
|
+
|
|
373
|
+
# Draw bounding box
|
|
374
|
+
draw.rectangle([x, y, x + w, y + h], outline=color, width=3)
|
|
375
|
+
|
|
376
|
+
# Draw badge label (e.g. [1])
|
|
377
|
+
badge_w, badge_h = 24, 20
|
|
378
|
+
badge_x = max(0, x)
|
|
379
|
+
badge_y = max(0, y - badge_h)
|
|
380
|
+
draw.rectangle([badge_x, badge_y, badge_x + badge_w, badge_y + badge_h], fill=color)
|
|
381
|
+
draw.text((badge_x + 6, badge_y + 3), str(idx), fill="#000000")
|
|
382
|
+
|
|
383
|
+
marks_map[idx] = {
|
|
384
|
+
"mark_id": idx,
|
|
385
|
+
"label": el.get("label") or el.get("text") or el.get("role") or f"Element_{idx}",
|
|
386
|
+
"tag": el.get("tag", "element"),
|
|
387
|
+
"bounds": [x, y, x + w, y + h],
|
|
388
|
+
"center_x": center_x,
|
|
389
|
+
"center_y": center_y,
|
|
390
|
+
"playwright_action": f"await page.mouse.click({center_x}, {center_y});"
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
combined = Image.alpha_composite(img, overlay).convert("RGB")
|
|
394
|
+
out_file = output_path or image_path.replace(".png", "_som_marked.png")
|
|
395
|
+
combined.save(out_file)
|
|
396
|
+
|
|
397
|
+
return out_file, marks_map
|
|
398
|
+
|
|
399
|
+
def resolve_mark_coordinates(
|
|
400
|
+
self,
|
|
401
|
+
marks_map: Dict[int, Dict[str, Any]],
|
|
402
|
+
target_mark_id: Optional[int] = None,
|
|
403
|
+
target_label: Optional[str] = None
|
|
404
|
+
) -> Optional[Dict[str, Any]]:
|
|
405
|
+
"""
|
|
406
|
+
Resolves the exact (x, y) coordinates for a given Mark ID or fuzzy label.
|
|
407
|
+
"""
|
|
408
|
+
if target_mark_id and target_mark_id in marks_map:
|
|
409
|
+
return marks_map[target_mark_id]
|
|
410
|
+
|
|
411
|
+
if target_label:
|
|
412
|
+
target_clean = target_label.strip().lower()
|
|
413
|
+
for mark in marks_map.values():
|
|
414
|
+
if target_clean in mark.get("label", "").lower():
|
|
415
|
+
return mark
|
|
416
|
+
|
|
417
|
+
return None
|
|
418
|
+
|
|
419
|
+
def format_vision_prompt(self, marks_map: Dict[int, Dict[str, Any]], goal: str) -> str:
|
|
420
|
+
"""
|
|
421
|
+
Formats a compact Vision brief for multimodal models (Gemini / Claude).
|
|
422
|
+
"""
|
|
423
|
+
lines = [
|
|
424
|
+
f"GOAL: {goal}",
|
|
425
|
+
f"SET-OF-MARKS (SoM) CANDIDATE REGIONS DETECTED: {len(marks_map)}",
|
|
426
|
+
"",
|
|
427
|
+
"Inspect the attached marked screenshot and identify the target Mark ID:",
|
|
428
|
+
]
|
|
429
|
+
for m_id, data in marks_map.items():
|
|
430
|
+
lines.append(f" [{m_id}] '{data['label']}' -> Center ({data['center_x']}, {data['center_y']})")
|
|
431
|
+
lines.append("")
|
|
432
|
+
lines.append("RESPOND WITH FORMAT: CLICK_MARK: <ID>")
|
|
433
|
+
return "\n".join(lines)
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def main():
|
|
437
|
+
parser = argparse.ArgumentParser(description="Multimodal Vision Grounding & Set-of-Marks Engine")
|
|
438
|
+
parser.add_argument("--image", default="screenshot.png", help="Path to input screenshot image")
|
|
439
|
+
parser.add_argument("--output", default="", help="Path to output marked image")
|
|
440
|
+
parser.add_argument("--elements-json", default="", help="JSON string or file of candidate bounding boxes")
|
|
441
|
+
parser.add_argument("--mark-id", type=int, default=None, help="Target Mark ID to resolve")
|
|
442
|
+
parser.add_argument("--label", default="", help="Target semantic label to resolve")
|
|
443
|
+
parser.add_argument("--component", default="", help="Target component name for visual cache lookup/save")
|
|
444
|
+
parser.add_argument("--cache-lookup", action="store_true", help="Attempt zero-token visual cache resolution first")
|
|
445
|
+
parser.add_argument("--cache-save", action="store_true", help="Save resolved coordinates to visual cache")
|
|
446
|
+
parser.add_argument("--threshold", type=float, default=0.98, help="Visual similarity threshold for cache hit (default: 0.98)")
|
|
447
|
+
parser.add_argument("--detect-only", action="store_true", help="Run pure contour/blob detector and print detected bounding boxes")
|
|
448
|
+
parser.add_argument("--json", action="store_true", help="Output results in JSON format")
|
|
449
|
+
args = parser.parse_args()
|
|
450
|
+
|
|
451
|
+
helper = VisualSoMHelper()
|
|
452
|
+
|
|
453
|
+
# Step 1: Zero-Token Visual Cache Lookup
|
|
454
|
+
if (args.cache_lookup or args.component) and args.component and os.path.exists(args.image):
|
|
455
|
+
cached_result = helper.lookup_cached_coordinates(
|
|
456
|
+
component_name=args.component,
|
|
457
|
+
image_source=args.image,
|
|
458
|
+
threshold=args.threshold
|
|
459
|
+
)
|
|
460
|
+
if cached_result:
|
|
461
|
+
if args.json:
|
|
462
|
+
print(json.dumps({"status": "CACHE_HIT", "data": cached_result}, indent=2))
|
|
463
|
+
else:
|
|
464
|
+
print(f"\n⚡ [CACHE HIT - ZERO VISION TOKEN] Component '{args.component}' matched ({cached_result['visual_similarity']*100:.1f}% Similarity)")
|
|
465
|
+
print(f" Coordinates : ({cached_result['center_x']}, {cached_result['center_y']})")
|
|
466
|
+
print(f" Hit Count : {cached_result['hit_count']}")
|
|
467
|
+
print(f" Action : {cached_result['playwright_action']}\n")
|
|
468
|
+
return 0
|
|
469
|
+
|
|
470
|
+
# Auto-generate mock canvas if image doesn't exist
|
|
471
|
+
if not os.path.exists(args.image):
|
|
472
|
+
img = Image.new("RGB", (helper.viewport_width, helper.viewport_height), color="#1E1E2E")
|
|
473
|
+
draw = ImageDraw.Draw(img)
|
|
474
|
+
draw.text((50, 50), "Mock GIS & Financial Dashboard (HTML5 Canvas)", fill="#FFFFFF")
|
|
475
|
+
draw.rectangle([100, 150, 220, 190], fill="#00CCFF", outline="#FFFFFF", width=2)
|
|
476
|
+
draw.rectangle([240, 150, 360, 190], fill="#FF0055", outline="#FFFFFF", width=2)
|
|
477
|
+
draw.rectangle([500, 300, 700, 350], fill="#00FF66", outline="#FFFFFF", width=2)
|
|
478
|
+
img.save(args.image)
|
|
479
|
+
|
|
480
|
+
# Detect Only
|
|
481
|
+
if args.detect_only:
|
|
482
|
+
blobs = helper.detect_canvas_interactive_regions(args.image)
|
|
483
|
+
if args.json:
|
|
484
|
+
print(json.dumps(blobs, indent=2))
|
|
485
|
+
else:
|
|
486
|
+
print(f"\n🔍 [CANVAS CONTOURS DETECTED] Found {len(blobs)} interactive regions:")
|
|
487
|
+
for b in blobs:
|
|
488
|
+
print(f" • {b['label']}: Bounds {b['x']},{b['y']} ({b['width']}x{b['height']} px)")
|
|
489
|
+
return 0
|
|
490
|
+
|
|
491
|
+
# Load elements
|
|
492
|
+
elements = []
|
|
493
|
+
if args.elements_json:
|
|
494
|
+
if os.path.exists(args.elements_json):
|
|
495
|
+
with open(args.elements_json, "r", encoding="utf-8") as f:
|
|
496
|
+
elements = json.load(f)
|
|
497
|
+
else:
|
|
498
|
+
try:
|
|
499
|
+
elements = json.loads(args.elements_json)
|
|
500
|
+
except json.JSONDecodeError:
|
|
501
|
+
elements = []
|
|
502
|
+
|
|
503
|
+
if not elements:
|
|
504
|
+
detected = helper.detect_canvas_interactive_regions(args.image)
|
|
505
|
+
elements = detected if detected else [
|
|
506
|
+
{"x": 100, "y": 150, "width": 120, "height": 40, "label": "TradingView Zoom In", "tag": "canvas_control"},
|
|
507
|
+
{"x": 240, "y": 150, "width": 120, "height": 40, "label": "TradingView Zoom Out", "tag": "canvas_control"},
|
|
508
|
+
{"x": 500, "y": 300, "width": 200, "height": 50, "label": "Execute Trade Order", "tag": "shadow_button"},
|
|
509
|
+
{"x": 720, "y": 300, "width": 150, "height": 50, "label": "Cancel Order", "tag": "shadow_button"}
|
|
510
|
+
]
|
|
511
|
+
|
|
512
|
+
marked_img, marks_map = helper.generate_som_overlay(args.image, elements, output_path=args.output)
|
|
513
|
+
|
|
514
|
+
resolved = None
|
|
515
|
+
target_query = args.label or args.component
|
|
516
|
+
if args.mark_id is not None or target_query:
|
|
517
|
+
resolved = helper.resolve_mark_coordinates(marks_map, target_mark_id=args.mark_id, target_label=target_query)
|
|
518
|
+
|
|
519
|
+
# Cache Save if requested or if component is specified and resolved
|
|
520
|
+
if resolved and (args.cache_save or args.component):
|
|
521
|
+
comp_name = args.component or resolved["label"]
|
|
522
|
+
helper.save_cached_coordinates(
|
|
523
|
+
component_name=comp_name,
|
|
524
|
+
image_source=args.image,
|
|
525
|
+
center_x=resolved["center_x"],
|
|
526
|
+
center_y=resolved["center_y"]
|
|
527
|
+
)
|
|
528
|
+
|
|
529
|
+
if args.json:
|
|
530
|
+
result = {
|
|
531
|
+
"marked_image": marked_img,
|
|
532
|
+
"total_marks": len(marks_map),
|
|
533
|
+
"marks": marks_map,
|
|
534
|
+
"resolved_target": resolved
|
|
535
|
+
}
|
|
536
|
+
print(json.dumps(result, indent=2))
|
|
537
|
+
else:
|
|
538
|
+
print(f"\n🎨 [SET-OF-MARKS GENERATED] Marked image: {marked_img}")
|
|
539
|
+
print(f" Candidate Visual Regions: {len(marks_map)}")
|
|
540
|
+
for mid, data in marks_map.items():
|
|
541
|
+
print(f" [{mid}] {data['label']} at ({data['center_x']}, {data['center_y']})")
|
|
542
|
+
if resolved:
|
|
543
|
+
print(f"\n🎯 [COORDINATE GROUNDED] Target '{resolved['label']}' -> Mark #{resolved['mark_id']}")
|
|
544
|
+
print(f" Playwright Mouse Action: {resolved['playwright_action']}\n")
|
|
545
|
+
|
|
546
|
+
return 0
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
if __name__ == "__main__":
|
|
550
|
+
sys.exit(main())
|