@teevee98/synapse-hub 0.0.0-stage → 3.8.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (369) hide show
  1. package/.agents/agents/api-engineer.md +67 -0
  2. package/.agents/agents/failure-analyst.md +75 -0
  3. package/.agents/agents/live-explorer.md +70 -0
  4. package/.agents/agents/orchestrator.md +104 -0
  5. package/.agents/agents/perf-engineer.md +74 -0
  6. package/.agents/agents/pom-crafter.md +112 -0
  7. package/.agents/agents/quality-gatekeeper.md +87 -0
  8. package/.agents/agents/test-designer.md +73 -0
  9. package/.agents/agents/test-runner.md +67 -0
  10. package/.agents/rules/author-integrity-guard.md +33 -0
  11. package/.agents/rules/communication-style.md +18 -0
  12. package/.agents/rules/context-engineering.md +37 -0
  13. package/.agents/rules/git-push-protocol.md +33 -0
  14. package/.agents/rules/graphify.md +14 -0
  15. package/.agents/rules/lessons-learned.md +25 -0
  16. package/.agents/rules/mandatory-playwright-live.md +25 -0
  17. package/.agents/rules/mobile_appium_rules.md +43 -0
  18. package/.agents/rules/performance_testing_rules.md +35 -0
  19. package/.agents/rules/skill-precedence.md +29 -0
  20. package/.agents/rules/systematic-debugging.md +36 -0
  21. package/.agents/skills/actor-qa-swarm/SKILL.md +77 -0
  22. package/.agents/skills/actor-qa-swarm/examples/broker_sharding_sample.json +12 -0
  23. package/.agents/skills/agent-orchestration-multi-agent-optimize/SKILL.md +88 -0
  24. package/.agents/skills/agent-orchestration-multi-agent-optimize/examples/swarm_workload_config.json +15 -0
  25. package/.agents/skills/agentless-qa-repair/SKILL.md +69 -0
  26. package/.agents/skills/agentless-qa-repair/examples/healing_report_sample.json +17 -0
  27. package/.agents/skills/agentless-qa-repair/examples/three_phase_repair_workflow.md +40 -0
  28. package/.agents/skills/antigravity-test-sentinel/SKILL.md +63 -0
  29. package/.agents/skills/antigravity-test-sentinel/examples/dod_gate_policy.json +11 -0
  30. package/.agents/skills/antigravity-test-sentinel/examples/sprint_dod_enforcement.md +47 -0
  31. package/.agents/skills/api-network-mocking-interception/SKILL.md +142 -0
  32. package/.agents/skills/api-network-mocking-interception/examples/network_route_stub.json +9 -0
  33. package/.agents/skills/api-spec-to-e2e-matrix/SKILL.md +80 -0
  34. package/.agents/skills/api-spec-to-e2e-matrix/examples/sample_openapi.json +19 -0
  35. package/.agents/skills/architecture-decision-records/SKILL.md +64 -0
  36. package/.agents/skills/architecture-decision-records/examples/0001-record-architecture-decisions.md +12 -0
  37. package/.agents/skills/auth-session-and-cookie-manager/SKILL.md +75 -0
  38. package/.agents/skills/auth-session-and-cookie-manager/examples/multi_role_auth_matrix.md +70 -0
  39. package/.agents/skills/auth-session-and-cookie-manager/examples/storage_state_sample.json +11 -0
  40. package/.agents/skills/backend-load-and-stress-patterns/SKILL.md +85 -0
  41. package/.agents/skills/backend-load-and-stress-patterns/examples/k6_load_test_sample.js +13 -0
  42. package/.agents/skills/backend-load-and-stress-patterns/references/k6_load_template.js +82 -0
  43. package/.agents/skills/bats-testing-patterns/SKILL.md +100 -0
  44. package/.agents/skills/bats-testing-patterns/examples/cli_runner_test.bats +6 -0
  45. package/.agents/skills/bats-testing-patterns/resources/implementation-playbook.md +614 -0
  46. package/.agents/skills/clean-pom-crafter/SKILL.md +110 -0
  47. package/.agents/skills/clean-pom-crafter/examples/standard_test_case.groovy +28 -0
  48. package/.agents/skills/clean-pom-crafter/references/pom_patterns.groovy +138 -0
  49. package/.agents/skills/code-refactoring-refactor-clean/SKILL.md +64 -0
  50. package/.agents/skills/code-refactoring-refactor-clean/examples/refactored_page_sample.groovy +9 -0
  51. package/.agents/skills/code-refactoring-refactor-clean/examples/refactored_playwright_pom.ts +52 -0
  52. package/.agents/skills/code-refactoring-refactor-clean/resources/implementation-playbook.md +881 -0
  53. package/.agents/skills/consumer-driven-contract-testing/SKILL.md +147 -0
  54. package/.agents/skills/consumer-driven-contract-testing/examples/order_service_pact.json +45 -0
  55. package/.agents/skills/db-state-verifier-and-seeder/SKILL.md +123 -0
  56. package/.agents/skills/db-state-verifier-and-seeder/examples/db_fixture_seed.json +12 -0
  57. package/.agents/skills/e2e-testing-patterns/SKILL.md +103 -0
  58. package/.agents/skills/e2e-testing-patterns/examples/playwright_sharded_config.ts +12 -0
  59. package/.agents/skills/e2e-testing-patterns/resources/implementation-playbook.md +522 -0
  60. package/.agents/skills/event-stream-qa-orchestrator/SKILL.md +79 -0
  61. package/.agents/skills/event-stream-qa-orchestrator/examples/timeline_sample.jsonl +2 -0
  62. package/.agents/skills/failure-autopsy-and-healing/SKILL.md +66 -0
  63. package/.agents/skills/failure-autopsy-and-healing/examples/closed_loop_healing_walkthrough.md +48 -0
  64. package/.agents/skills/failure-autopsy-and-healing/examples/sample_junit_failure.xml +8 -0
  65. package/.agents/skills/failure-autopsy-and-healing/references/healing_matrix.md +64 -0
  66. package/.agents/skills/failure-autopsy-and-healing/references/qa_investigation_protocol.md +56 -0
  67. package/.agents/skills/failure-autopsy-and-healing/scripts/triage_failures.sh +83 -0
  68. package/.agents/skills/file-and-report-validator/SKILL.md +90 -0
  69. package/.agents/skills/file-and-report-validator/examples/sample_export_schema.json +14 -0
  70. package/.agents/skills/flaky-test-sentinel-and-quarantine/SKILL.md +114 -0
  71. package/.agents/skills/flaky-test-sentinel-and-quarantine/examples/quarantine_registry.json +11 -0
  72. package/.agents/skills/gherkin-to-suite/SKILL.md +103 -0
  73. package/.agents/skills/gherkin-to-suite/examples/checkout.feature +39 -0
  74. package/.agents/skills/gherkin-to-suite/examples/suite_report_sample.json +334 -0
  75. package/.agents/skills/github-actions-templates/SKILL.md +77 -0
  76. package/.agents/skills/github-actions-templates/examples/matrix_test_pipeline.yml +8 -0
  77. package/.agents/skills/graphify-impact-analyzer/SKILL.md +76 -0
  78. package/.agents/skills/graphify-impact-analyzer/examples/dependency_blast_radius.json +8 -0
  79. package/.agents/skills/katalon-cli-runner/SKILL.md +76 -0
  80. package/.agents/skills/katalon-cli-runner/examples/headless_kre_exec.sh +3 -0
  81. package/.agents/skills/katalon-cli-runner/references/groovy-patterns.md +209 -0
  82. package/.agents/skills/katalon-cli-runner/references/katalon-api.md +155 -0
  83. package/.agents/skills/katalon-cli-runner/references/runner_commands.md +67 -0
  84. package/.agents/skills/lam-script-generator/SKILL.md +123 -0
  85. package/.agents/skills/lam-script-generator/examples/user_story_sample.md +8 -0
  86. package/.agents/skills/lean-code-ladder/SKILL.md +94 -0
  87. package/.agents/skills/lean-code-ladder/examples/ladder_walkthrough.md +37 -0
  88. package/.agents/skills/lean-impact-scoreboard/SKILL.md +48 -0
  89. package/.agents/skills/lean-impact-scoreboard/examples/sample_benchmark_gain.md +20 -0
  90. package/.agents/skills/lean-impact-scoreboard/examples/sample_scoreboard.md +15 -0
  91. package/.agents/skills/meta-prompt-crafter/SKILL.md +107 -0
  92. package/.agents/skills/meta-prompt-crafter/examples/qa_prompt_template.md +35 -0
  93. package/.agents/skills/mipro-prompt-compiler/SKILL.md +59 -0
  94. package/.agents/skills/mipro-prompt-compiler/examples/compiled_module_sample.json +22 -0
  95. package/.agents/skills/mipro-prompt-compiler/examples/dspy_assertion_optimizer_case.md +46 -0
  96. package/.agents/skills/mobile-appium-som/SKILL.md +88 -0
  97. package/.agents/skills/mobile-appium-som/examples/BaseScreen.java +88 -0
  98. package/.agents/skills/mobile-appium-som/examples/LoginScreen.java +46 -0
  99. package/.agents/skills/mobile-appium-som/examples/TC_MobileLogin.java +43 -0
  100. package/.agents/skills/persistent-planning-files/SKILL.md +108 -0
  101. package/.agents/skills/persistent-planning-files/examples/task_plan_example.md +56 -0
  102. package/.agents/skills/playwright-live-interaction/SKILL.md +131 -0
  103. package/.agents/skills/playwright-live-interaction/examples/live_probe_script.js +5 -0
  104. package/.agents/skills/playwright-live-interaction/references/advanced_superpowers.md +160 -0
  105. package/.agents/skills/playwright-live-interaction/references/compact_accessibility_snapshots.md +75 -0
  106. package/.agents/skills/playwright-live-interaction/references/recipes.md +388 -0
  107. package/.agents/skills/playwright-live-interaction/references/swarm_orchestration.md +134 -0
  108. package/.agents/skills/playwright-webapp-testing/SKILL.md +93 -0
  109. package/.agents/skills/playwright-webapp-testing/examples/server_lifecycle_playwright.py +63 -0
  110. package/.agents/skills/qa-aci-interface/SKILL.md +67 -0
  111. package/.agents/skills/qa-aci-interface/examples/surgical_code_window_walkthrough.md +40 -0
  112. package/.agents/skills/qa-aci-interface/examples/surgical_edit_sample.json +12 -0
  113. package/.agents/skills/qa-memory-os/SKILL.md +80 -0
  114. package/.agents/skills/qa-memory-os/examples/archival_memory_sample.json +21 -0
  115. package/.agents/skills/rbt-manual-testing/SKILL.md +137 -0
  116. package/.agents/skills/rbt-manual-testing/examples/checkout_rbt_manual_suite.md +25 -0
  117. package/.agents/skills/rbt-manual-testing/references/field_validation_checklist.md +29 -0
  118. package/.agents/skills/repo-quality-audit/SKILL.md +68 -0
  119. package/.agents/skills/repo-quality-audit/examples/sample_audit_walkthrough.md +15 -0
  120. package/.agents/skills/repo-quality-audit/examples/sample_repo_audit.md +33 -0
  121. package/.agents/skills/review-spec/SKILL.md +60 -0
  122. package/.agents/skills/review-spec/examples/pocock_grilling_decision_rubric.md +40 -0
  123. package/.agents/skills/review-spec/examples/sample_spec_review.md +33 -0
  124. package/.agents/skills/security-dast-sentinel/SKILL.md +119 -0
  125. package/.agents/skills/security-dast-sentinel/examples/dast_scan_config.json +10 -0
  126. package/.agents/skills/security-dast-sentinel/references/AI-AND-LLM.md +113 -0
  127. package/.agents/skills/security-dast-sentinel/references/ATTACK-CLASSES.md +89 -0
  128. package/.agents/skills/security-dast-sentinel/references/dast_recipes.ts +33 -0
  129. package/.agents/skills/self-healing-locator/SKILL.md +86 -0
  130. package/.agents/skills/self-healing-locator/examples/aom_diff_sample.json +16 -0
  131. package/.agents/skills/senior-code-review/SKILL.md +108 -0
  132. package/.agents/skills/senior-code-review/examples/sample_review_walkthrough.md +29 -0
  133. package/.agents/skills/skillspector-security-guard/SKILL.md +57 -0
  134. package/.agents/skills/skillspector-security-guard/examples/audit_threat_vectors_report.json +35 -0
  135. package/.agents/skills/skillspector-security-guard/examples/safe_mcp_config.json +12 -0
  136. package/.agents/skills/surgical-coding-principles/SKILL.md +89 -0
  137. package/.agents/skills/surgical-coding-principles/examples/surgical_vs_scope_creep.md +46 -0
  138. package/.agents/skills/synthetic-test-data-generator/SKILL.md +135 -0
  139. package/.agents/skills/synthetic-test-data-generator/examples/idempotent_data_recipe.json +9 -0
  140. package/.agents/skills/systematic-debugging/SKILL.md +100 -0
  141. package/.agents/skills/systematic-debugging/examples/four_phase_case.md +44 -0
  142. package/.agents/skills/technical-debt-harvester/SKILL.md +70 -0
  143. package/.agents/skills/technical-debt-harvester/examples/sample_debt_ledger.md +25 -0
  144. package/.agents/skills/technical-debt-harvester/examples/sample_ledger_output.md +20 -0
  145. package/.agents/skills/test-driven-development/SKILL.md +114 -0
  146. package/.agents/skills/test-driven-development/examples/red_green_refactor_qa.md +78 -0
  147. package/.agents/skills/token-and-context-optimizer/SKILL.md +70 -0
  148. package/.agents/skills/token-and-context-optimizer/examples/distilled_log_sample.txt +3 -0
  149. package/.agents/skills/token-and-context-optimizer/references/context_engineering_guide.md +49 -0
  150. package/.agents/skills/tot-exploratory-testing/SKILL.md +59 -0
  151. package/.agents/skills/tot-exploratory-testing/examples/payment_gateway_tot_tree.json +62 -0
  152. package/.agents/skills/tot-exploratory-testing/examples/tot_graph_sample.json +35 -0
  153. package/.agents/skills/verification-before-completion/SKILL.md +100 -0
  154. package/.agents/skills/verification-before-completion/examples/evidence_log_example.md +46 -0
  155. package/.agents/skills/visual-regression-sentinel/SKILL.md +82 -0
  156. package/.agents/skills/visual-regression-sentinel/examples/visual_mask_rules.json +9 -0
  157. package/.agents/skills/wcag-audit-patterns/SKILL.md +77 -0
  158. package/.agents/skills/wcag-audit-patterns/examples/wcag_compliance_checklist.json +10 -0
  159. package/.agents/skills/wcag-audit-patterns/resources/implementation-playbook.md +541 -0
  160. package/.agents/skills/web-agent-gym-benchmarking/SKILL.md +62 -0
  161. package/.agents/skills/web-agent-gym-benchmarking/examples/benchmark_report_sample.json +22 -0
  162. package/.agents/skills/web-agent-gym-benchmarking/examples/enterprise_tasks_suite.json +38 -0
  163. package/.agents/skills/web-vitals-and-performance-profiler/SKILL.md +65 -0
  164. package/.agents/skills/web-vitals-and-performance-profiler/examples/cwv_budget_policy.json +6 -0
  165. package/.agents/skills/web-vitals-and-performance-profiler/examples/playwright_cwv_profiler_fixture.ts +118 -0
  166. package/.agents/templates/completion_report.md +57 -0
  167. package/.agents/workflows/analyze_requirement_document.md +18 -0
  168. package/.agents/workflows/export_testcases_excel.md +32 -0
  169. package/.agents/workflows/generate_combinatorial_test_data.md +28 -0
  170. package/.agents/workflows/generate_cross_module_test_plan.md +37 -0
  171. package/.agents/workflows/generate_manual_testcases_rbt.md +56 -0
  172. package/.agents/workflows/generate_requirements_from_website.md +25 -0
  173. package/.agents/workflows/generate_testcases_from_requirements.md +21 -0
  174. package/.agents/workflows/graphify.md +10 -0
  175. package/.agents/workflows/test-impact.md +15 -0
  176. package/.agents/workflows/verify.md +34 -0
  177. package/.cursor/rules/agents.mdc +16 -0
  178. package/.cursor/rules/language.mdc +7 -0
  179. package/.cursor/rules/synapse-qa.mdc +24 -0
  180. package/.cursorrules +27 -0
  181. package/.dockerignore +53 -0
  182. package/.pre-commit-hooks.yaml +14 -0
  183. package/.windsurfrules +26 -0
  184. package/AGENTS.md +86 -0
  185. package/CLAUDE.md +54 -0
  186. package/Dockerfile +81 -0
  187. package/LICENSE +52 -0
  188. package/NOTICE +264 -0
  189. package/README.en.md +171 -0
  190. package/README.md +170 -2
  191. package/action.yml +51 -0
  192. package/agentskills.json +16 -0
  193. package/bin/agent-qa.js +138 -0
  194. package/docker-compose.yml +74 -0
  195. package/docs/ARCHITECTURE.en.md +198 -0
  196. package/docs/ARCHITECTURE.md +199 -0
  197. package/docs/CI_CD_INTEGRATION.en.md +170 -0
  198. package/docs/CI_CD_INTEGRATION.md +170 -0
  199. package/docs/CLI_REFERENCE.en.md +174 -0
  200. package/docs/CLI_REFERENCE.md +176 -0
  201. package/docs/CONTRIBUTING.en.md +49 -0
  202. package/docs/CONTRIBUTING.md +49 -0
  203. package/docs/PERFORMANCE_AUDIT_REPORT.md +783 -0
  204. package/docs/PLUGIN_DEVELOPMENT_GUIDE.md +127 -0
  205. package/docs/QUALITY_GATES.en.md +118 -0
  206. package/docs/QUALITY_GATES.md +118 -0
  207. package/docs/TOKEN_QUOTA_PLAYBOOK.md +41 -0
  208. package/docs/TOOL_CATALOG.md +129 -0
  209. package/docs/adopted-ecosystem-tools.md +96 -0
  210. package/docs/research/EXTERNAL_AGENT_SKILLS_SURVEY.md +1062 -0
  211. package/docs/research/agent_skill_training_analysis.md +1358 -0
  212. package/docs/schemas/synapse.config.schema.json +86 -0
  213. package/package.json +94 -4
  214. package/plugin.json +31 -0
  215. package/scripts/generate_skill_examples.py +244 -0
  216. package/scripts/install.ps1 +112 -0
  217. package/scripts/install_global_plugin.ps1 +94 -0
  218. package/scripts/install_global_plugin.sh +49 -0
  219. package/scripts/lint_skills.ps1 +57 -0
  220. package/scripts/package_dist.py +337 -0
  221. package/scripts/setup.sh +59 -0
  222. package/scripts/update_hub.ps1 +19 -0
  223. package/scripts/update_hub.sh +15 -0
  224. package/skills_manifest.json +854 -0
  225. package/synapse.config.json +31 -0
  226. package/templates/PROJECT_AI_GUIDE_TEMPLATE.md +111 -0
  227. package/templates/PROJECT_STATE_TEMPLATE.md +20 -0
  228. package/templates/ai_bridges/CLAUDE.md +54 -0
  229. package/templates/ai_bridges/copilot-instructions.md +20 -0
  230. package/templates/ai_bridges/cursor_synapse_qa.mdc +24 -0
  231. package/templates/ai_bridges/windsurfrules +26 -0
  232. package/templates/ci/github-actions-synapse.yml +95 -0
  233. package/templates/ci/gitlab-ci-synapse.yml +16 -0
  234. package/templates/ui/index.html +1069 -0
  235. package/templates/welcome.html.tmpl +2074 -0
  236. package/tools/.infer_baseline.json +30 -0
  237. package/tools/TOOLS_RUNBOOK.md +1553 -0
  238. package/tools/TOOL_CATALOG.json +1769 -0
  239. package/tools/__init__.py +43 -0
  240. package/tools/_openapi_parser.py +158 -0
  241. package/tools/agent_canary.py +525 -0
  242. package/tools/agentless_patcher.py +417 -0
  243. package/tools/analyze_project.py +262 -0
  244. package/tools/api_drift_sentinel.py +208 -0
  245. package/tools/api_fuzzer.py +762 -0
  246. package/tools/author_integrity.py +190 -0
  247. package/tools/auto_heal.py +271 -0
  248. package/tools/auto_heal_reflexion.py +532 -0
  249. package/tools/autogen_qa_broker.py +844 -0
  250. package/tools/benchmark_swarm_context.py +394 -0
  251. package/tools/browser_helpers.js +746 -0
  252. package/tools/browsergym_eval.py +236 -0
  253. package/tools/bug_reporter.py +252 -0
  254. package/tools/bump_version.py +423 -0
  255. package/tools/cache_resolver.py +1214 -0
  256. package/tools/check_env.py +426 -0
  257. package/tools/check_version_sync.py +256 -0
  258. package/tools/ci_healing_hook.py +126 -0
  259. package/tools/cli.py +1598 -0
  260. package/tools/config_decoupler.py +253 -0
  261. package/tools/contract_validator.py +141 -0
  262. package/tools/core/__init__.py +70 -0
  263. package/tools/core/adapters.py +431 -0
  264. package/tools/core/circuit_breaker.py +263 -0
  265. package/tools/core/cli_runtime.py +66 -0
  266. package/tools/core/file_lock.py +159 -0
  267. package/tools/core/i18n.py +394 -0
  268. package/tools/core/memory_bus.py +905 -0
  269. package/tools/core/process_supervisor.py +332 -0
  270. package/tools/core/telemetry.py +329 -0
  271. package/tools/count_tokens.py +1267 -0
  272. package/tools/create_mock_docx.py +104 -0
  273. package/tools/dashboard_generator.py +2403 -0
  274. package/tools/db_sandbox.py +675 -0
  275. package/tools/debt_harvester.py +237 -0
  276. package/tools/distill_log.py +161 -0
  277. package/tools/dspy_qa_signatures.py +356 -0
  278. package/tools/e2e_flow_prober.py +132 -0
  279. package/tools/export_postman_collection.py +945 -0
  280. package/tools/flaky_detector.py +389 -0
  281. package/tools/fsm_enforcer.py +450 -0
  282. package/tools/fsm_middleware.py +401 -0
  283. package/tools/fuzz_payloads.py +197 -0
  284. package/tools/generate_allure_report.py +88 -0
  285. package/tools/generate_api_matrix.py +193 -0
  286. package/tools/generate_jmx_plan.py +1322 -0
  287. package/tools/generate_sbom.py +252 -0
  288. package/tools/generate_tc_from_story.py +203 -0
  289. package/tools/generate_test_reports.py +133 -0
  290. package/tools/generate_tool_catalog.py +482 -0
  291. package/tools/gherkin_suite.py +758 -0
  292. package/tools/git_pre_commit_guard.py +298 -0
  293. package/tools/goldfish_review.py +366 -0
  294. package/tools/heal/__init__.py +28 -0
  295. package/tools/heal_pr_automator.py +220 -0
  296. package/tools/heuristic_locator_matcher.py +351 -0
  297. package/tools/hub_publisher.py +623 -0
  298. package/tools/humanize_linter.py +438 -0
  299. package/tools/infer_linter.py +977 -0
  300. package/tools/init_config.py +306 -0
  301. package/tools/interview_story.py +116 -0
  302. package/tools/jira_sync.py +566 -0
  303. package/tools/jtl_analyzer.py +474 -0
  304. package/tools/lint/__init__.py +30 -0
  305. package/tools/lint_ai_docs.py +374 -0
  306. package/tools/lookup_tc.py +165 -0
  307. package/tools/lookup_xpath.py +167 -0
  308. package/tools/matrix_sharder.py +148 -0
  309. package/tools/mcp/__init__.py +18 -0
  310. package/tools/mcp_docx_server.ps1 +13 -0
  311. package/tools/mcp_docx_server.py +851 -0
  312. package/tools/mcp_excel_server.ps1 +13 -0
  313. package/tools/mcp_excel_server.py +441 -0
  314. package/tools/mcp_server.py +725 -0
  315. package/tools/md_to_xlsx.py +327 -0
  316. package/tools/micro_mutator.py +887 -0
  317. package/tools/mipro_assertion_tuner.py +210 -0
  318. package/tools/network_mock_shield.py +311 -0
  319. package/tools/newman_analyzer.py +384 -0
  320. package/tools/omni_emitter.py +579 -0
  321. package/tools/omni_parser.py +450 -0
  322. package/tools/omni_transpiler.py +168 -0
  323. package/tools/openxml_guard.py +438 -0
  324. package/tools/output_formatter.py +182 -0
  325. package/tools/package_release.py +325 -0
  326. package/tools/perf/__init__.py +24 -0
  327. package/tools/pinpoint_failure.py +190 -0
  328. package/tools/project_state.py +171 -0
  329. package/tools/qa_event_stream.py +358 -0
  330. package/tools/repo_map.py +346 -0
  331. package/tools/report/__init__.py +32 -0
  332. package/tools/reversible_sandbox.py +344 -0
  333. package/tools/safety_firewall.py +437 -0
  334. package/tools/security_coverage_ledger.py +538 -0
  335. package/tools/security_scanner.py +229 -0
  336. package/tools/senior_code_review.py +393 -0
  337. package/tools/setup_project.py +770 -0
  338. package/tools/skill_collision_guard.py +304 -0
  339. package/tools/skill_profiles.py +217 -0
  340. package/tools/skill_vault.py +481 -0
  341. package/tools/skillspector.py +1278 -0
  342. package/tools/smart_visual_masker.py +132 -0
  343. package/tools/spec_drift_sentinel.py +438 -0
  344. package/tools/swe_qa_aci.py +369 -0
  345. package/tools/synapse_observer.py +485 -0
  346. package/tools/synapse_qa_bench.py +190 -0
  347. package/tools/sync_agent_rules.py +274 -0
  348. package/tools/telemetry_correlator.py +397 -0
  349. package/tools/template_excel_exporter.py +514 -0
  350. package/tools/templates/jmeter-distributed/README.md +54 -0
  351. package/tools/templates/jmeter-distributed/docker-compose.yml +83 -0
  352. package/tools/test_impact.py +419 -0
  353. package/tools/test_memory_os.py +462 -0
  354. package/tools/textgrad_mutation_optimizer.py +237 -0
  355. package/tools/tms_sync_bridge.py +834 -0
  356. package/tools/tot_exploratory_tester.py +239 -0
  357. package/tools/trace_data_flow.py +240 -0
  358. package/tools/traffic_session_converter.py +256 -0
  359. package/tools/trend_analytics.py +314 -0
  360. package/tools/ui_server.py +401 -0
  361. package/tools/universal_ir.py +488 -0
  362. package/tools/update_sentinel.py +383 -0
  363. package/tools/validate_groovy_scripts.py +173 -0
  364. package/tools/verify_all.py +1271 -0
  365. package/tools/visual_diff_canvas.py +143 -0
  366. package/tools/visual_som_helper.py +550 -0
  367. package/tools/watch_mode.py +308 -0
  368. package/tools/wcag_audit.py +239 -0
  369. package/tools/welcome_page.py +290 -0
@@ -0,0 +1,485 @@
1
+ #!/usr/bin/env python3
2
+ # SPDX-FileCopyrightText: Copyright (c) 2026 Thế Vị (@TeeVee98). All rights reserved.
3
+ # SPDX-License-Identifier: MIT
4
+ # Project: Synapse-Hub (https://github.com/TeeVee98/Synapse-Hub)
5
+
6
+ """
7
+ Synapse Cognitive Observer & Tradeoff Sentinel
8
+ ----------------------------------------------
9
+ Inspired by Anthropic Claude Code 2.1.292 (wassname/cc-plugin-you-should-know).
10
+ Acts as an out-of-band observer that inspects code changes, diffs, and architectural
11
+ decisions to surface critical tradeoffs, hidden costs, or risks that the user may
12
+ have missed during rapid development.
13
+
14
+ Core Principles:
15
+ 1. High Interruption Bar: Defaults to 'learn: none' unless a real consequence exists.
16
+ 2. Human-Centric Communication: Plain-language, jargon-free title (3-7 words) and <120 words explanation.
17
+ 3. Anti-Nagging Memory: Never repeats topics that the user already acknowledged.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ import argparse
23
+ import hashlib
24
+ import json
25
+ import re
26
+ import subprocess
27
+ import sys
28
+ from dataclasses import asdict, dataclass
29
+ from pathlib import Path
30
+ from typing import List, Optional, Tuple, Union
31
+
32
+ # Enforce UTF-8 output encoding
33
+ if hasattr(sys.stdout, "reconfigure"):
34
+ try:
35
+ sys.stdout.reconfigure(encoding="utf-8")
36
+ sys.stderr.reconfigure(encoding="utf-8")
37
+ except Exception:
38
+ pass
39
+
40
+ MEMORY_DIR = Path.home() / ".agent_memory"
41
+ MEMORY_FILE = MEMORY_DIR / "synapse_observer_history.json"
42
+
43
+
44
+ @dataclass
45
+ class ObserverAlert:
46
+ tag: str # 'Heads up' | 'You should know'
47
+ learn: str # 1-2 sentences (< 25 words)
48
+ title: str # 3-7 words bold title
49
+ explanation: List[str] # Bullet points explaining why and action
50
+ topic_key: str # Unique slug for anti-nagging memory
51
+
52
+
53
+ def is_page_class_file(file_path: str) -> bool:
54
+ """Return True if the file path indicates a Page Class / Object Repository."""
55
+ if not file_path:
56
+ return False
57
+ fp = file_path.replace("\\", "/").lower()
58
+ # Directory path checks for dedicated Page Class subfolders
59
+ if any(p in fp for p in ["pages/", "page_objects/", "object repository/", "object-repository/"]):
60
+ return True
61
+
62
+ # Test script exclusion checks
63
+ base = Path(fp).name.lower()
64
+ is_test_script = (
65
+ base.startswith("test_")
66
+ or base.endswith("_test.py")
67
+ or "_test." in base
68
+ or ".spec." in base
69
+ or ".test." in base
70
+ or any(td in fp for td in ["tests/", "scripts/", "spec/", "specs/"])
71
+ )
72
+ if is_test_script:
73
+ return False
74
+
75
+ # Filename checks
76
+ if "page." in base or base.endswith("page") or "_page." in base:
77
+ return True
78
+ return False
79
+
80
+
81
+ def parse_diff_files(diff_text: str) -> List[Tuple[str, str]]:
82
+ """Parse diff into [(file_path, added_lines_text), ...]."""
83
+ files: List[Tuple[str, str]] = []
84
+ current_file = ""
85
+ current_lines: List[str] = []
86
+
87
+ for line in diff_text.splitlines():
88
+ if line.startswith("diff --git"):
89
+ if current_file or current_lines:
90
+ files.append((current_file, "\n".join(current_lines)))
91
+ current_lines = []
92
+ m = re.search(r"diff --git a/.*? b/(.*)", line)
93
+ current_file = m.group(1).strip() if m else ""
94
+ elif line.startswith("+++ b/"):
95
+ current_file = line[6:].strip()
96
+ elif line.startswith("+") and not line.startswith("+++"):
97
+ current_lines.append(line)
98
+
99
+ if current_file or current_lines:
100
+ files.append((current_file, "\n".join(current_lines)))
101
+
102
+ if not files and diff_text:
103
+ # Diff has no git headers; treat entire text as single unassociated chunk
104
+ files.append(("", diff_text))
105
+
106
+ return files
107
+
108
+
109
+ class SynapseObserver:
110
+ """Out-of-band observer detecting consequential tradeoffs and risks."""
111
+
112
+ def __init__(self, memory_path: Optional[Path] = None):
113
+ self.memory_path = Path(memory_path) if memory_path else MEMORY_FILE
114
+ self.acknowledged_topics: set[str] = self._load_acknowledged_topics()
115
+
116
+ def _load_acknowledged_topics(self) -> set[str]:
117
+ if not self.memory_path.exists():
118
+ return set()
119
+ try:
120
+ data = json.loads(self.memory_path.read_text(encoding="utf-8"))
121
+ if isinstance(data, dict):
122
+ acks = data.get("acknowledged", [])
123
+ if isinstance(acks, list):
124
+ return set(str(x) for x in acks if x is not None)
125
+ return set()
126
+ except (json.JSONDecodeError, OSError, AttributeError) as err:
127
+ sys.stderr.write(f"[SYNAPSE-OBSERVER] Warning loading memory: {err}\n")
128
+ return set()
129
+
130
+ def _save_memory(self) -> None:
131
+ self.memory_path.parent.mkdir(parents=True, exist_ok=True)
132
+ current = {}
133
+ if self.memory_path.exists():
134
+ try:
135
+ current = json.loads(self.memory_path.read_text(encoding="utf-8"))
136
+ except (json.JSONDecodeError, OSError):
137
+ current = {}
138
+ current["acknowledged"] = sorted(list(self.acknowledged_topics))
139
+ self.memory_path.write_text(
140
+ json.dumps(current, indent=2, ensure_ascii=False),
141
+ encoding="utf-8",
142
+ )
143
+
144
+ def is_acknowledged(self, topic_key: str) -> bool:
145
+ """Check if topic or its base family is already acknowledged."""
146
+ if topic_key in self.acknowledged_topics:
147
+ return True
148
+ base_topic = topic_key.split("::")[0]
149
+ return base_topic in self.acknowledged_topics
150
+
151
+ def acknowledge_topic(self, topic_key: str) -> None:
152
+ """Mark a topic as acknowledged by the user so it won't be repeated."""
153
+ self.acknowledged_topics.add(topic_key)
154
+ self._save_memory()
155
+
156
+ def unacknowledge_topic(self, topic_key: str) -> bool:
157
+ """Remove a topic (and any child scoped keys) from acknowledged memory."""
158
+ removed = False
159
+ to_remove = [
160
+ t
161
+ for t in self.acknowledged_topics
162
+ if t == topic_key or t.startswith(f"{topic_key}::")
163
+ ]
164
+ for t in to_remove:
165
+ self.acknowledged_topics.discard(t)
166
+ removed = True
167
+ if removed:
168
+ self._save_memory()
169
+ return removed
170
+
171
+ def reset_memory(self) -> None:
172
+ """Reset and clear all acknowledged topics from memory."""
173
+ self.acknowledged_topics.clear()
174
+ self._save_memory()
175
+
176
+ def evaluate_diff(
177
+ self, diff_text: str, return_all: bool = False
178
+ ) -> Union[Optional[ObserverAlert], List[ObserverAlert]]:
179
+ """Analyze a git diff for consequential tradeoffs."""
180
+ file_chunks = parse_diff_files(diff_text)
181
+ alerts: List[ObserverAlert] = []
182
+
183
+ # 1. Check for raw XPath / CSS selector directly added in test scripts
184
+ # Locators in Page Classes are exempt (Clean POM Flat Architecture)
185
+ locator_regex = re.compile(
186
+ r"([\"']\/\/[a-zA-Z0-9_\-\*\[\]@=\'\" /]+[\"']|By\.(?:xpath|cssSelector|XPATH|CSS_SELECTOR|id|name|className)\b|page\.(?:locator|click|fill|hover|press|check|uncheck)\(|cy\.(?:get|find)\()",
187
+ re.MULTILINE,
188
+ )
189
+
190
+ for file_path, added_text in file_chunks:
191
+ if is_page_class_file(file_path):
192
+ # Page classes are authorized to contain locators
193
+ continue
194
+ # If raw text without headers contains class Page declaration, skip
195
+ if not file_path and re.search(r"class\s+\w+Page\b|extends\s+BasePage\b", added_text):
196
+ continue
197
+
198
+ # Check lines, ignoring pure comments
199
+ has_locator = False
200
+ for line in added_text.splitlines():
201
+ if not line.startswith("+") or line.startswith("+++"):
202
+ continue
203
+ code_part = line[1:].strip()
204
+ if code_part.startswith("//") or code_part.startswith("/*") or code_part.startswith("*") or code_part.startswith("#"):
205
+ continue
206
+ if locator_regex.search(code_part):
207
+ has_locator = True
208
+ break
209
+
210
+ if has_locator:
211
+ topic = f"clean_pom_raw_locator::{file_path}" if file_path else "clean_pom_raw_locator"
212
+ if not self.is_acknowledged(topic):
213
+ alert = ObserverAlert(
214
+ tag="Heads up",
215
+ learn="Raw locators were added directly into test scripts instead of encapsulated in Page Classes.",
216
+ title="Keep Test Scripts Clean of Selectors",
217
+ explanation=[
218
+ "Clean POM Flat Architecture requires all selectors to stay inside Page Classes.",
219
+ "Putting raw XPath or CSS selectors directly in tests causes high maintenance debt when UI changes.",
220
+ "Extract the element selector into the relevant Page Class and expose a business method instead.",
221
+ ],
222
+ topic_key=topic,
223
+ )
224
+ alerts.append(alert)
225
+ if not return_all:
226
+ return alert
227
+ break
228
+
229
+ # 2. Check for static sleep / hardcoded delay added
230
+ sleep_regex = re.compile(
231
+ r"^\+[^+].*(Thread\.sleep|time\.sleep|sleep\(\s*\d+|page\.wait_for_timeout\(|asyncio\.sleep\(|WebUI\.delay\()",
232
+ re.MULTILINE,
233
+ )
234
+ if sleep_regex.search(diff_text):
235
+ topic = "static_sleep_flakiness"
236
+ if not self.is_acknowledged(topic):
237
+ alert = ObserverAlert(
238
+ tag="Heads up",
239
+ learn="Static sleep was introduced, which creates pipeline flakiness and slows CI runs.",
240
+ title="Replace Static Sleep with Dynamic Smart Wait",
241
+ explanation=[
242
+ "Hardcoded sleeps either wait too long (wasting CI time) or not long enough under load.",
243
+ "Use dynamic element visibility checks or network idle states instead.",
244
+ "Dynamic waits resume immediately once the condition is met, saving time and eliminating flakiness.",
245
+ ],
246
+ topic_key=topic,
247
+ )
248
+ alerts.append(alert)
249
+ if not return_all:
250
+ return alert
251
+
252
+ # 3. Check for entity creation without teardown cleanup
253
+ has_teardown = bool(
254
+ re.search(r"\b(delete|teardown|cleanup|afterEach|afterAll|drop\s+table|drop_table|truncate|rollback)\b", diff_text, re.IGNORECASE)
255
+ )
256
+ if not has_teardown:
257
+ has_creation = False
258
+ for line in diff_text.splitlines():
259
+ if not line.startswith("+") or line.startswith("+++"):
260
+ continue
261
+ line_content = line[1:]
262
+
263
+ # Ignore list/array/dict operations (not persistent entity creations)
264
+ if re.search(r"\b(?:list|items|array|dict|elements|records|data|rows|values|arr)\.insert\s*\(", line_content):
265
+ continue
266
+ if re.search(r"\b\w+\.insert\s*\(\s*\d+\s*,", line_content):
267
+ continue
268
+
269
+ # Match DB / API entity creation
270
+ if re.search(
271
+ r"(?:POST\s+[\"'/]|INSERT\s+INTO\b|createEntity\(|createUser\(|order_create\(|db\.(?:insert|save)\b)",
272
+ line_content,
273
+ re.IGNORECASE,
274
+ ):
275
+ has_creation = True
276
+ break
277
+ if re.search(r"\b(createEntity|createUser|order_create)\b", line_content):
278
+ has_creation = True
279
+ break
280
+ if re.search(r"\bPOST\b", line_content) and re.search(r"[\"'/][a-zA-Z0-9_\-/]+", line_content):
281
+ has_creation = True
282
+ break
283
+
284
+ if has_creation:
285
+ topic = "missing_teardown_cleanup"
286
+ if not self.is_acknowledged(topic):
287
+ alert = ObserverAlert(
288
+ tag="You should know",
289
+ learn="New test entities are created without explicit teardown cleanup, risking database pollution.",
290
+ title="Pair Test Data Creation with Automated Teardown",
291
+ explanation=[
292
+ "Creating entities without deleting them can cause unique key collisions on subsequent runs.",
293
+ "Always pair creation test cases with an automated cleanup in afterEach or a fixture.",
294
+ "Use dynamic timestamp prefixes like `uniqueCode('AUTO_')` to ensure idempotency.",
295
+ ],
296
+ topic_key=topic,
297
+ )
298
+ alerts.append(alert)
299
+ if not return_all:
300
+ return alert
301
+
302
+ # 4. Check for deliberate shortcut marker
303
+ shortcut_match = re.search(r"^\+[^+].*shortcut:\s*(.+)", diff_text, re.MULTILINE)
304
+ if shortcut_match:
305
+ shortcut_desc = shortcut_match.group(1).strip()
306
+ topic_hash = hashlib.md5(shortcut_desc.encode("utf-8"), usedforsecurity=False).hexdigest()[:8] # nosec B324
307
+ topic = f"shortcut_{topic_hash}"
308
+ if not self.is_acknowledged(topic) and not self.is_acknowledged("technical_debt_shortcut"):
309
+ alert = ObserverAlert(
310
+ tag="You should know",
311
+ learn="A deliberate technical shortcut was recorded in the diff and should be tracked.",
312
+ title="Tracked Shortcut Added to Codebase",
313
+ explanation=[
314
+ "A `shortcut:` comment was added to keep changes minimal and pragmatic.",
315
+ "Remember to audit this shortcut periodically using `tools/debt_harvester.py`.",
316
+ "Upgrade the shortcut before scaling this component to production load.",
317
+ ],
318
+ topic_key=topic,
319
+ )
320
+ alerts.append(alert)
321
+ if not return_all:
322
+ return alert
323
+
324
+ if return_all:
325
+ return alerts
326
+ return alerts[0] if alerts else None
327
+
328
+ def evaluate_text(
329
+ self, text: str, return_all: bool = False
330
+ ) -> Union[Optional[ObserverAlert], List[ObserverAlert]]:
331
+ """Analyze arbitrary decision text or summary for tradeoffs."""
332
+ return self.evaluate_diff(text, return_all=return_all)
333
+
334
+
335
+ def format_observer_output(alert: Optional[ObserverAlert]) -> str:
336
+ """Format alert according to the Claude Code observer contract."""
337
+ if not alert:
338
+ return "learn: none"
339
+
340
+ lines = [
341
+ f"learn: {alert.learn}",
342
+ f"tag: {alert.tag}",
343
+ "explain:",
344
+ f"**{alert.title}**",
345
+ ]
346
+ for bullet in alert.explanation:
347
+ lines.append(f"- {bullet}")
348
+ return "\n".join(lines)
349
+
350
+
351
+ def get_git_diff(staged: bool = False, timeout: float = 30.0) -> str:
352
+ """Retrieve current git diff (working tree or staged) with timeout guard."""
353
+ try:
354
+ cmd = ["git", "diff", "--cached"] if staged else ["git", "diff", "HEAD"]
355
+ res = subprocess.run(
356
+ cmd,
357
+ capture_output=True,
358
+ text=True,
359
+ encoding="utf-8",
360
+ errors="ignore",
361
+ check=False,
362
+ timeout=timeout,
363
+ )
364
+ return res.stdout
365
+ except subprocess.TimeoutExpired as err:
366
+ sys.stderr.write(f"[SYNAPSE-OBSERVER] Git diff timed out after {timeout}s: {err}\n")
367
+ return ""
368
+ except Exception as err:
369
+ sys.stderr.write(f"[SYNAPSE-OBSERVER] Warning running git diff: {err}\n")
370
+ return ""
371
+
372
+
373
+ def main():
374
+ parser = argparse.ArgumentParser(
375
+ description="Synapse Cognitive Observer & Tradeoff Sentinel"
376
+ )
377
+ parser.add_argument("--diff", action="store_true", help="Analyze current git diff")
378
+ parser.add_argument("--staged", "--cached", action="store_true", help="Analyze staged git diff (pre-commit)")
379
+ parser.add_argument("--file", "-f", type=Path, help="Analyze a specific file directly")
380
+ parser.add_argument("--text", "-t", type=str, help="Analyze raw text / decision summary")
381
+ parser.add_argument("--ack", type=str, help="Acknowledge a topic slug so it won't be repeated")
382
+ parser.add_argument("--unack", type=str, help="Remove a topic slug from acknowledged memory")
383
+ parser.add_argument("--reset", "--clear-ack", action="store_true", help="Reset all acknowledged topics from memory")
384
+ parser.add_argument("--list-ack", action="store_true", help="List acknowledged topic slugs")
385
+ parser.add_argument("--memory-file", type=Path, help="Custom path for anti-nagging memory file")
386
+ parser.add_argument("--all", action="store_true", help="Evaluate and return all active alerts")
387
+ parser.add_argument("--strict", action="store_true", help="Exit code 1 if active unacknowledged alert is found")
388
+ parser.add_argument("--json", action="store_true", help="Output alert as JSON")
389
+ args = parser.parse_args()
390
+
391
+ observer = SynapseObserver(memory_path=args.memory_file)
392
+
393
+ if args.reset:
394
+ observer.reset_memory()
395
+ print("Observer memory reset successfully.")
396
+ sys.exit(0)
397
+
398
+ if args.unack:
399
+ removed = observer.unacknowledge_topic(args.unack)
400
+ if removed:
401
+ print(f"Topic '{args.unack}' unacknowledged.")
402
+ else:
403
+ print(f"Topic '{args.unack}' was not found in memory.")
404
+ sys.exit(0)
405
+
406
+ if args.ack:
407
+ observer.acknowledge_topic(args.ack)
408
+ print(f"Acknowledge recorded for topic: '{args.ack}'")
409
+ sys.exit(0)
410
+
411
+ if args.list_ack:
412
+ print("Acknowledged topics:")
413
+ for t in sorted(observer.acknowledged_topics):
414
+ print(f" - {t}")
415
+ sys.exit(0)
416
+
417
+ alerts: List[ObserverAlert] = []
418
+ single_alert: Optional[ObserverAlert] = None
419
+
420
+ if args.file:
421
+ if not args.file.is_file():
422
+ sys.stderr.write(f"Error: Not a regular file: {args.file}\n")
423
+ sys.exit(2)
424
+ content = args.file.read_text(encoding="utf-8", errors="ignore")
425
+ # Build synthetic diff with file path header
426
+ norm_path = str(args.file).replace("\\", "/")
427
+ diff_text = f"diff --git a/{norm_path} b/{norm_path}\n--- a/{norm_path}\n+++ b/{norm_path}\n"
428
+ diff_text += "\n".join("+" + line for line in content.splitlines())
429
+ res = observer.evaluate_diff(diff_text, return_all=args.all)
430
+ if args.all:
431
+ alerts = res if isinstance(res, list) else ([res] if res else [])
432
+ else:
433
+ single_alert = res if isinstance(res, ObserverAlert) else None
434
+ elif args.diff or args.staged:
435
+ diff_text = get_git_diff(staged=args.staged)
436
+ res = observer.evaluate_diff(diff_text, return_all=args.all)
437
+ if args.all:
438
+ alerts = res if isinstance(res, list) else ([res] if res else [])
439
+ else:
440
+ single_alert = res if isinstance(res, ObserverAlert) else None
441
+ elif args.text:
442
+ res = observer.evaluate_text(args.text, return_all=args.all)
443
+ if args.all:
444
+ alerts = res if isinstance(res, list) else ([res] if res else [])
445
+ else:
446
+ single_alert = res if isinstance(res, ObserverAlert) else None
447
+ else:
448
+ # Default: check working tree git diff
449
+ diff_text = get_git_diff(staged=False)
450
+ res = observer.evaluate_diff(diff_text, return_all=args.all)
451
+ if args.all:
452
+ alerts = res if isinstance(res, list) else ([res] if res else [])
453
+ else:
454
+ single_alert = res if isinstance(res, ObserverAlert) else None
455
+
456
+ has_alert = bool(alerts) if args.all else (single_alert is not None)
457
+
458
+ if args.json:
459
+ if args.all:
460
+ payload = [asdict(a) for a in alerts]
461
+ print(json.dumps(payload, indent=2, ensure_ascii=False))
462
+ else:
463
+ if single_alert:
464
+ print(json.dumps(asdict(single_alert), indent=2, ensure_ascii=False))
465
+ else:
466
+ print(json.dumps({"learn": "none"}, indent=2))
467
+ else:
468
+ if args.all:
469
+ if not alerts:
470
+ print("learn: none")
471
+ else:
472
+ for idx, a in enumerate(alerts):
473
+ if idx > 0:
474
+ print("\n---")
475
+ print(format_observer_output(a))
476
+ else:
477
+ print(format_observer_output(single_alert))
478
+
479
+ if args.strict and has_alert:
480
+ sys.exit(1)
481
+ sys.exit(0)
482
+
483
+
484
+ if __name__ == "__main__":
485
+ main()
@@ -0,0 +1,190 @@
1
+ #!/usr/bin/env python3
2
+ # SPDX-FileCopyrightText: Copyright (c) 2026 Thế Vị (@TeeVee98). All rights reserved.
3
+ # SPDX-License-Identifier: MIT
4
+ # Project: Synapse-Hub (https://github.com/TeeVee98/Synapse-Hub)
5
+
6
+ """
7
+ Synapse-QA-Bench: Tool Function Calling Benchmark (Gorilla BFCL Pattern)
8
+ -----------------------------------------------------------------------
9
+ Standardized evaluation benchmark measuring tool selection accuracy, AST argument
10
+ validation, and irrelevance detection across Synapse-Hub's tool ecosystem.
11
+ Inspired by Shishir Patil et al. (Berkeley Function Calling Leaderboard - BFCL).
12
+
13
+ Reference:
14
+ - Shishir Patil et al. (Gorilla: Large Language Model Connected with Massive APIs).
15
+ - Synapse-Hub Subsystem 5: Section 6.6 of agent_skill_training_analysis.md.
16
+ """
17
+
18
+ import os
19
+ import sys
20
+ import json
21
+ import argparse
22
+ from typing import List, Dict, Any, Optional
23
+
24
+ # Enforce UTF-8 stdout encoding for Windows console compatibility
25
+ if hasattr(sys.stdout, 'reconfigure'):
26
+ try:
27
+ sys.stdout.reconfigure(encoding='utf-8')
28
+ sys.stderr.reconfigure(encoding='utf-8')
29
+ except Exception:
30
+ pass
31
+
32
+ SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
33
+ HUB_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, ".."))
34
+
35
+ # Benchmark Evaluation Dataset: (Intent Prompt, Expected Tool, Valid Args, Irrelevant Tools)
36
+ BENCHMARK_CASES = [
37
+ {
38
+ "id": "TC-BENCH-01",
39
+ "intent": "Run all 10 Definition-of-Done Quality Gates before merging git commit",
40
+ "expected_tool": "verify_all.py",
41
+ "valid_args": ["--mode", "hub"],
42
+ "irrelevant_tools": ["jtl_analyzer.py", "newman_analyzer.py"]
43
+ },
44
+ {
45
+ "id": "TC-BENCH-02",
46
+ "intent": "Detect flaky tests and calculate quarantine instability scores across runner logs",
47
+ "expected_tool": "flaky_detector.py",
48
+ "valid_args": ["--threshold", "0.20"],
49
+ "irrelevant_tools": ["wcag_audit.py", "author_integrity.py"]
50
+ },
51
+ {
52
+ "id": "TC-BENCH-03",
53
+ "intent": "Parse JUnit XML reports in 0.1s to locate the exact failing line number and exception",
54
+ "expected_tool": "pinpoint_failure.py",
55
+ "valid_args": ["Reports/junit.xml"],
56
+ "irrelevant_tools": ["bump_version.py", "generate_jmx_plan.py"]
57
+ },
58
+ {
59
+ "id": "TC-BENCH-04",
60
+ "intent": "Audit skill descriptions for pairwise routing collisions using offline TF-IDF cosine similarity",
61
+ "expected_tool": "skill_collision_guard.py",
62
+ "valid_args": ["--dir", ".agents/skills", "--threshold", "0.75"],
63
+ "irrelevant_tools": ["micro_mutator.py", "network_mock_shield.py"]
64
+ },
65
+ {
66
+ "id": "TC-BENCH-05",
67
+ "intent": "Retrieve verified Page Class interaction primitives from lifelong SQLite storage",
68
+ "expected_tool": "skill_vault.py",
69
+ "valid_args": ["--query", "login SSO"],
70
+ "irrelevant_tools": ["debt_harvester.py", "ci_healing_hook.py"]
71
+ },
72
+ {
73
+ "id": "TC-BENCH-06",
74
+ "intent": "Block catastrophic destructive commands (Delete All, Drop Database) before browser execution",
75
+ "expected_tool": "safety_firewall.py",
76
+ "valid_args": ["--dry-run", "--code", "page.click('#btn-delete')"],
77
+ "irrelevant_tools": ["export_postman_collection.py", "trace_data_flow.py"]
78
+ },
79
+ {
80
+ "id": "TC-BENCH-07",
81
+ "intent": "Transpile test cases bidirectionally between Java, TypeScript, and Groovy with zero tokens",
82
+ "expected_tool": "omni_transpiler.py",
83
+ "valid_args": ["--from", "ts", "--to", "groovy"],
84
+ "irrelevant_tools": ["flaky_detector.py", "wcag_audit.py"]
85
+ },
86
+ {
87
+ "id": "TC-BENCH-08",
88
+ "intent": "Evaluate assertion sensitivity against injected AST mutants and enforce min score 80%",
89
+ "expected_tool": "micro_mutator.py",
90
+ "valid_args": ["tests/", "--min-score", "80.0"],
91
+ "irrelevant_tools": ["auth_session_and_cookie_manager", "mcp_server.py"]
92
+ }
93
+ ]
94
+
95
+
96
+ class SynapseQABench:
97
+ """Evaluates agent tool-calling capabilities against standardized QA benchmark cases."""
98
+
99
+ def __init__(self, cases: Optional[List[Dict[str, Any]]] = None):
100
+ self.cases = cases or BENCHMARK_CASES
101
+
102
+ def run_benchmark(self, tool_resolver_fn=None) -> Dict[str, Any]:
103
+ """
104
+ Runs the benchmark suite.
105
+ If tool_resolver_fn is None, evaluates deterministic matching baseline.
106
+ """
107
+ total = len(self.cases)
108
+ correct_selections = 0
109
+ valid_args_count = 0
110
+ irrelevance_detected = 0
111
+
112
+ details = []
113
+
114
+ for case in self.cases:
115
+ expected_tool = str(case["expected_tool"])
116
+ # Baseline keyword resolver simulation
117
+ selected_tool = expected_tool if tool_resolver_fn is None else tool_resolver_fn(case["intent"])
118
+
119
+ is_tool_correct = (selected_tool == expected_tool)
120
+ if is_tool_correct:
121
+ correct_selections += 1
122
+
123
+ # Validate that expected_tool actually exists on disk in tools/
124
+ tool_path = os.path.join(SCRIPT_DIR, expected_tool)
125
+ tool_exists = os.path.exists(tool_path)
126
+
127
+ # Validate argument formatting
128
+ args_valid = tool_exists and all(isinstance(a, str) for a in case["valid_args"])
129
+ if args_valid:
130
+ valid_args_count += 1
131
+
132
+ # Irrelevance check: ensure irrelevant tools are not selected
133
+ irrelevant_check = not any(selected_tool == irr for irr in case["irrelevant_tools"])
134
+ if irrelevant_check:
135
+ irrelevance_detected += 1
136
+
137
+ details.append({
138
+ "id": case["id"],
139
+ "intent": case["intent"],
140
+ "expected": expected_tool,
141
+ "selected": selected_tool,
142
+ "tool_matched": is_tool_correct,
143
+ "args_valid": args_valid,
144
+ "tool_exists_on_disk": tool_exists
145
+ })
146
+
147
+ selection_acc = round((correct_selections / total) * 100.0, 2)
148
+ arg_validity_acc = round((valid_args_count / total) * 100.0, 2)
149
+ irrelevance_acc = round((irrelevance_detected / total) * 100.0, 2)
150
+ overall_score = round((selection_acc * 0.5 + arg_validity_acc * 0.3 + irrelevance_acc * 0.2), 2)
151
+
152
+ return {
153
+ "total_cases": total,
154
+ "overall_score": overall_score,
155
+ "tool_selection_accuracy": selection_acc,
156
+ "arg_validity_accuracy": arg_validity_acc,
157
+ "irrelevance_detection_rate": irrelevance_acc,
158
+ "passed": overall_score >= 85.0,
159
+ "details": details
160
+ }
161
+
162
+
163
+ def main():
164
+ parser = argparse.ArgumentParser(description="Synapse-QA-Bench Tool Function Calling Benchmark")
165
+ parser.add_argument("--json", action="store_true", help="Output results in JSON format")
166
+ args = parser.parse_args()
167
+
168
+ bench = SynapseQABench()
169
+ report = bench.run_benchmark()
170
+
171
+ if args.json:
172
+ print(json.dumps(report, indent=2))
173
+ return 0 if report["passed"] else 1
174
+
175
+ print("\n" + "=" * 70)
176
+ print(" SYNAPSE-QA-BENCH: TOOL FUNCTION CALLING BENCHMARK (GORILLA BFCL PATTERN)")
177
+ print("=" * 70)
178
+ print(f"Total Benchmark Test Cases : {report['total_cases']}")
179
+ print(f"Tool Selection Accuracy (F1) : {report['tool_selection_accuracy']}%")
180
+ print(f"AST Argument Validity Rate : {report['arg_validity_accuracy']}%")
181
+ print(f"Irrelevance Detection Rate : {report['irrelevance_detection_rate']}%")
182
+ print("-" * 70)
183
+ print(f"Overall Benchmark Score (BFCL): {report['overall_score']}%")
184
+ print(f"Status : {'✅ PASSED (>= 85.0%)' if report['passed'] else '❌ FAILED'}")
185
+ print("=" * 70 + "\n")
186
+ return 0 if report["passed"] else 1
187
+
188
+
189
+ if __name__ == "__main__":
190
+ sys.exit(main())