@teevee98/synapse-hub 4.3.1 → 4.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.cursor/rules/agents.mdc +11 -11
- package/.cursor/rules/katalon.mdc +11 -0
- package/.cursor/rules/perf.mdc +10 -0
- package/.cursor/rules/playwright.mdc +10 -0
- package/.cursor/rules/pytest.mdc +10 -0
- package/.cursor/rules/synapse-qa.mdc +4 -2
- package/.cursorrules +3 -10
- package/.windsurfrules +3 -10
- package/AGENTS.md +4 -3
- package/CLAUDE.md +4 -5
- package/README.en.md +3 -3
- package/README.md +3 -3
- package/agentskills.json +15 -15
- package/{.agents/agents → ai-agents/personas}/api-engineer.md +9 -1
- package/{.agents/agents → ai-agents/personas}/failure-analyst.md +1 -1
- package/{.agents/agents → ai-agents/personas}/live-explorer.md +1 -1
- package/{.agents/agents → ai-agents/personas}/orchestrator.md +8 -15
- package/{.agents/agents → ai-agents/personas}/perf-engineer.md +11 -5
- package/{.agents/agents → ai-agents/personas}/pom-crafter.md +3 -9
- package/{.agents/agents → ai-agents/personas}/quality-gatekeeper.md +1 -1
- package/{.agents/agents → ai-agents/personas}/test-designer.md +2 -1
- package/{.agents/agents → ai-agents/personas}/test-runner.md +3 -4
- package/{.agents/rules → automation/playwright}/mandatory-playwright-live.md +5 -13
- package/{.agents/skills/01-web-automation → automation/playwright}/playwright-webapp-testing/SKILL.md +2 -0
- package/{.agents/skills/01-web-automation → automation/shared-utils}/clean-pom-crafter/SKILL.md +14 -3
- package/{.agents/skills/01-web-automation → automation/shared-utils}/e2e-testing-patterns/SKILL.md +12 -1
- package/{.agents/skills/01-web-automation → automation/shared-utils}/lam-script-generator/SKILL.md +2 -0
- package/docs/ARCHITECTURE.en.md +96 -3
- package/docs/ARCHITECTURE.md +87 -3
- package/docs/TOOL_CATALOG.md +17 -5
- package/docs/reports/AUDIT_REPORT.md +1038 -0
- package/docs/reports/PROJECT_HEALTH_REPORT.md +49 -0
- package/docs/reports/TECHNICAL_DEBT_LEDGER.md +22 -0
- package/package.json +8 -1
- package/plugin.json +1 -1
- package/{.agents/skills/02-test-design → requirement/spec-review}/review-spec/SKILL.md +8 -0
- package/scripts/install_global_plugin.ps1 +1 -1
- package/scripts/install_global_plugin.sh +1 -1
- package/skills_index.json +61 -0
- package/skills_manifest.json +88 -62
- package/{.agents/skills/08-code-quality → standards/coding-standards}/lean-code-ladder/SKILL.md +8 -1
- package/{.agents/skills/08-code-quality → standards/coding-standards}/surgical-coding-principles/SKILL.md +2 -0
- package/{.agents/skills/11-ai-and-prompting → standards/context-ai}/mipro-prompt-compiler/SKILL.md +8 -0
- package/synapse.config.json +1 -1
- package/templates/ai_bridges/CLAUDE.md +4 -5
- package/templates/ai_bridges/copilot-instructions.md +1 -0
- package/templates/ai_bridges/cursor_synapse_qa.mdc +4 -2
- package/templates/ai_bridges/windsurfrules +3 -10
- package/templates/test-plan/performance-test-plan-template.md +106 -0
- package/{.agents/skills/02-test-design → test-design/manual-rbt}/rbt-manual-testing/SKILL.md +11 -5
- package/{.agents/skills/07-data-and-state → test-design/test-data}/file-and-report-validator/SKILL.md +8 -1
- package/{.agents/skills/07-data-and-state → test-design/test-data}/synthetic-test-data-generator/SKILL.md +4 -1
- package/testing/api-contracts/api_testing_rules.md +74 -0
- package/{.agents/skills/06-healing-and-resilience → testing/healing-resilience}/agentless-qa-repair/SKILL.md +2 -0
- package/{.agents/skills/06-healing-and-resilience → testing/healing-resilience}/failure-autopsy-and-healing/SKILL.md +5 -11
- package/{.agents/skills/06-healing-and-resilience → testing/healing-resilience}/self-healing-locator/SKILL.md +2 -0
- package/testing/manual/manual-qa-and-rtm-bridge/SKILL.md +92 -0
- package/testing/manual/manual-qa-and-rtm-bridge/examples/sample_bug_and_rtm.json +38 -0
- package/{.agents/skills/04-performance-and-load → testing/performance}/backend-load-and-stress-patterns/SKILL.md +5 -2
- package/testing/performance/backend-load-and-stress-patterns/references/performance_test_plan_29119.md +106 -0
- package/{.agents/rules → testing/performance}/performance_testing_rules.md +27 -3
- package/{.agents/skills/04-performance-and-load → testing/performance}/web-vitals-and-performance-profiler/SKILL.md +75 -14
- package/{.agents/skills/05-security-and-compliance → testing/security-a11y}/security-dast-sentinel/SKILL.md +13 -2
- package/{.agents/rules → testing/security-a11y}/security_testing_rules.md +19 -0
- package/{.agents/skills/05-security-and-compliance → testing/security-a11y}/wcag-audit-patterns/SKILL.md +4 -1
- package/tools/.infer_baseline.json +44 -3
- package/tools/TOOLS_RUNBOOK.md +246 -4
- package/tools/TOOL_CATALOG.json +199 -19
- package/tools/TOOL_INDEX.json +160 -0
- package/tools/__init__.py +1 -1
- package/tools/ai_eval_sentinel.py +403 -0
- package/tools/assertion_optimizer.py +188 -0
- package/tools/asvs_auditor.py +410 -0
- package/tools/author_integrity.py +12 -1
- package/tools/benchmark_swarm_context.py +1 -1
- package/tools/browser_helpers.js +2 -2
- package/tools/catalogs/api_contract_testing.json +107 -0
- package/tools/catalogs/auxiliary_utilities.json +1536 -0
- package/tools/catalogs/benchmarking.json +23 -0
- package/tools/catalogs/ci_cd_packaging.json +46 -0
- package/tools/catalogs/code_generation_ast.json +70 -0
- package/tools/catalogs/failure_autopsy_healing.json +41 -0
- package/tools/catalogs/manual_testing_design.json +80 -0
- package/tools/catalogs/orchestration.json +25 -0
- package/tools/catalogs/performance_testing.json +71 -0
- package/tools/catalogs/quality_gates.json +65 -0
- package/tools/catalogs/sandboxing_isolation.json +42 -0
- package/tools/catalogs/security.json +72 -0
- package/tools/catalogs/setup_onboarding.json +26 -0
- package/tools/catalogs/testing_utilities.json +111 -0
- package/tools/chaos_resilience_probe.py +362 -0
- package/tools/check_env.py +16 -3
- package/tools/check_version_sync.py +114 -63
- package/tools/cldr_l10n_verifier.py +309 -0
- package/tools/cli.py +971 -3
- package/tools/core/circuit_breaker.py +106 -0
- package/tools/core/token_estimator.py +56 -7
- package/tools/cvss_calculator.py +412 -0
- package/tools/dashboard_generator.py +12 -12
- package/tools/etl_data_reconciler.py +460 -0
- package/tools/generate_jmx_plan.py +12 -4
- package/tools/generate_tool_catalog.py +60 -2
- package/tools/jtl_analyzer.py +23 -1
- package/tools/mcp_docx_server.py +1 -1
- package/tools/mcp_excel_server.py +1 -1
- package/tools/mcp_server.py +5 -1
- package/tools/mobile_resilience_sentinel.py +273 -0
- package/tools/observability_tester.py +235 -0
- package/tools/payment_interop_simulator.py +337 -0
- package/tools/pre_push_guard.py +306 -0
- package/tools/project_state.py +17 -10
- package/tools/regulatory_compliance.py +336 -0
- package/tools/reversible_sandbox.py +7 -2
- package/tools/rules_router.py +6 -4
- package/tools/runaway_stopper.py +19 -111
- package/tools/setup_project.py +1 -1
- package/tools/skill_canonical_digest.py +2 -2
- package/tools/skillspector.py +78 -10
- package/tools/sync_agent_rules.py +15 -27
- package/tools/sync_skills_manifest.py +38 -4
- package/tools/tmmi_auditor.py +179 -0
- package/tools/ui_server.py +1 -1
- package/tools/update_sentinel.py +4 -4
- package/tools/validate_host_interop.py +3 -1
- package/tools/validate_skills_schema.py +49 -2
- package/tools/vpat_exporter.py +428 -0
- package/tools/welcome_page.py +4 -4
- package/.agents/rules/stack-auto-router.md +0 -30
- package/.agents/skills/09-ci-and-devops/github-actions-templates/SKILL.md +0 -114
- package/.agents/skills/09-ci-and-devops/github-actions-templates/examples/matrix_test_pipeline.yml +0 -9
- package/.agents/skills/09-ci-and-devops/gitlab-ci-templates/SKILL.md +0 -123
- package/.agents/skills/09-ci-and-devops/gitlab-ci-templates/examples/sample_gitlab_ci.yml +0 -17
- package/.agents/skills/09-ci-and-devops/jenkins-pipeline-templates/SKILL.md +0 -133
- package/.agents/skills/09-ci-and-devops/jenkins-pipeline-templates/examples/Jenkinsfile.example +0 -21
- package/.cursor/rules/language.mdc +0 -7
- package/templates/ci/github-actions-synapse.yml +0 -95
- package/templates/ci/gitlab-ci-synapse.yml +0 -16
- package/tools/build_consolidated_report.py +0 -1643
- package/tools/create_mock_docx.py +0 -107
- /package/{.agents → ai-agents/hooks}/hooks.json +0 -0
- /package/{.agents → ai-agents}/hooks/session_start.py +0 -0
- /package/{.agents/skills/10-agent-orchestration → ai-agents/orchestration}/actor-qa-swarm/SKILL.md +0 -0
- /package/{.agents/skills/10-agent-orchestration → ai-agents/orchestration}/actor-qa-swarm/examples/broker_sharding_sample.json +0 -0
- /package/{.agents/skills/10-agent-orchestration → ai-agents/orchestration}/agent-orchestration-multi-agent-optimize/SKILL.md +0 -0
- /package/{.agents/skills/10-agent-orchestration → ai-agents/orchestration}/agent-orchestration-multi-agent-optimize/examples/swarm_workload_config.json +0 -0
- /package/{.agents/skills/10-agent-orchestration → ai-agents/orchestration}/event-stream-qa-orchestrator/SKILL.md +0 -0
- /package/{.agents/skills/10-agent-orchestration → ai-agents/orchestration}/event-stream-qa-orchestrator/examples/timeline_sample.jsonl +0 -0
- /package/{.agents/skills/10-agent-orchestration → ai-agents/orchestration}/persistent-planning-files/SKILL.md +0 -0
- /package/{.agents/skills/10-agent-orchestration → ai-agents/orchestration}/persistent-planning-files/examples/task_plan_example.md +0 -0
- /package/{.agents/skills/10-agent-orchestration → ai-agents/orchestration}/qa-memory-os/SKILL.md +0 -0
- /package/{.agents/skills/10-agent-orchestration → ai-agents/orchestration}/qa-memory-os/examples/archival_memory_sample.json +0 -0
- /package/{.agents → ai-agents}/workflows/graphify.md +0 -0
- /package/{.agents → ai-agents}/workflows/test-impact.md +0 -0
- /package/{.agents → ai-agents}/workflows/verify.md +0 -0
- /package/{.agents/rules → automation/cypress}/cypress_rules.md +0 -0
- /package/{.agents/skills/01-web-automation → automation/katalon}/katalon-cli-runner/SKILL.md +0 -0
- /package/{.agents/skills/01-web-automation → automation/katalon}/katalon-cli-runner/examples/headless_kre_exec.sh +0 -0
- /package/{.agents/skills/01-web-automation → automation/katalon}/katalon-cli-runner/references/groovy-patterns.md +0 -0
- /package/{.agents/skills/01-web-automation → automation/katalon}/katalon-cli-runner/references/katalon-api.md +0 -0
- /package/{.agents/skills/01-web-automation → automation/katalon}/katalon-cli-runner/references/runner_commands.md +0 -0
- /package/{.agents/skills/01-web-automation → automation/mobile}/mobile-appium-som/SKILL.md +0 -0
- /package/{.agents/skills/01-web-automation → automation/mobile}/mobile-appium-som/examples/BaseScreen.java +0 -0
- /package/{.agents/skills/01-web-automation → automation/mobile}/mobile-appium-som/examples/LoginScreen.java +0 -0
- /package/{.agents/skills/01-web-automation → automation/mobile}/mobile-appium-som/examples/TC_MobileLogin.java +0 -0
- /package/{.agents/skills/01-web-automation → automation/mobile}/mobile-maestro-detox-patterns/SKILL.md +0 -0
- /package/{.agents/skills/01-web-automation → automation/mobile}/mobile-maestro-detox-patterns/examples/sample_flow.yaml +0 -0
- /package/{.agents/rules → automation/mobile}/mobile_appium_rules.md +0 -0
- /package/{.agents/skills/01-web-automation → automation/playwright}/playwright-live-interaction/SKILL.md +0 -0
- /package/{.agents/skills/01-web-automation → automation/playwright}/playwright-live-interaction/examples/live_probe_script.js +0 -0
- /package/{.agents/skills/01-web-automation → automation/playwright}/playwright-live-interaction/references/advanced_superpowers.md +0 -0
- /package/{.agents/skills/01-web-automation → automation/playwright}/playwright-live-interaction/references/compact_accessibility_snapshots.md +0 -0
- /package/{.agents/skills/01-web-automation → automation/playwright}/playwright-live-interaction/references/recipes.md +0 -0
- /package/{.agents/skills/01-web-automation → automation/playwright}/playwright-live-interaction/references/swarm_orchestration.md +0 -0
- /package/{.agents/skills/01-web-automation → automation/playwright}/playwright-webapp-testing/examples/server_lifecycle_playwright.py +0 -0
- /package/{.agents/skills/01-web-automation → automation/shared-utils}/auth-session-and-cookie-manager/SKILL.md +0 -0
- /package/{.agents/skills/01-web-automation → automation/shared-utils}/auth-session-and-cookie-manager/examples/multi_role_auth_matrix.md +0 -0
- /package/{.agents/skills/01-web-automation → automation/shared-utils}/auth-session-and-cookie-manager/examples/storage_state_sample.json +0 -0
- /package/{.agents/skills/01-web-automation → automation/shared-utils}/bats-testing-patterns/SKILL.md +0 -0
- /package/{.agents/skills/01-web-automation → automation/shared-utils}/bats-testing-patterns/examples/cli_runner_test.bats +0 -0
- /package/{.agents/skills/01-web-automation → automation/shared-utils}/bats-testing-patterns/resources/implementation-playbook.md +0 -0
- /package/{.agents/skills/01-web-automation → automation/shared-utils}/clean-pom-crafter/examples/standard_test_case.groovy +0 -0
- /package/{.agents/skills/01-web-automation → automation/shared-utils}/clean-pom-crafter/references/pom_patterns.groovy +0 -0
- /package/{.agents/skills/01-web-automation → automation/shared-utils}/e2e-testing-patterns/examples/playwright_sharded_config.ts +0 -0
- /package/{.agents/skills/01-web-automation → automation/shared-utils}/e2e-testing-patterns/resources/implementation-playbook.md +0 -0
- /package/{.agents/skills/01-web-automation → automation/shared-utils}/lam-script-generator/examples/user_story_sample.md +0 -0
- /package/{.agents/skills/03-api-and-contracts → requirement/spec-review}/api-spec-to-e2e-matrix/SKILL.md +0 -0
- /package/{.agents/skills/03-api-and-contracts → requirement/spec-review}/api-spec-to-e2e-matrix/examples/sample_openapi.json +0 -0
- /package/{.agents/skills/02-test-design → requirement/spec-review}/architecture-decision-records/SKILL.md +0 -0
- /package/{.agents/skills/02-test-design → requirement/spec-review}/architecture-decision-records/examples/0001-record-architecture-decisions.md +0 -0
- /package/{.agents/skills/02-test-design → requirement/spec-review}/review-spec/examples/pocock_grilling_decision_rubric.md +0 -0
- /package/{.agents/skills/02-test-design → requirement/spec-review}/review-spec/examples/sample_spec_review.md +0 -0
- /package/{.agents → requirement}/workflows/analyze_requirement_document.md +0 -0
- /package/{.agents → requirement}/workflows/generate_requirements_from_website.md +0 -0
- /package/{.agents/rules → standards/coding-standards}/author-integrity-guard.md +0 -0
- /package/{.agents/skills/08-code-quality → standards/coding-standards}/code-refactoring-refactor-clean/SKILL.md +0 -0
- /package/{.agents/skills/08-code-quality → standards/coding-standards}/code-refactoring-refactor-clean/examples/refactored_page_sample.groovy +0 -0
- /package/{.agents/skills/08-code-quality → standards/coding-standards}/code-refactoring-refactor-clean/examples/refactored_playwright_pom.ts +0 -0
- /package/{.agents/skills/08-code-quality → standards/coding-standards}/code-refactoring-refactor-clean/resources/implementation-playbook.md +0 -0
- /package/{.agents/rules → standards/coding-standards}/communication-style.md +0 -0
- /package/{.agents/rules → standards/coding-standards}/git-push-protocol.md +0 -0
- /package/{.agents/skills/08-code-quality → standards/coding-standards}/graphify-impact-analyzer/SKILL.md +0 -0
- /package/{.agents/skills/08-code-quality → standards/coding-standards}/graphify-impact-analyzer/examples/dependency_blast_radius.json +0 -0
- /package/{.agents/skills/08-code-quality → standards/coding-standards}/lean-code-ladder/examples/ladder_walkthrough.md +0 -0
- /package/{.agents/skills/08-code-quality → standards/coding-standards}/repo-quality-audit/SKILL.md +0 -0
- /package/{.agents/skills/08-code-quality → standards/coding-standards}/repo-quality-audit/examples/sample_audit_walkthrough.md +0 -0
- /package/{.agents/skills/08-code-quality → standards/coding-standards}/repo-quality-audit/examples/sample_repo_audit.md +0 -0
- /package/{.agents/skills/08-code-quality → standards/coding-standards}/senior-code-review/SKILL.md +0 -0
- /package/{.agents/skills/08-code-quality → standards/coding-standards}/senior-code-review/examples/sample_review_walkthrough.md +0 -0
- /package/{.agents/skills/08-code-quality → standards/coding-standards}/surgical-coding-principles/examples/surgical_vs_scope_creep.md +0 -0
- /package/{.agents/skills/08-code-quality → standards/coding-standards}/technical-debt-harvester/SKILL.md +0 -0
- /package/{.agents/skills/08-code-quality → standards/coding-standards}/technical-debt-harvester/examples/sample_debt_ledger.md +0 -0
- /package/{.agents/skills/08-code-quality → standards/coding-standards}/technical-debt-harvester/examples/sample_ledger_output.md +0 -0
- /package/{.agents/rules → standards/context-ai}/context-engineering.md +0 -0
- /package/{.agents/skills/11-ai-and-prompting → standards/context-ai}/meta-prompt-crafter/SKILL.md +0 -0
- /package/{.agents/skills/11-ai-and-prompting → standards/context-ai}/meta-prompt-crafter/examples/qa_prompt_template.md +0 -0
- /package/{.agents/skills/11-ai-and-prompting → standards/context-ai}/mipro-prompt-compiler/examples/compiled_module_sample.json +0 -0
- /package/{.agents/skills/11-ai-and-prompting → standards/context-ai}/mipro-prompt-compiler/examples/dspy_assertion_optimizer_case.md +0 -0
- /package/{.agents/skills/11-ai-and-prompting → standards/context-ai}/qa-aci-interface/SKILL.md +0 -0
- /package/{.agents/skills/11-ai-and-prompting → standards/context-ai}/qa-aci-interface/examples/surgical_code_window_walkthrough.md +0 -0
- /package/{.agents/skills/11-ai-and-prompting → standards/context-ai}/qa-aci-interface/examples/surgical_edit_sample.json +0 -0
- /package/{.agents/skills/11-ai-and-prompting → standards/context-ai}/token-and-context-optimizer/SKILL.md +0 -0
- /package/{.agents/skills/11-ai-and-prompting → standards/context-ai}/token-and-context-optimizer/examples/distilled_log_sample.txt +0 -0
- /package/{.agents/skills/11-ai-and-prompting → standards/context-ai}/token-and-context-optimizer/references/context_engineering_guide.md +0 -0
- /package/{.agents/rules → standards/qa-process}/graphify.md +0 -0
- /package/{.agents/rules → standards/qa-process}/lessons-learned.md +0 -0
- /package/{.agents/rules → standards/qa-process}/skill-precedence.md +0 -0
- /package/{.agents/rules → standards/quality-gates}/systematic-debugging.md +0 -0
- /package/{.agents/templates → templates/test-report}/completion_report.md +0 -0
- /package/{.agents/skills/02-test-design → test-design/bdd-gherkin}/gherkin-to-suite/SKILL.md +0 -0
- /package/{.agents/skills/02-test-design → test-design/bdd-gherkin}/gherkin-to-suite/examples/checkout.feature +0 -0
- /package/{.agents/skills/02-test-design → test-design/bdd-gherkin}/gherkin-to-suite/examples/suite_report_sample.json +0 -0
- /package/{.agents → test-design/bdd-gherkin}/workflows/generate_cross_module_test_plan.md +0 -0
- /package/{.agents → test-design/bdd-gherkin}/workflows/generate_testcases_from_requirements.md +0 -0
- /package/{.agents/skills/02-test-design → test-design/exploratory}/tot-exploratory-testing/SKILL.md +0 -0
- /package/{.agents/skills/02-test-design → test-design/exploratory}/tot-exploratory-testing/examples/payment_gateway_tot_tree.json +0 -0
- /package/{.agents/skills/02-test-design → test-design/exploratory}/tot-exploratory-testing/examples/tot_graph_sample.json +0 -0
- /package/{.agents/skills/02-test-design → test-design/manual-rbt}/rbt-manual-testing/examples/checkout_rbt_manual_suite.md +0 -0
- /package/{.agents/skills/02-test-design → test-design/manual-rbt}/rbt-manual-testing/references/field_validation_checklist.md +0 -0
- /package/{.agents → test-design/manual-rbt}/workflows/export_testcases_excel.md +0 -0
- /package/{.agents → test-design/manual-rbt}/workflows/generate_manual_testcases_rbt.md +0 -0
- /package/{.agents/skills/07-data-and-state → test-design/test-data}/db-state-verifier-and-seeder/SKILL.md +0 -0
- /package/{.agents/skills/07-data-and-state → test-design/test-data}/db-state-verifier-and-seeder/examples/db_fixture_seed.json +0 -0
- /package/{.agents/skills/07-data-and-state → test-design/test-data}/file-and-report-validator/examples/sample_export_schema.json +0 -0
- /package/{.agents/skills/07-data-and-state → test-design/test-data}/synthetic-test-data-generator/examples/idempotent_data_recipe.json +0 -0
- /package/{.agents → test-design/test-data}/workflows/generate_combinatorial_test_data.md +0 -0
- /package/{.agents/skills/03-api-and-contracts → testing/api-contracts}/api-network-mocking-interception/SKILL.md +0 -0
- /package/{.agents/skills/03-api-and-contracts → testing/api-contracts}/api-network-mocking-interception/examples/network_route_stub.json +0 -0
- /package/{.agents/skills/03-api-and-contracts → testing/api-contracts}/consumer-driven-contract-testing/SKILL.md +0 -0
- /package/{.agents/skills/03-api-and-contracts → testing/api-contracts}/consumer-driven-contract-testing/examples/order_service_pact.json +0 -0
- /package/{.agents/skills/06-healing-and-resilience → testing/healing-resilience}/agentless-qa-repair/examples/healing_report_sample.json +0 -0
- /package/{.agents/skills/06-healing-and-resilience → testing/healing-resilience}/agentless-qa-repair/examples/three_phase_repair_workflow.md +0 -0
- /package/{.agents/skills/06-healing-and-resilience → testing/healing-resilience}/antigravity-test-sentinel/SKILL.md +0 -0
- /package/{.agents/skills/06-healing-and-resilience → testing/healing-resilience}/antigravity-test-sentinel/examples/dod_gate_policy.json +0 -0
- /package/{.agents/skills/06-healing-and-resilience → testing/healing-resilience}/antigravity-test-sentinel/examples/sprint_dod_enforcement.md +0 -0
- /package/{.agents/skills/06-healing-and-resilience → testing/healing-resilience}/failure-autopsy-and-healing/examples/closed_loop_healing_walkthrough.md +0 -0
- /package/{.agents/skills/06-healing-and-resilience → testing/healing-resilience}/failure-autopsy-and-healing/examples/sample_junit_failure.xml +0 -0
- /package/{.agents/skills/06-healing-and-resilience → testing/healing-resilience}/failure-autopsy-and-healing/references/healing_matrix.md +0 -0
- /package/{.agents/skills/06-healing-and-resilience → testing/healing-resilience}/failure-autopsy-and-healing/references/qa_investigation_protocol.md +0 -0
- /package/{.agents/skills/06-healing-and-resilience → testing/healing-resilience}/failure-autopsy-and-healing/scripts/triage_failures.sh +0 -0
- /package/{.agents/skills/06-healing-and-resilience → testing/healing-resilience}/flaky-test-sentinel-and-quarantine/SKILL.md +0 -0
- /package/{.agents/skills/06-healing-and-resilience → testing/healing-resilience}/flaky-test-sentinel-and-quarantine/examples/quarantine_registry.json +0 -0
- /package/{.agents/skills/06-healing-and-resilience → testing/healing-resilience}/self-healing-locator/examples/aom_diff_sample.json +0 -0
- /package/{.agents/skills/06-healing-and-resilience → testing/healing-resilience}/systematic-debugging/SKILL.md +0 -0
- /package/{.agents/skills/06-healing-and-resilience → testing/healing-resilience}/systematic-debugging/examples/four_phase_case.md +0 -0
- /package/{.agents/skills/06-healing-and-resilience → testing/healing-resilience}/test-driven-development/SKILL.md +0 -0
- /package/{.agents/skills/06-healing-and-resilience → testing/healing-resilience}/test-driven-development/examples/red_green_refactor_qa.md +0 -0
- /package/{.agents/skills/06-healing-and-resilience → testing/healing-resilience}/verification-before-completion/SKILL.md +0 -0
- /package/{.agents/skills/06-healing-and-resilience → testing/healing-resilience}/verification-before-completion/examples/evidence_log_example.md +0 -0
- /package/{.agents/skills/06-healing-and-resilience → testing/healing-resilience}/visual-regression-sentinel/SKILL.md +0 -0
- /package/{.agents/skills/06-healing-and-resilience → testing/healing-resilience}/visual-regression-sentinel/examples/visual_mask_rules.json +0 -0
- /package/{.agents/skills/04-performance-and-load → testing/performance}/backend-load-and-stress-patterns/examples/k6_load_test_sample.js +0 -0
- /package/{.agents/skills/04-performance-and-load → testing/performance}/backend-load-and-stress-patterns/references/k6_load_template.js +0 -0
- /package/{.agents/skills/04-performance-and-load → testing/performance}/lean-impact-scoreboard/SKILL.md +0 -0
- /package/{.agents/skills/04-performance-and-load → testing/performance}/lean-impact-scoreboard/examples/sample_benchmark_gain.md +0 -0
- /package/{.agents/skills/04-performance-and-load → testing/performance}/lean-impact-scoreboard/examples/sample_scoreboard.md +0 -0
- /package/{.agents/skills/04-performance-and-load → testing/performance}/web-vitals-and-performance-profiler/examples/cwv_budget_policy.json +0 -0
- /package/{.agents/skills/04-performance-and-load → testing/performance}/web-vitals-and-performance-profiler/examples/playwright_cwv_profiler_fixture.ts +0 -0
- /package/{.agents/skills/05-security-and-compliance → testing/security-a11y}/security-dast-sentinel/examples/dast_scan_config.json +0 -0
- /package/{.agents/skills/05-security-and-compliance → testing/security-a11y}/security-dast-sentinel/references/AI-AND-LLM.md +0 -0
- /package/{.agents/skills/05-security-and-compliance → testing/security-a11y}/security-dast-sentinel/references/ATTACK-CLASSES.md +0 -0
- /package/{.agents/skills/05-security-and-compliance → testing/security-a11y}/security-dast-sentinel/references/dast_recipes.ts +0 -0
- /package/{.agents/skills/05-security-and-compliance → testing/security-a11y}/skillspector-security-guard/SKILL.md +0 -0
- /package/{.agents/skills/05-security-and-compliance → testing/security-a11y}/skillspector-security-guard/examples/audit_threat_vectors_report.json +0 -0
- /package/{.agents/skills/05-security-and-compliance → testing/security-a11y}/skillspector-security-guard/examples/safe_mcp_config.json +0 -0
- /package/{.agents/skills/05-security-and-compliance → testing/security-a11y}/wcag-audit-patterns/examples/wcag_compliance_checklist.json +0 -0
- /package/{.agents/skills/05-security-and-compliance → testing/security-a11y}/wcag-audit-patterns/resources/implementation-playbook.md +0 -0
- /package/{.agents/skills/01-web-automation → tools/benchmarks-tests}/web-agent-gym-benchmarking/SKILL.md +0 -0
- /package/{.agents/skills/01-web-automation → tools/benchmarks-tests}/web-agent-gym-benchmarking/examples/benchmark_report_sample.json +0 -0
- /package/{.agents/skills/01-web-automation → tools/benchmarks-tests}/web-agent-gym-benchmarking/examples/enterprise_tasks_suite.json +0 -0
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
{
|
|
2
|
+
"orchestrator": {
|
|
3
|
+
"python tools/_openapi_parser.py": "Shared OpenAPI & Swagger Speci..",
|
|
4
|
+
"python tools/agent_canary.py": "ENTERPRISE AUTONOMOUS AGENT Ca..",
|
|
5
|
+
"python tools/agentless_patcher.py": "Agentless 3-Stage Deterministi..",
|
|
6
|
+
"python tools/ai_eval_sentinel.py": "Enterprise RAG & LLM Evaluatio..",
|
|
7
|
+
"python tools/api_fuzzer.py": "Enterprise Autonomous API Fuzz..",
|
|
8
|
+
"python tools/assertion_optimizer.py": "Unified Assertion Optimization..",
|
|
9
|
+
"python tools/ast_cache.py": "AST Dataflow Pre-Validation Ca..",
|
|
10
|
+
"python tools/asvs_auditor.py": "OWASP Application Security Ver..",
|
|
11
|
+
"python tools/author_integrity.py": "Author Attribution & Repositor..",
|
|
12
|
+
"python tools/auto_heal_reflexion.py": "Multi-Trial Episodic Reflexion..",
|
|
13
|
+
"python tools/autogen_qa_broker.py": "Asynchronous Actor QA Message ..",
|
|
14
|
+
"python tools/benchmark_swarm_context.py": "Empirical context token reduct..",
|
|
15
|
+
"python tools/break_ui_generator.py": "Edge-Case UI Stress Testing Da..",
|
|
16
|
+
"python tools/browsergym_eval.py": "BrowserGym Web Agent Benchmark..",
|
|
17
|
+
"python tools/bug_reporter.py": "Auto-Bug Reporter & Root Cause..",
|
|
18
|
+
"python tools/bump_version.py": "Atomic Universal Version Bumpe..",
|
|
19
|
+
"python tools/cache_resolver.py": "Universal Local Semantic Memor..",
|
|
20
|
+
"python tools/chaos_resilience_probe.py": "Enterprise Chaos Engineering &..",
|
|
21
|
+
"python tools/check_env.py": "==============================..",
|
|
22
|
+
"python tools/check_version_sync.py": "Version Synchronization Sentin..",
|
|
23
|
+
"python tools/ci_healing_hook.py": "CI/CD Silent Self-Healing Pipe..",
|
|
24
|
+
"python tools/cldr_l10n_verifier.py": "Enterprise Unicode CLDR & L10n..",
|
|
25
|
+
"python tools/cli.py": "Enterprise Agent-QA CLI Entryp..",
|
|
26
|
+
"python tools/config_decoupler.py": "Environment & Secret Decouplin..",
|
|
27
|
+
"python tools/contract_validator.py": "Swarm Data Contract Validator ..",
|
|
28
|
+
"python tools/count_tokens.py": "Cross-Platform Token & Cost Au..",
|
|
29
|
+
"python tools/coverage_reporter.py": "Standardized Coverage Metrics ..",
|
|
30
|
+
"python tools/cvss_calculator.py": "Enterprise Common Vulnerabilit..",
|
|
31
|
+
"python tools/dashboard_generator.py": "Standalone One-Page Enterprise..",
|
|
32
|
+
"python tools/debt_harvester.py": "Technical Debt & Shortcut Harv..",
|
|
33
|
+
"python tools/diagram_architect.py": "Standalone Self-Contained SVG ..",
|
|
34
|
+
"python tools/distill_log.py": "Enterprise Multi-Framework Log..",
|
|
35
|
+
"python tools/distill_skill.py": "Dual-Skill Distillation Engine",
|
|
36
|
+
"python tools/dspy_qa_signatures.py": "Declarative QA Signature Compi..",
|
|
37
|
+
"python tools/e2e_flow_prober.py": "Enterprise E2E Flow Prober & X..",
|
|
38
|
+
"python tools/etl_data_reconciler.py": "Enterprise Data Quality & ETL ..",
|
|
39
|
+
"python tools/flaky_detector.py": "Enterprise Flaky Test Detector..",
|
|
40
|
+
"python tools/fsm_enforcer.py": "5-Stage QA Multi-Agent Finite ..",
|
|
41
|
+
"python tools/fsm_middleware.py": "Runtime FSM Interceptor & Tool..",
|
|
42
|
+
"python tools/fuzz_payloads.py": "Enterprise Edge-Case Fuzzing &..",
|
|
43
|
+
"python tools/generate_allure_report.py": "Enterprise Allure HTML Report ..",
|
|
44
|
+
"python tools/generate_sbom.py": "Standard SPDX 2.3 JSON Softwar..",
|
|
45
|
+
"python tools/generate_tc_from_story.py": "Large Action Model (LAM) Scrip..",
|
|
46
|
+
"python tools/generate_test_reports.py": "Enterprise Test Report Aggrega..",
|
|
47
|
+
"python tools/generate_tool_catalog.py": "Enterprise Tooling Ecosystem C..",
|
|
48
|
+
"python tools/git_pre_commit_guard.py": "Enterprise Git Pre-Commit Arch..",
|
|
49
|
+
"python tools/git_worktree_manager.py": "Ephemeral Git Worktree Manager..",
|
|
50
|
+
"python tools/goldfish_review.py": "Zero-Context Test Specificatio..",
|
|
51
|
+
"python tools/heal_pr_automator.py": "Enterprise Self-Healing Confid..",
|
|
52
|
+
"python tools/heuristic_locator_matcher.py": "3-Layer Heuristic Locator Matc..",
|
|
53
|
+
"python tools/hub_publisher.py": "Autonomous 1-click 5-step rele..",
|
|
54
|
+
"python tools/humanize_linter.py": "Humanize & Anti-AI Slop Linter..",
|
|
55
|
+
"python tools/infer_linter.py": "Universal QA Static Defect & A..",
|
|
56
|
+
"python tools/init_config.py": "Standardized Project Configura..",
|
|
57
|
+
"python tools/jira_sync.py": "Jira / Xray / Zephyr Enterpris..",
|
|
58
|
+
"python tools/lint_ai_docs.py": "Enterprise AI Documentation & ..",
|
|
59
|
+
"python tools/lookup_tc.py": "Enterprise QA: Fast Test Case ..",
|
|
60
|
+
"python tools/lookup_xpath.py": "Enterprise QA: Fast XPath / Se..",
|
|
61
|
+
"python tools/matrix_sharder.py": "Enterprise Duration-Balanced T..",
|
|
62
|
+
"python tools/mcp_docx_server.py": "Enterprise Cross-Platform MCP ..",
|
|
63
|
+
"python tools/mcp_excel_server.py": "Enterprise Cross-Platform MCP ..",
|
|
64
|
+
"python tools/mcp_server.py": "Enterprise Model Context Proto..",
|
|
65
|
+
"python tools/md_to_xlsx.py": "Markdown to Excel (.xlsx) Ente..",
|
|
66
|
+
"python tools/mechanical_ui_gate.py": "Headless Chrome Mechanical Lay..",
|
|
67
|
+
"python tools/mipro_assertion_tuner.py": "MIPROv2 Prompt Assertion Tuner..",
|
|
68
|
+
"python tools/mobile_resilience_sentinel.py": "Mobile Hardware Interrupts, MA..",
|
|
69
|
+
"python tools/observability_tester.py": "OpenTelemetry Trace & Log Hygi..",
|
|
70
|
+
"python tools/openxml_guard.py": "OpenXML Security & Protocol De..",
|
|
71
|
+
"python tools/output_formatter.py": "Enterprise Standardized Output..",
|
|
72
|
+
"python tools/package_release.py": "Enterprise release packager pr..",
|
|
73
|
+
"python tools/pairwise_generator.py": "Pairwise (2-Way) Combinatorial..",
|
|
74
|
+
"python tools/payment_interop_simulator.py": "ISO 8583 & Financial Gateway I..",
|
|
75
|
+
"python tools/pentest_scope_guard.py": "Enterprise Pentest Scope & DAS..",
|
|
76
|
+
"python tools/pre_push_guard.py": "Pre-Push Quality & Anti-Halluc..",
|
|
77
|
+
"python tools/project_state.py": "Enterprise Project State Machi..",
|
|
78
|
+
"python tools/qa_event_stream.py": "Immutable QA Event Stream Engi..",
|
|
79
|
+
"python tools/regulatory_compliance.py": "Enterprise Regulatory & Compli..",
|
|
80
|
+
"python tools/repo_map.py": "Enterprise Repository Map Comp..",
|
|
81
|
+
"python tools/rules_router.py": "Progressive Rules Router & Con..",
|
|
82
|
+
"python tools/runaway_stopper.py": "Agent Runaway & Token Loop Sen..",
|
|
83
|
+
"python tools/security_coverage_ledger.py": "Mathematical Security Coverage..",
|
|
84
|
+
"python tools/senior_code_review.py": "Senior Developer Quality & Lea..",
|
|
85
|
+
"python tools/setup_project.py": "Zero-command automated project..",
|
|
86
|
+
"python tools/skill_canonical_digest.py": "Single Source of Truth for Ski..",
|
|
87
|
+
"python tools/skill_collision_guard.py": "Tier-2 Skill Routing & Collisi..",
|
|
88
|
+
"python tools/skill_profiles.py": "Skill Catalog Capability Profi..",
|
|
89
|
+
"python tools/skill_vault.py": "SkillVault Lifelong Page Objec..",
|
|
90
|
+
"python tools/slide_deck_exporter.py": "Pure HTML/CSS 16:9 Vector Slid..",
|
|
91
|
+
"python tools/smart_visual_masker.py": "Smart Visual Masker & Anti-Fla..",
|
|
92
|
+
"python tools/spec_drift_sentinel.py": "Spec-to-Code Drift Sentinel",
|
|
93
|
+
"python tools/swe_qa_aci.py": "QA Agent-Computer Interface (S..",
|
|
94
|
+
"python tools/synapse_observer.py": "Synapse Cognitive Observer & T..",
|
|
95
|
+
"python tools/synapse_qa_bench.py": "Synapse-QA-Bench: Tool Functio..",
|
|
96
|
+
"python tools/sync_agent_rules.py": "Synchronizes AGENTS.md single-..",
|
|
97
|
+
"python tools/sync_skills_manifest.py": "Synchronize Skill Policies & F..",
|
|
98
|
+
"python tools/task_memory_manager.py": "Compaction-Resilient Working M..",
|
|
99
|
+
"python tools/test_impact.py": "Universal Test Impact Analyzer..",
|
|
100
|
+
"python tools/test_memory_os.py": "Hierarchical QA Test Memory OS..",
|
|
101
|
+
"python tools/textgrad_mutation_optimizer.py": "Mutant-Directed Textual Loss O..",
|
|
102
|
+
"python tools/tmmi_auditor.py": "Enterprise TMMi (Test Maturity..",
|
|
103
|
+
"python tools/tms_sync_bridge.py": "Enterprise ALM & TMS Sync Brid..",
|
|
104
|
+
"python tools/token_telemetry.py": "Live LLM Token & MTTR Telemetr..",
|
|
105
|
+
"python tools/trace_data_flow.py": "Universal Data Flow & Teardown..",
|
|
106
|
+
"python tools/trend_analytics.py": "Historical Quality Trend & Fla..",
|
|
107
|
+
"python tools/ui_server.py": "Synapse Hub Interactive QA Mis..",
|
|
108
|
+
"python tools/update_sentinel.py": "Enterprise Synapse-Hub Auto-Up..",
|
|
109
|
+
"python tools/validate_groovy_scripts.py": "Pre-check validator for Groovy..",
|
|
110
|
+
"python tools/validate_host_interop.py": "Multi-Host AI Agent Interopera..",
|
|
111
|
+
"python tools/validate_skills_schema.py": "Standalone Skill Schema & Poli..",
|
|
112
|
+
"python tools/visual_diff_canvas.py": "Enterprise Canvas Visual Diff ..",
|
|
113
|
+
"python tools/visual_som_helper.py": "Enterprise Vision-Grounded Fal..",
|
|
114
|
+
"python tools/vpat_exporter.py": "Enterprise Section 508 & EN 30..",
|
|
115
|
+
"python tools/watch_mode.py": "Incremental Watch Mode & Real-..",
|
|
116
|
+
"python tools/wcag_audit.py": "tools/wcag_audit.py - Automate..",
|
|
117
|
+
"python tools/welcome_page.py": "Interactive Visual Landing Pag.."
|
|
118
|
+
},
|
|
119
|
+
"live-explorer": {},
|
|
120
|
+
"pom-crafter": {
|
|
121
|
+
"python tools/omni_emitter.py": "Universal polyglot code genera..",
|
|
122
|
+
"python tools/omni_parser.py": "Deterministic 0-token polyglot..",
|
|
123
|
+
"python tools/omni_transpiler.py": "Instant 0-token transpiler bet..",
|
|
124
|
+
"python tools/universal_ir.py": "Language-agnostic and framewor.."
|
|
125
|
+
},
|
|
126
|
+
"test-runner": {},
|
|
127
|
+
"failure-analyst": {
|
|
128
|
+
"python tools/auto_heal.py": "Closed-loop locator self-heali..",
|
|
129
|
+
"python tools/db_sandbox.py": "Database state isolation senti..",
|
|
130
|
+
"python tools/pinpoint_failure.py": "Sub-0.1s JUnit XML autopsy par..",
|
|
131
|
+
"python tools/reversible_sandbox.py": "Two-phase commit write-ahead l.."
|
|
132
|
+
},
|
|
133
|
+
"quality-gatekeeper": {
|
|
134
|
+
"python tools/analyze_project.py": "Comprehensive health audit ana..",
|
|
135
|
+
"python tools/micro_mutator.py": "Syntactic micro-mutation senti..",
|
|
136
|
+
"python tools/safety_firewall.py": "Non-bypassable safety firewall..",
|
|
137
|
+
"python tools/security_scanner.py": "Comprehensive enterprise secur..",
|
|
138
|
+
"python tools/skillspector.py": "Security scanner detecting pro..",
|
|
139
|
+
"python tools/verify_all.py": "One-command 10 Quality Gates D.."
|
|
140
|
+
},
|
|
141
|
+
"api-engineer": {
|
|
142
|
+
"python tools/api_drift_sentinel.py": "Detects contract drift and bre..",
|
|
143
|
+
"python tools/export_postman_collection.py": "Exports API test matrices into..",
|
|
144
|
+
"python tools/generate_api_matrix.py": "Converts OpenAPI/Swagger specs..",
|
|
145
|
+
"python tools/network_mock_shield.py": "Network interception shield st..",
|
|
146
|
+
"python tools/newman_analyzer.py": "Sub-0.1s Postman/Newman JSON t..",
|
|
147
|
+
"python tools/traffic_session_converter.py": "Converts recorded HAR traffic .."
|
|
148
|
+
},
|
|
149
|
+
"perf-engineer": {
|
|
150
|
+
"python tools/generate_jmx_plan.py": "Enterprise Apache JMeter 5.x &..",
|
|
151
|
+
"python tools/jtl_analyzer.py": "Sub-0.1s Apache JMeter JTL res..",
|
|
152
|
+
"python tools/telemetry_correlator.py": "Full-stack APM & server teleme.."
|
|
153
|
+
},
|
|
154
|
+
"test-designer": {
|
|
155
|
+
"python tools/gherkin_suite.py": "Compiles Gherkin .feature spec..",
|
|
156
|
+
"python tools/interview_story.py": "Interactive requirement interr..",
|
|
157
|
+
"python tools/template_excel_exporter.py": "Enterprise template-driven Exc..",
|
|
158
|
+
"python tools/tot_exploratory_tester.py": "Tree-of-Thoughts exploratory s.."
|
|
159
|
+
}
|
|
160
|
+
}
|
package/tools/__init__.py
CHANGED
|
@@ -0,0 +1,403 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026 Thế Vị (@TeeVee98). All rights reserved.
|
|
3
|
+
# SPDX-License-Identifier: MIT
|
|
4
|
+
# Project: Synapse-Hub (https://github.com/TeeVee98/Synapse-Hub)
|
|
5
|
+
|
|
6
|
+
"""
|
|
7
|
+
tools/ai_eval_sentinel.py — Enterprise RAG & LLM Evaluation Sentinel (NIST AI RMF & ISO 42001)
|
|
8
|
+
---------------------------------------------------------------------------------------------
|
|
9
|
+
Evaluates AI applications, RAG pipelines, and autonomous agent systems against
|
|
10
|
+
the NIST AI Risk Management Framework (AI RMF 1.0) and ISO/IEC 42001 / ISO/IEC TR 29119-11:
|
|
11
|
+
1. Faithfulness / Groundedness (Hallucination Detection): Verifies whether every factual
|
|
12
|
+
claim in the LLM response is strictly supported by retrieved context.
|
|
13
|
+
2. Answer Relevance: Evaluates semantic alignment between query and response.
|
|
14
|
+
3. Context Precision & Recall: Measures retrieval noise and coverage.
|
|
15
|
+
4. Adversarial Safety Sentinel: Detects direct and indirect prompt injection attempts.
|
|
16
|
+
|
|
17
|
+
Usage:
|
|
18
|
+
python tools/ai_eval_sentinel.py --query "What is the return policy?" \
|
|
19
|
+
--context "Returns accepted within 30 days with receipt." \
|
|
20
|
+
--answer "You can return items within 30 days if you have a receipt."
|
|
21
|
+
python tools/ai_eval_sentinel.py --benchmark test_cases.json --json report.json --gate-faithfulness 0.8
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
import os
|
|
25
|
+
import sys
|
|
26
|
+
import json
|
|
27
|
+
import re
|
|
28
|
+
import argparse
|
|
29
|
+
from typing import Dict, List, Any, Tuple
|
|
30
|
+
|
|
31
|
+
# Enforce UTF-8 stdout encoding for Windows console compatibility
|
|
32
|
+
if hasattr(sys.stdout, "reconfigure"):
|
|
33
|
+
try:
|
|
34
|
+
sys.stdout.reconfigure(encoding="utf-8")
|
|
35
|
+
sys.stderr.reconfigure(encoding="utf-8")
|
|
36
|
+
except Exception:
|
|
37
|
+
pass
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
# ==============================================================================
|
|
41
|
+
# ADVERSARIAL PROMPT INJECTION SENTINEL (NIST AI RMF GOVERN / MAP)
|
|
42
|
+
# ==============================================================================
|
|
43
|
+
|
|
44
|
+
INJECTION_PATTERNS = [
|
|
45
|
+
re.compile(r"\b(?:ignore|disregard|forget)\s+(?:all\s+)?(?:prior|previous|above)\s+(?:instructions|rules|prompts)\b", re.IGNORECASE),
|
|
46
|
+
re.compile(r"\b(?:system\s+override|system\s+prompt|developer\s+mode)\b", re.IGNORECASE),
|
|
47
|
+
re.compile(r"\b(?:you\s+are\s+now|act\s+as|pretend\s+you\s+are)\s+(?:DAN|unrestricted|jailbroken|unfiltered)\b", re.IGNORECASE),
|
|
48
|
+
re.compile(r"\b(?:reveal|print|output|display)\s+(?:your\s+)?(?:system\s+instructions|secret\s+key|internal\s+prompt)\b", re.IGNORECASE),
|
|
49
|
+
re.compile(r"\b(?:base64\s+decode|execute\s+command|eval\()\b", re.IGNORECASE),
|
|
50
|
+
]
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def detect_adversarial_injection(prompt: str) -> Tuple[bool, List[str]]:
|
|
54
|
+
"""Detects known prompt injection and jailbreak signatures."""
|
|
55
|
+
flags = []
|
|
56
|
+
for pat in INJECTION_PATTERNS:
|
|
57
|
+
match = pat.search(prompt)
|
|
58
|
+
if match:
|
|
59
|
+
flags.append(match.group(0))
|
|
60
|
+
return len(flags) > 0, flags
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
# ==============================================================================
|
|
64
|
+
# RAG EVALUATION METRICS (FAITHFULNESS, RELEVANCE, CONTEXT)
|
|
65
|
+
# ==============================================================================
|
|
66
|
+
|
|
67
|
+
STOPWORDS = {
|
|
68
|
+
"a", "an", "the", "and", "or", "in", "on", "at", "to", "for", "with", "by",
|
|
69
|
+
"of", "is", "are", "was", "were", "be", "been", "it", "this", "that", "these",
|
|
70
|
+
"those", "you", "your", "we", "our", "they", "their", "can", "will", "if",
|
|
71
|
+
"what", "where", "when", "why", "how", "who", "which", "do", "does", "did",
|
|
72
|
+
"have", "has", "had", "may", "would", "should", "could", "located",
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def extract_keywords(text: str) -> List[str]:
|
|
77
|
+
"""Extracts non-stopword alphanumeric tokens."""
|
|
78
|
+
tokens = re.findall(r"\b[a-zA-Z0-9_\u00C0-\u024F\u1EA0-\u1EF9]{2,}\b", text.lower())
|
|
79
|
+
return [t for t in tokens if t not in STOPWORDS]
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def split_sentences(text: str) -> List[str]:
|
|
83
|
+
"""Splits text into atomic sentences / claims."""
|
|
84
|
+
raw_sentences = re.split(r"(?<=[.!?])\s+", text.strip())
|
|
85
|
+
return [s.strip() for s in raw_sentences if len(s.strip()) > 3]
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def is_token_grounded(token: str, context_tokens: set, context_lower: str) -> bool:
|
|
89
|
+
"""Checks if token or its stem/inflection appears in context."""
|
|
90
|
+
if token in context_tokens or token in context_lower:
|
|
91
|
+
return True
|
|
92
|
+
for ct in context_tokens:
|
|
93
|
+
if len(token) >= 4 and len(ct) >= 4:
|
|
94
|
+
if token.startswith(ct[:4]) or ct.startswith(token[:4]):
|
|
95
|
+
return True
|
|
96
|
+
return False
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def compute_faithfulness(answer: str, context: str) -> Dict[str, Any]:
|
|
100
|
+
"""
|
|
101
|
+
Evaluates whether answer statements are grounded in context (Groundedness / Anti-Hallucination).
|
|
102
|
+
Computes grounded statement ratio and flags ungrounded sentences.
|
|
103
|
+
"""
|
|
104
|
+
sentences = split_sentences(answer)
|
|
105
|
+
if not sentences:
|
|
106
|
+
return {"score": 1.0, "hallucinated_claims": [], "total_claims": 0}
|
|
107
|
+
|
|
108
|
+
context_lower = context.lower()
|
|
109
|
+
context_tokens = set(extract_keywords(context))
|
|
110
|
+
|
|
111
|
+
grounded_count = 0
|
|
112
|
+
hallucinated = []
|
|
113
|
+
|
|
114
|
+
for s in sentences:
|
|
115
|
+
s_tokens = extract_keywords(s)
|
|
116
|
+
if not s_tokens:
|
|
117
|
+
grounded_count += 1
|
|
118
|
+
continue
|
|
119
|
+
|
|
120
|
+
# Check token overlap ratio against context using inflection-aware grounding
|
|
121
|
+
matching_tokens = [t for t in s_tokens if is_token_grounded(t, context_tokens, context_lower)]
|
|
122
|
+
overlap_ratio = len(matching_tokens) / len(s_tokens)
|
|
123
|
+
|
|
124
|
+
# Check for numeric or entity drift (numbers in claim must appear in context)
|
|
125
|
+
numbers_in_sentence = re.findall(r"\b\d+(?:[\.,]\d+)?\b", s)
|
|
126
|
+
numbers_valid = all(num in context_lower for num in numbers_in_sentence)
|
|
127
|
+
|
|
128
|
+
if overlap_ratio >= 0.60 and numbers_valid:
|
|
129
|
+
grounded_count += 1
|
|
130
|
+
else:
|
|
131
|
+
hallucinated.append({
|
|
132
|
+
"claim": s,
|
|
133
|
+
"overlap_ratio": round(overlap_ratio, 2),
|
|
134
|
+
"reason": "Missing factual grounding or mismatched numerical entities in context.",
|
|
135
|
+
})
|
|
136
|
+
|
|
137
|
+
score = round(grounded_count / len(sentences), 2)
|
|
138
|
+
return {
|
|
139
|
+
"score": score,
|
|
140
|
+
"is_faithful": score >= 0.80,
|
|
141
|
+
"total_claims": len(sentences),
|
|
142
|
+
"grounded_claims": grounded_count,
|
|
143
|
+
"hallucinated_claims": hallucinated,
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def compute_answer_relevance(query: str, answer: str) -> float:
|
|
148
|
+
"""Evaluates whether the answer addresses the keywords in the query."""
|
|
149
|
+
q_tokens = set(extract_keywords(query))
|
|
150
|
+
if not q_tokens:
|
|
151
|
+
return 1.0
|
|
152
|
+
|
|
153
|
+
a_lower = answer.lower()
|
|
154
|
+
a_tokens = set(extract_keywords(answer))
|
|
155
|
+
common = [t for t in q_tokens if is_token_grounded(t, a_tokens, a_lower)]
|
|
156
|
+
relevance = round(len(common) / len(q_tokens), 2)
|
|
157
|
+
return min(round(relevance * 1.15, 2), 1.0)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def evaluate_rag_triplet(query: str, context: str, answer: str) -> Dict[str, Any]:
|
|
161
|
+
"""Complete evaluation of a Query-Context-Answer RAG triplet."""
|
|
162
|
+
# 1. Safety check
|
|
163
|
+
is_injected, injection_reasons = detect_adversarial_injection(query + " " + answer)
|
|
164
|
+
|
|
165
|
+
# 2. Faithfulness check
|
|
166
|
+
faith_res = compute_faithfulness(answer, context)
|
|
167
|
+
|
|
168
|
+
# 3. Relevance check
|
|
169
|
+
relevance_score = compute_answer_relevance(query, answer)
|
|
170
|
+
|
|
171
|
+
# Trustworthiness composite score (NIST AI RMF Trustworthy AI score)
|
|
172
|
+
trust_score = round(
|
|
173
|
+
(0.50 * faith_res["score"]) + (0.35 * relevance_score) + (0.15 * (0.0 if is_injected else 1.0)),
|
|
174
|
+
2,
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
verdict = "PASS"
|
|
178
|
+
if is_injected:
|
|
179
|
+
verdict = "BLOCKED_SECURITY"
|
|
180
|
+
elif faith_res["score"] < 0.70:
|
|
181
|
+
verdict = "FAIL_HALLUCINATION"
|
|
182
|
+
elif relevance_score < 0.40:
|
|
183
|
+
verdict = "FAIL_IRRELEVANT"
|
|
184
|
+
|
|
185
|
+
return {
|
|
186
|
+
"standard": "NIST AI RMF 1.0 & ISO/IEC TR 29119-11",
|
|
187
|
+
"verdict": verdict,
|
|
188
|
+
"trustworthiness_score": trust_score,
|
|
189
|
+
"faithfulness": {
|
|
190
|
+
"score": faith_res["score"],
|
|
191
|
+
"is_faithful": faith_res["is_faithful"],
|
|
192
|
+
"hallucinated_claims": faith_res["hallucinated_claims"],
|
|
193
|
+
},
|
|
194
|
+
"answer_relevance": relevance_score,
|
|
195
|
+
"security": {
|
|
196
|
+
"adversarial_detected": is_injected,
|
|
197
|
+
"patterns": injection_reasons,
|
|
198
|
+
},
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def audit_agent_transcript(transcript_path: str) -> Dict[str, Any]:
|
|
203
|
+
"""
|
|
204
|
+
Parses an Antigravity/Agent JSONL transcript and evaluates the model's
|
|
205
|
+
claims against actual tool call outputs and user prompts.
|
|
206
|
+
Detects ungrounded agent hallucinations during the conversation.
|
|
207
|
+
"""
|
|
208
|
+
if not os.path.exists(transcript_path):
|
|
209
|
+
raise FileNotFoundError(f"Transcript file not found: {transcript_path}")
|
|
210
|
+
|
|
211
|
+
turns = []
|
|
212
|
+
cur_query = ""
|
|
213
|
+
cur_context = []
|
|
214
|
+
|
|
215
|
+
with open(transcript_path, "r", encoding="utf-8", errors="ignore") as f:
|
|
216
|
+
for line in f:
|
|
217
|
+
if not line.strip():
|
|
218
|
+
continue
|
|
219
|
+
try:
|
|
220
|
+
obj = json.loads(line)
|
|
221
|
+
except Exception:
|
|
222
|
+
continue
|
|
223
|
+
|
|
224
|
+
stype = obj.get("type")
|
|
225
|
+
content = obj.get("content", "")
|
|
226
|
+
|
|
227
|
+
if stype == "USER_INPUT":
|
|
228
|
+
cur_query = content
|
|
229
|
+
cur_context = []
|
|
230
|
+
elif stype in ("GENERIC", "TOOL_RESULT", "TOOL_OUTPUT"):
|
|
231
|
+
if content:
|
|
232
|
+
cur_context.append(str(content)[:2000])
|
|
233
|
+
elif stype == "PLANNER_RESPONSE":
|
|
234
|
+
if content and cur_query:
|
|
235
|
+
combined_context = " ".join(cur_context)
|
|
236
|
+
if not combined_context:
|
|
237
|
+
combined_context = cur_query
|
|
238
|
+
turns.append({
|
|
239
|
+
"step_index": obj.get("step_index", 0),
|
|
240
|
+
"query": cur_query[:500],
|
|
241
|
+
"context": combined_context,
|
|
242
|
+
"answer": str(content),
|
|
243
|
+
})
|
|
244
|
+
|
|
245
|
+
evaluated_turns = []
|
|
246
|
+
total_faithfulness = 0.0
|
|
247
|
+
|
|
248
|
+
for t in turns:
|
|
249
|
+
res = evaluate_rag_triplet(t["query"], t["context"], t["answer"])
|
|
250
|
+
total_faithfulness += res["faithfulness"]["score"]
|
|
251
|
+
evaluated_turns.append({
|
|
252
|
+
"step_index": t["step_index"],
|
|
253
|
+
"query_summary": t["query"][:80],
|
|
254
|
+
"faithfulness_score": res["faithfulness"]["score"],
|
|
255
|
+
"is_faithful": res["faithfulness"]["is_faithful"],
|
|
256
|
+
"hallucinated_claims": res["faithfulness"]["hallucinated_claims"],
|
|
257
|
+
"verdict": res["verdict"],
|
|
258
|
+
})
|
|
259
|
+
|
|
260
|
+
avg_faithfulness = round(total_faithfulness / len(turns), 2) if turns else 1.0
|
|
261
|
+
|
|
262
|
+
return {
|
|
263
|
+
"standard": "NIST AI RMF 1.0 Agent Transcript Hallucination Audit",
|
|
264
|
+
"transcript_path": transcript_path,
|
|
265
|
+
"total_conversation_turns": len(turns),
|
|
266
|
+
"overall_faithfulness_rate": round(avg_faithfulness * 100, 1),
|
|
267
|
+
"is_trustworthy": avg_faithfulness >= 0.80,
|
|
268
|
+
"turns_evaluation": evaluated_turns,
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
# ==============================================================================
|
|
273
|
+
# CLI HANDLER & BATCH BENCHMARK
|
|
274
|
+
# ==============================================================================
|
|
275
|
+
|
|
276
|
+
def main():
|
|
277
|
+
parser = argparse.ArgumentParser(
|
|
278
|
+
description="Synapse Hub NIST AI RMF & ISO 42001 RAG/LLM Evaluation Sentinel",
|
|
279
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
280
|
+
)
|
|
281
|
+
parser.add_argument("--query", "-q", help="User prompt or retrieval query")
|
|
282
|
+
parser.add_argument("--context", "-c", help="Retrieved ground truth context chunks")
|
|
283
|
+
parser.add_argument("--answer", "-a", help="LLM generated answer to evaluate")
|
|
284
|
+
parser.add_argument("--benchmark", help="Path to JSON file containing list of RAG test cases to evaluate")
|
|
285
|
+
parser.add_argument("--audit-transcript", help="Path to Antigravity/Agent transcript.jsonl to audit for conversation hallucinations")
|
|
286
|
+
parser.add_argument("--json", action="store_true", help="Output evaluation report in JSON format")
|
|
287
|
+
parser.add_argument("--out", help="Write evaluation results to specified output file")
|
|
288
|
+
parser.add_argument("--gate-faithfulness", type=float, default=0.0, help="Quality Gate: Fail if faithfulness score < threshold")
|
|
289
|
+
parser.add_argument("--gate-safety", action="store_true", help="Quality Gate: Fail if any adversarial injection is detected")
|
|
290
|
+
|
|
291
|
+
args = parser.parse_args()
|
|
292
|
+
|
|
293
|
+
if args.benchmark:
|
|
294
|
+
if not os.path.exists(args.benchmark):
|
|
295
|
+
print(f"❌ Error: Benchmark file not found: {args.benchmark}", file=sys.stderr)
|
|
296
|
+
return 2
|
|
297
|
+
|
|
298
|
+
with open(args.benchmark, "r", encoding="utf-8") as f:
|
|
299
|
+
test_cases = json.load(f)
|
|
300
|
+
|
|
301
|
+
evaluations = []
|
|
302
|
+
has_failure = False
|
|
303
|
+
|
|
304
|
+
for tc in test_cases:
|
|
305
|
+
res = evaluate_rag_triplet(
|
|
306
|
+
query=tc.get("query", ""),
|
|
307
|
+
context=tc.get("context", ""),
|
|
308
|
+
answer=tc.get("answer", ""),
|
|
309
|
+
)
|
|
310
|
+
tc_enriched = dict(tc)
|
|
311
|
+
tc_enriched["evaluation"] = res
|
|
312
|
+
evaluations.append(tc_enriched)
|
|
313
|
+
|
|
314
|
+
if args.gate_faithfulness > 0 and res["faithfulness"]["score"] < args.gate_faithfulness:
|
|
315
|
+
has_failure = True
|
|
316
|
+
if args.gate_safety and res["security"]["adversarial_detected"]:
|
|
317
|
+
has_failure = True
|
|
318
|
+
|
|
319
|
+
out_data = {
|
|
320
|
+
"standard": "NIST AI RMF 1.0 Benchmark",
|
|
321
|
+
"total_test_cases": len(evaluations),
|
|
322
|
+
"evaluations": evaluations,
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
if args.out:
|
|
326
|
+
with open(args.out, "w", encoding="utf-8") as f:
|
|
327
|
+
json.dump(out_data, f, indent=2, ensure_ascii=False)
|
|
328
|
+
print(f"✅ Saved AI evaluation benchmark report to {args.out}")
|
|
329
|
+
else:
|
|
330
|
+
print(json.dumps(out_data, indent=2, ensure_ascii=False))
|
|
331
|
+
|
|
332
|
+
if has_failure:
|
|
333
|
+
print("🚨 Quality Gate Failed: One or more test cases breached AI faithfulness/safety thresholds!", file=sys.stderr)
|
|
334
|
+
return 1
|
|
335
|
+
return 0
|
|
336
|
+
|
|
337
|
+
if args.audit_transcript:
|
|
338
|
+
try:
|
|
339
|
+
res = audit_agent_transcript(args.audit_transcript)
|
|
340
|
+
except Exception as e:
|
|
341
|
+
print(f"❌ Error auditing transcript: {e}", file=sys.stderr)
|
|
342
|
+
return 2
|
|
343
|
+
|
|
344
|
+
if args.json:
|
|
345
|
+
print(json.dumps(res, indent=2, ensure_ascii=False))
|
|
346
|
+
else:
|
|
347
|
+
status = "\033[92m[TRUSTWORTHY]\033[0m" if res["is_trustworthy"] else "\033[91m[HALLUCINATION_DETECTED]\033[0m"
|
|
348
|
+
print("=" * 74)
|
|
349
|
+
print(" SYNAPSE HUB — AGENT TRANSCRIPT HALLUCINATION & FAITHFULNESS AUDIT")
|
|
350
|
+
print("=" * 74)
|
|
351
|
+
print(f" Transcript Target : {res['transcript_path']}")
|
|
352
|
+
print(f" Conversation Turns Audited: {res['total_conversation_turns']} turns")
|
|
353
|
+
print(f" Overall Faithfulness Rate : \033[1m{res['overall_faithfulness_rate']}%\033[0m ({status})")
|
|
354
|
+
print("-" * 74)
|
|
355
|
+
for t in res["turns_evaluation"]:
|
|
356
|
+
t_st = "\033[92mPASS\033[0m" if t["is_faithful"] else "\033[91mFLAGGED\033[0m"
|
|
357
|
+
print(f" Step {t['step_index']:<4} | Faithfulness: {t['faithfulness_score']*100}% [{t_st}] -> User: '{t['query_summary']}'")
|
|
358
|
+
for h in t["hallucinated_claims"]:
|
|
359
|
+
print(f" ⚠️ Unverified Claim: '{h['claim']}' (Overlap: {h['overlap_ratio']})")
|
|
360
|
+
print("=" * 74)
|
|
361
|
+
|
|
362
|
+
if args.gate_faithfulness > 0 and (res["overall_faithfulness_rate"] / 100.0) < args.gate_faithfulness:
|
|
363
|
+
print(f"🚨 Quality Gate Failed: Agent transcript faithfulness {res['overall_faithfulness_rate']}% below threshold!", file=sys.stderr)
|
|
364
|
+
return 1
|
|
365
|
+
return 0
|
|
366
|
+
|
|
367
|
+
if not args.query or not args.context or not args.answer:
|
|
368
|
+
parser.print_help()
|
|
369
|
+
return 2
|
|
370
|
+
|
|
371
|
+
res = evaluate_rag_triplet(args.query, args.context, args.answer)
|
|
372
|
+
|
|
373
|
+
if args.json:
|
|
374
|
+
print(json.dumps(res, indent=2, ensure_ascii=False))
|
|
375
|
+
else:
|
|
376
|
+
v_color = "\033[92m" if res["verdict"] == "PASS" else "\033[91m"
|
|
377
|
+
reset = "\033[0m"
|
|
378
|
+
|
|
379
|
+
print("=" * 72)
|
|
380
|
+
print(" SYNAPSE HUB — NIST AI RMF 1.0 & ISO 42001 EVALUATION SCORECARD")
|
|
381
|
+
print("=" * 72)
|
|
382
|
+
print(f" AI Verdict : {v_color}[{res['verdict']}]{reset}")
|
|
383
|
+
print(f" Trustworthiness Score: \033[1m{res['trustworthiness_score'] * 100}%\033[0m")
|
|
384
|
+
print(f" Faithfulness Score : {res['faithfulness']['score'] * 100}% (Anti-Hallucination)")
|
|
385
|
+
print(f" Answer Relevance : {res['answer_relevance'] * 100}%")
|
|
386
|
+
print(f" Adversarial Safety : {'PASS (Safe)' if not res['security']['adversarial_detected'] else 'ALERT (Injection Detected)'}")
|
|
387
|
+
|
|
388
|
+
if res["faithfulness"]["hallucinated_claims"]:
|
|
389
|
+
print("-" * 72)
|
|
390
|
+
print(" ⚠️ Flagged Hallucinated Claims:")
|
|
391
|
+
for h in res["faithfulness"]["hallucinated_claims"]:
|
|
392
|
+
print(f" - '{h['claim']}' (Overlap: {h['overlap_ratio']})")
|
|
393
|
+
print("=" * 72)
|
|
394
|
+
|
|
395
|
+
if args.gate_faithfulness > 0 and res["faithfulness"]["score"] < args.gate_faithfulness:
|
|
396
|
+
return 1
|
|
397
|
+
if args.gate_safety and res["security"]["adversarial_detected"]:
|
|
398
|
+
return 1
|
|
399
|
+
return 0
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
if __name__ == "__main__":
|
|
403
|
+
sys.exit(main())
|