gabbe-kit 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +25 -0
- package/LICENSE-CODE +202 -0
- package/README.md +817 -0
- package/agents/AGENTS.md +542 -0
- package/agents/CONSTITUTION.md +188 -0
- package/agents/guides/00-index.md +142 -0
- package/agents/guides/ai/a2ui-protocols.md +71 -0
- package/agents/guides/ai/actor-agent-frameworks.md +82 -0
- package/agents/guides/ai/agent-communication.md +146 -0
- package/agents/guides/ai/agent-identity-trust.md +69 -0
- package/agents/guides/ai/agent-only-cognition.md +73 -0
- package/agents/guides/ai/agent-only-swarms.md +70 -0
- package/agents/guides/ai/agent-operating-ergonomics.md +52 -0
- package/agents/guides/ai/agent-ui.md +106 -0
- package/agents/guides/ai/ai-agentic.md +309 -0
- package/agents/guides/ai/beyond-llms.md +60 -0
- package/agents/guides/ai/dynamic-capability-loading.md +121 -0
- package/agents/guides/ai/gabbe-innovations.md +111 -0
- package/agents/guides/ai/model-customization-decisions.md +78 -0
- package/agents/guides/ai/multi-agent-systems.md +85 -0
- package/agents/guides/ai/self-evolving-skills.md +137 -0
- package/agents/guides/ai/self-healing-summary.md +21 -0
- package/agents/guides/ai/visual-mcp-integration.md +106 -0
- package/agents/guides/architecture/api-standards.md +61 -0
- package/agents/guides/architecture/blockchain-dlt.md +36 -0
- package/agents/guides/architecture/critical-systems-arch.md +66 -0
- package/agents/guides/architecture/event-driven-architecture.md +39 -0
- package/agents/guides/architecture/microservices.md +292 -0
- package/agents/guides/architecture/monolith.md +303 -0
- package/agents/guides/architecture/orchestration.md +75 -0
- package/agents/guides/architecture/system-qualities.md +51 -0
- package/agents/guides/architecture/system-scalability.md +47 -0
- package/agents/guides/architecture/systems-architecture.md +506 -0
- package/agents/guides/data/semantic-web.md +23 -0
- package/agents/guides/data/sql-nosql.md +305 -0
- package/agents/guides/industry/engineering-standards.md +36 -0
- package/agents/guides/industry/global-standards.md +24 -0
- package/agents/guides/industry/healthcare-fhir.md +33 -0
- package/agents/guides/industry/industrial-iot.md +33 -0
- package/agents/guides/industry/telecom-networks.md +34 -0
- package/agents/guides/languages/c-sharp.md +66 -0
- package/agents/guides/languages/go-lang.md +251 -0
- package/agents/guides/languages/java.md +73 -0
- package/agents/guides/languages/js-ts-nodejs.md +361 -0
- package/agents/guides/languages/nodejs-advanced.md +56 -0
- package/agents/guides/languages/php-laravel.md +376 -0
- package/agents/guides/languages/python-fastapi-ai.md +257 -0
- package/agents/guides/languages/rust.md +73 -0
- package/agents/guides/ops/compliance-audit.md +177 -0
- package/agents/guides/ops/cost-optimization.md +57 -0
- package/agents/guides/ops/dev-environments.md +67 -0
- package/agents/guides/ops/dev-workflow.md +44 -0
- package/agents/guides/ops/gabbe-cli-workflows.md +252 -0
- package/agents/guides/ops/production-health.md +62 -0
- package/agents/guides/ops/troubleshooting-guide.md +69 -0
- package/agents/guides/patterns/agentic-design-patterns.md +308 -0
- package/agents/guides/patterns/agentic-patterns.md +43 -0
- package/agents/guides/patterns/ai-native-scenarios.md +71 -0
- package/agents/guides/patterns/autonomous-swarm-patterns.md +42 -0
- package/agents/guides/patterns/design-patterns.md +68 -0
- package/agents/guides/patterns/enterprise-migration-scenario.md +58 -0
- package/agents/guides/patterns/enterprise-patterns.md +56 -0
- package/agents/guides/patterns/software-design-patterns.md +106 -0
- package/agents/guides/patterns/time-complexity-analysis.md +116 -0
- package/agents/guides/planning/product-requirements.md +407 -0
- package/agents/guides/planning/strategic-analysis.md +98 -0
- package/agents/guides/planning/visual-product-specs.md +162 -0
- package/agents/guides/principles/clean-code-standards.md +51 -0
- package/agents/guides/principles/computing-foundations.md +87 -0
- package/agents/guides/principles/diagramming-standards.md +568 -0
- package/agents/guides/principles/evaluation-strategy.md +103 -0
- package/agents/guides/principles/future-tech.md +35 -0
- package/agents/guides/principles/human-agent-collaboration.md +54 -0
- package/agents/guides/principles/legacy-tech.md +61 -0
- package/agents/guides/principles/mathematical-foundations.md +103 -0
- package/agents/guides/principles/modeling-methods.md +97 -0
- package/agents/guides/principles/no-code-integration.md +84 -0
- package/agents/guides/principles/testing-strategy.md +97 -0
- package/agents/guides/principles/visual-design-system.md +44 -0
- package/agents/guides/processes/RARV_CYCLE.md +262 -0
- package/agents/guides/processes/extension-protocol.md +183 -0
- package/agents/guides/processes/full-system-lifecycle.md +47 -0
- package/agents/guides/processes/knowledge-integration.md +82 -0
- package/agents/guides/processes/loki-sdlc-phases.md +446 -0
- package/agents/guides/product/green-software.md +38 -0
- package/agents/guides/product/sustainability-checks.md +29 -0
- package/agents/guides/security/ai-risk-standards-map.md +98 -0
- package/agents/guides/security/api-security.md +51 -0
- package/agents/guides/security/cryptography-standards.md +38 -0
- package/agents/guides/security/privacy-data-protection.md +41 -0
- package/agents/guides/security/secure-architecture.md +43 -0
- package/agents/guides/security/threat-modeling.md +46 -0
- package/agents/memory/AUDIT_LOG.md +48 -0
- package/agents/memory/CONTINUITY.md +49 -0
- package/agents/memory/PROJECT_STATE.md +134 -0
- package/agents/memory/RESUME_POINTER.md +15 -0
- package/agents/memory/episodic/DECISION_LOG_TEMPLATE.md +104 -0
- package/agents/memory/semantic/PROJECT_KNOWLEDGE_TEMPLATE.md +194 -0
- package/agents/memory/semantic/VECTOR_DB_CONFIG_TEMPLATE.json +46 -0
- package/agents/personas/00-index.md +148 -0
- package/agents/personas/biz-compliance.md +112 -0
- package/agents/personas/biz-legal.md +131 -0
- package/agents/personas/eng-api.md +110 -0
- package/agents/personas/eng-backend.md +113 -0
- package/agents/personas/eng-data.md +50 -0
- package/agents/personas/eng-database.md +95 -0
- package/agents/personas/eng-frontend.md +75 -0
- package/agents/personas/eng-infra.md +105 -0
- package/agents/personas/eng-messaging.md +50 -0
- package/agents/personas/eng-ml.md +50 -0
- package/agents/personas/eng-mobile.md +75 -0
- package/agents/personas/eng-perf.md +90 -0
- package/agents/personas/eng-qa.md +125 -0
- package/agents/personas/eng-tooling.md +49 -0
- package/agents/personas/ops-compliance.md +109 -0
- package/agents/personas/ops-cost.md +92 -0
- package/agents/personas/ops-devops.md +85 -0
- package/agents/personas/ops-incident.md +123 -0
- package/agents/personas/ops-monitor.md +89 -0
- package/agents/personas/ops-release.md +91 -0
- package/agents/personas/ops-security.md +104 -0
- package/agents/personas/ops-sre.md +85 -0
- package/agents/personas/orch-coordinator.md +140 -0
- package/agents/personas/orch-judge.md +135 -0
- package/agents/personas/orch-planner.md +114 -0
- package/agents/personas/orch-researcher.md +165 -0
- package/agents/personas/prod-architect.md +110 -0
- package/agents/personas/prod-design.md +112 -0
- package/agents/personas/prod-ethicist.md +50 -0
- package/agents/personas/prod-integration.md +120 -0
- package/agents/personas/prod-pm.md +115 -0
- package/agents/personas/prod-product-ops.md +128 -0
- package/agents/personas/prod-research.md +49 -0
- package/agents/personas/prod-safety-engineer.md +28 -0
- package/agents/personas/prod-tech-lead.md +119 -0
- package/agents/personas/ui-design.md +81 -0
- package/agents/scripts/check_skills_docs.py +120 -0
- package/agents/scripts/compile_skills.py +284 -0
- package/agents/scripts/comprehensive_checker.py +187 -0
- package/agents/scripts/dump-context.sh +37 -0
- package/agents/scripts/eval_skills.py +268 -0
- package/agents/scripts/preflight_summary.sh +80 -0
- package/agents/scripts/report_duplicates.py +125 -0
- package/agents/scripts/setup-context.ps1 +250 -0
- package/agents/scripts/setup-context.sh +398 -0
- package/agents/scripts/state_export.sh +84 -0
- package/agents/scripts/state_import.sh +99 -0
- package/agents/scripts/validate_integrity.py +98 -0
- package/agents/scripts/validate_links.py +96 -0
- package/agents/scripts/validate_methodology_graph.py +194 -0
- package/agents/scripts/validate_skills.py +81 -0
- package/agents/scripts/verify_e2e_init_flows.py +246 -0
- package/agents/scripts/verify_setup_simulation.py +104 -0
- package/agents/scripts/verify_triggers_and_mcps.py +159 -0
- package/agents/scripts/verify_use_cases.py +498 -0
- package/agents/skills/00-index.md +322 -0
- package/agents/skills/ai/a2ui-protocols.skill.md +31 -0
- package/agents/skills/ai/actor-agent-frameworks.skill.md +31 -0
- package/agents/skills/ai/agent-communication.skill.md +36 -0
- package/agents/skills/ai/agent-ui.skill.md +31 -0
- package/agents/skills/ai/autonomous-swarm-patterns.skill.md +31 -0
- package/agents/skills/ai/beyond-llms.skill.md +31 -0
- package/agents/skills/ai/context-engineering.skill.md +79 -0
- package/agents/skills/ai/llm-as-judge.skill.md +97 -0
- package/agents/skills/ai/multi-agent-systems.skill.md +31 -0
- package/agents/skills/ai/rag-evaluation.skill.md +93 -0
- package/agents/skills/ai/reasoning-patterns.skill.md +125 -0
- package/agents/skills/architecture/adaptive-architecture.skill.md +51 -0
- package/agents/skills/architecture/api-design.skill.md +92 -0
- package/agents/skills/architecture/api-standards.skill.md +31 -0
- package/agents/skills/architecture/arch-debt.skill.md +180 -0
- package/agents/skills/architecture/arch-design.skill.md +304 -0
- package/agents/skills/architecture/arch-patterns.skill.md +52 -0
- package/agents/skills/architecture/arch-review.skill.md +326 -0
- package/agents/skills/architecture/architecture-governance.skill.md +62 -0
- package/agents/skills/architecture/attribute-driven-design.skill.md +108 -0
- package/agents/skills/architecture/blockchain-dlt.skill.md +28 -0
- package/agents/skills/architecture/compatibility-design.skill.md +51 -0
- package/agents/skills/architecture/critical-systems-arch.skill.md +31 -0
- package/agents/skills/architecture/design-patterns.skill.md +59 -0
- package/agents/skills/architecture/diagramming.skill.md +380 -0
- package/agents/skills/architecture/domain-model.skill.md +294 -0
- package/agents/skills/architecture/enterprise-migration-scenario.skill.md +31 -0
- package/agents/skills/architecture/enterprise-patterns.skill.md +31 -0
- package/agents/skills/architecture/error-handling-strategy.skill.md +58 -0
- package/agents/skills/architecture/event-driven-architecture.skill.md +31 -0
- package/agents/skills/architecture/event-governance.skill.md +55 -0
- package/agents/skills/architecture/fitness-functions.skill.md +111 -0
- package/agents/skills/architecture/graphql-schema.skill.md +64 -0
- package/agents/skills/architecture/legacy-modernization.skill.md +51 -0
- package/agents/skills/architecture/microservices.skill.md +31 -0
- package/agents/skills/architecture/middleware-design.skill.md +59 -0
- package/agents/skills/architecture/monolith.skill.md +31 -0
- package/agents/skills/architecture/realtime-comm.skill.md +52 -0
- package/agents/skills/architecture/state-management.skill.md +72 -0
- package/agents/skills/architecture/system-qualities.skill.md +31 -0
- package/agents/skills/architecture/system-scalability.skill.md +31 -0
- package/agents/skills/architecture/systems-architecture.skill.md +31 -0
- package/agents/skills/architecture/visual-whiteboarding.skill.md +33 -0
- package/agents/skills/brain/README.md +70 -0
- package/agents/skills/brain/active-inference.skill.md +74 -0
- package/agents/skills/brain/brain-mode.skill.md +125 -0
- package/agents/skills/brain/cognitive-architectures.skill.md +53 -0
- package/agents/skills/brain/cognitive-testing.skill.md +62 -0
- package/agents/skills/brain/consciousness-loop.skill.md +68 -0
- package/agents/skills/brain/cost-benefit-router.skill.md +89 -0
- package/agents/skills/brain/episodic-consolidation.skill.md +43 -0
- package/agents/skills/brain/epistemology-knowledge.skill.md +54 -0
- package/agents/skills/brain/global-workspace.skill.md +77 -0
- package/agents/skills/brain/learning-adaptation.skill.md +68 -0
- package/agents/skills/brain/loki-mode.skill.md +267 -0
- package/agents/skills/brain/neuroscience-foundations.skill.md +52 -0
- package/agents/skills/brain/scripts/active_inference_loop.py +92 -0
- package/agents/skills/brain/scripts/cognition_test.py +80 -0
- package/agents/skills/brain/scripts/global_workspace.py +104 -0
- package/agents/skills/brain/self-improvement.skill.md +80 -0
- package/agents/skills/brain/sensory-motor.skill.md +61 -0
- package/agents/skills/brain/sequential-thinking.skill.md +48 -0
- package/agents/skills/brain/working-memory.skill.md +47 -0
- package/agents/skills/coding/agent-trajectory-eval.skill.md +92 -0
- package/agents/skills/coding/artisan-commands.skill.md +52 -0
- package/agents/skills/coding/browser-tdd.skill.md +131 -0
- package/agents/skills/coding/chaos-fault-injection.skill.md +116 -0
- package/agents/skills/coding/ci-autofix.skill.md +53 -0
- package/agents/skills/coding/clean-coder.skill.md +58 -0
- package/agents/skills/coding/code-review.skill.md +100 -0
- package/agents/skills/coding/debug.skill.md +100 -0
- package/agents/skills/coding/documentation.skill.md +149 -0
- package/agents/skills/coding/eval-driven-development.skill.md +107 -0
- package/agents/skills/coding/evals/code-review.eval.yaml +29 -0
- package/agents/skills/coding/excalidraw.skill.md +90 -0
- package/agents/skills/coding/file-processing.skill.md +55 -0
- package/agents/skills/coding/git-workflow.skill.md +153 -0
- package/agents/skills/coding/metamorphic-testing.skill.md +113 -0
- package/agents/skills/coding/mobile-dev.skill.md +41 -0
- package/agents/skills/coding/nodejs-expert.skill.md +23 -0
- package/agents/skills/coding/output-validation.skill.md +57 -0
- package/agents/skills/coding/pbt-strategy.skill.md +117 -0
- package/agents/skills/coding/performant-ai.skill.md +62 -0
- package/agents/skills/coding/performant-go.skill.md +61 -0
- package/agents/skills/coding/performant-laravel.skill.md +70 -0
- package/agents/skills/coding/performant-nodejs.skill.md +67 -0
- package/agents/skills/coding/performant-php.skill.md +53 -0
- package/agents/skills/coding/performant-python.skill.md +62 -0
- package/agents/skills/coding/refactor.skill.md +124 -0
- package/agents/skills/coding/secure-coding.skill.md +57 -0
- package/agents/skills/coding/sketch-to-diagram.skill.md +106 -0
- package/agents/skills/coding/spec-driven-development.skill.md +110 -0
- package/agents/skills/coding/tdd-cycle.skill.md +92 -0
- package/agents/skills/coding/testing-strategy.skill.md +74 -0
- package/agents/skills/coding/time-complexity.skill.md +112 -0
- package/agents/skills/coding/tldraw-canvas.skill.md +109 -0
- package/agents/skills/coding/tool-construction.skill.md +47 -0
- package/agents/skills/coding/ui-gen.skill.md +64 -0
- package/agents/skills/coding/vibe-coding.skill.md +63 -0
- package/agents/skills/coding/visual-design.skill.md +65 -0
- package/agents/skills/coordination/agent-interop.skill.md +72 -0
- package/agents/skills/coordination/agent-protocol.skill.md +62 -0
- package/agents/skills/coordination/agent-workflow-patterns.skill.md +123 -0
- package/agents/skills/coordination/agentic-linter.skill.md +134 -0
- package/agents/skills/coordination/agentic-patterns.skill.md +79 -0
- package/agents/skills/coordination/meta-optimize.skill.md +62 -0
- package/agents/skills/coordination/meta-prompting.skill.md +46 -0
- package/agents/skills/coordination/multi-agent-orch.skill.md +78 -0
- package/agents/skills/coordination/persona-selector.skill.md +78 -0
- package/agents/skills/coordination/self-optimize.skill.md +81 -0
- package/agents/skills/coordination/swarm-consensus.skill.md +52 -0
- package/agents/skills/core/agent-analytics.skill.md +74 -0
- package/agents/skills/core/audit-trail.skill.md +122 -0
- package/agents/skills/core/clarify.skill.md +89 -0
- package/agents/skills/core/emerging-tech.skill.md +52 -0
- package/agents/skills/core/empirical-methods.skill.md +101 -0
- package/agents/skills/core/evals/clarify.eval.yaml +29 -0
- package/agents/skills/core/evals/integrity-check.eval.yaml +23 -0
- package/agents/skills/core/evals/research.eval.yaml +23 -0
- package/agents/skills/core/final-review.skill.md +66 -0
- package/agents/skills/core/integrity-check.skill.md +149 -0
- package/agents/skills/core/knowledge-connect.skill.md +77 -0
- package/agents/skills/core/knowledge-gap.skill.md +123 -0
- package/agents/skills/core/preflight.skill.md +106 -0
- package/agents/skills/core/research.skill.md +107 -0
- package/agents/skills/core/retrospective.skill.md +59 -0
- package/agents/skills/core/sdlc-checkpoint.skill.md +151 -0
- package/agents/skills/core/self-heal.skill.md +163 -0
- package/agents/skills/core/session-resume.skill.md +125 -0
- package/agents/skills/core/skills-registry.skill.md +65 -0
- package/agents/skills/core/state-portability.skill.md +89 -0
- package/agents/skills/core/state-preserve.skill.md +76 -0
- package/agents/skills/core/system-lifecycle.skill.md +51 -0
- package/agents/skills/core/update-scan.skill.md +92 -0
- package/agents/skills/data/data-engineering.skill.md +56 -0
- package/agents/skills/data/data-governance.skill.md +49 -0
- package/agents/skills/data/db-migration.skill.md +136 -0
- package/agents/skills/data/semantic-web.skill.md +28 -0
- package/agents/skills/data/sql-optimization.skill.md +78 -0
- package/agents/skills/industry/engineering-standards.skill.md +31 -0
- package/agents/skills/industry/global-standards.skill.md +30 -0
- package/agents/skills/industry/healthcare-fhir.skill.md +28 -0
- package/agents/skills/industry/industrial-iot.skill.md +30 -0
- package/agents/skills/industry/telecom-networks.skill.md +30 -0
- package/agents/skills/ops/caching-strategy.skill.md +77 -0
- package/agents/skills/ops/capacity-planning.skill.md +67 -0
- package/agents/skills/ops/cloud-deploy.skill.md +55 -0
- package/agents/skills/ops/configuration-management.skill.md +107 -0
- package/agents/skills/ops/cost-optimization.skill.md +39 -0
- package/agents/skills/ops/decommission-sunset.skill.md +64 -0
- package/agents/skills/ops/dependency-lifecycle.skill.md +63 -0
- package/agents/skills/ops/deployment.skill.md +217 -0
- package/agents/skills/ops/dev-environments.skill.md +31 -0
- package/agents/skills/ops/dev-workflow.skill.md +28 -0
- package/agents/skills/ops/docker-dev.skill.md +52 -0
- package/agents/skills/ops/enterprise-integration.skill.md +53 -0
- package/agents/skills/ops/feature-flag-management.skill.md +59 -0
- package/agents/skills/ops/incident-response.skill.md +40 -0
- package/agents/skills/ops/infra-devops.skill.md +53 -0
- package/agents/skills/ops/k8s-dev.skill.md +51 -0
- package/agents/skills/ops/memory-optimization.skill.md +45 -0
- package/agents/skills/ops/observability-stack-setup.skill.md +62 -0
- package/agents/skills/ops/performance-audit.skill.md +178 -0
- package/agents/skills/ops/performance-optimization.skill.md +51 -0
- package/agents/skills/ops/production-health.skill.md +85 -0
- package/agents/skills/ops/production-verifier.skill.md +50 -0
- package/agents/skills/ops/queue-management.skill.md +52 -0
- package/agents/skills/ops/release-management.skill.md +39 -0
- package/agents/skills/ops/release-validation.skill.md +51 -0
- package/agents/skills/ops/reliability-sre.skill.md +51 -0
- package/agents/skills/ops/runbook-authoring.skill.md +64 -0
- package/agents/skills/ops/system-benchmark.skill.md +69 -0
- package/agents/skills/ops/tech-debt.skill.md +143 -0
- package/agents/skills/ops/troubleshooting-guide.skill.md +31 -0
- package/agents/skills/product/accessibility.skill.md +143 -0
- package/agents/skills/product/adr-writer.skill.md +110 -0
- package/agents/skills/product/business-case.skill.md +70 -0
- package/agents/skills/product/change-management.skill.md +90 -0
- package/agents/skills/product/decompose.skill.md +75 -0
- package/agents/skills/product/design-thinking.skill.md +62 -0
- package/agents/skills/product/estimation-sizing.skill.md +87 -0
- package/agents/skills/product/evals/spec-writer.eval.yaml +22 -0
- package/agents/skills/product/financial-governance.skill.md +88 -0
- package/agents/skills/product/green-software.skill.md +46 -0
- package/agents/skills/product/ideation-facilitation.skill.md +87 -0
- package/agents/skills/product/market-analysis.skill.md +56 -0
- package/agents/skills/product/opportunity-assessment.skill.md +89 -0
- package/agents/skills/product/product-analytics.skill.md +92 -0
- package/agents/skills/product/professional-practice.skill.md +105 -0
- package/agents/skills/product/req-elicitation.skill.md +292 -0
- package/agents/skills/product/req-review.skill.md +280 -0
- package/agents/skills/product/spec-analyze.skill.md +98 -0
- package/agents/skills/product/spec-writer.skill.md +114 -0
- package/agents/skills/product/stakeholder-management.skill.md +54 -0
- package/agents/skills/product/sustainability-checks.skill.md +29 -0
- package/agents/skills/product/systems-thinking.skill.md +56 -0
- package/agents/skills/product/user-research-synthesis.skill.md +85 -0
- package/agents/skills/product/user-story-mapping.skill.md +54 -0
- package/agents/skills/product/visual-specs.skill.md +112 -0
- package/agents/skills/security/access-control.skill.md +45 -0
- package/agents/skills/security/agent-sandboxing.skill.md +69 -0
- package/agents/skills/security/ai-ethics-compliance.skill.md +46 -0
- package/agents/skills/security/ai-red-teaming.skill.md +74 -0
- package/agents/skills/security/ai-safety-guardrails.skill.md +60 -0
- package/agents/skills/security/api-security.skill.md +31 -0
- package/agents/skills/security/backup-recovery.skill.md +49 -0
- package/agents/skills/security/compliance-review.skill.md +152 -0
- package/agents/skills/security/cryptography-standards.skill.md +31 -0
- package/agents/skills/security/dependency-security.skill.md +47 -0
- package/agents/skills/security/hazard-analysis.skill.md +53 -0
- package/agents/skills/security/legal-review.skill.md +41 -0
- package/agents/skills/security/log-analysis.skill.md +68 -0
- package/agents/skills/security/network-security.skill.md +64 -0
- package/agents/skills/security/privacy-audit.skill.md +137 -0
- package/agents/skills/security/privacy-data-protection.skill.md +31 -0
- package/agents/skills/security/prompt-injection-defense.skill.md +67 -0
- package/agents/skills/security/reliability-engineering.skill.md +47 -0
- package/agents/skills/security/safety-scan.skill.md +49 -0
- package/agents/skills/security/secrets-management.skill.md +65 -0
- package/agents/skills/security/secure-architecture.skill.md +31 -0
- package/agents/skills/security/security-audit.skill.md +164 -0
- package/agents/skills/security/threat-model.skill.md +120 -0
- package/agents/skills/security/traceability-audit.skill.md +48 -0
- package/agents/templates/00-index.md +162 -0
- package/agents/templates/architecture/ADAPTIVE_SYSTEM_TEMPLATE.md +35 -0
- package/agents/templates/architecture/ADD_WORKBOOK_TEMPLATE.md +81 -0
- package/agents/templates/architecture/ADR_TEMPLATE.md +136 -0
- package/agents/templates/architecture/ARCHITECTURE_DECISION_MATRIX.md +169 -0
- package/agents/templates/architecture/ARCHITECTURE_REVIEW_TEMPLATE.md +241 -0
- package/agents/templates/architecture/ARCHITECTURE_VIEWS_TEMPLATE.md +353 -0
- package/agents/templates/architecture/ARCH_DECISION_FRAMEWORK.md +32 -0
- package/agents/templates/architecture/C4_ARCHITECTURE_TEMPLATE.md +185 -0
- package/agents/templates/architecture/CAPABILITY_MAP_TEMPLATE.md +153 -0
- package/agents/templates/architecture/CONTEXT_MAP_TEMPLATE.md +196 -0
- package/agents/templates/architecture/DESIGN_PATTERN_USAGE.md +24 -0
- package/agents/templates/architecture/DOMAIN_MODEL_TEMPLATE.md +254 -0
- package/agents/templates/architecture/FITNESS_FUNCTION_TEMPLATE.md +59 -0
- package/agents/templates/architecture/INTEGRATION_SPEC_TEMPLATE.md +38 -0
- package/agents/templates/architecture/LEGACY_AUDIT_TEMPLATE.md +34 -0
- package/agents/templates/architecture/QUALITY_ATTRIBUTES_TEMPLATE.md +210 -0
- package/agents/templates/architecture/SCALABILITY_ANALYSIS_TEMPLATE.md +46 -0
- package/agents/templates/architecture/SMART_CONTRACT_TEMPLATE.md +44 -0
- package/agents/templates/architecture/SYSTEM_CONTEXT_TEMPLATE.md +197 -0
- package/agents/templates/brain/ACTIVE_INFERENCE_LOOP_TEMPLATE.md +42 -0
- package/agents/templates/brain/EPISODIC_MEMORY_LOG_TEMPLATE.md +28 -0
- package/agents/templates/brain/GLOBAL_WORKSPACE_CONFIG_TEMPLATE.md +44 -0
- package/agents/templates/brain/KNOWLEDGE_MAP_TEMPLATE.md +53 -0
- package/agents/templates/brain/OODA_LOOP_TRACE_TEMPLATE.md +44 -0
- package/agents/templates/brain/PREDICTION_ERROR_LOG_TEMPLATE.md +33 -0
- package/agents/templates/brain/PROJECT_CONTEXT_TEMPLATE.md +36 -0
- package/agents/templates/brain/SELF_IMPROVEMENT_LOG_TEMPLATE.md +39 -0
- package/agents/templates/brain/WORKING_MEMORY_TEMPLATE.md +34 -0
- package/agents/templates/coding/CLEAN_CODE_CHECKLIST.md +30 -0
- package/agents/templates/coding/DESIGN_TOKENS_TEMPLATE.json +54 -0
- package/agents/templates/coding/DEVCONTAINER_TEMPLATE.json +42 -0
- package/agents/templates/coding/DEV_SPACE_TEMPLATE.yaml +48 -0
- package/agents/templates/coding/E2E_TEST_SUITE_TEMPLATE.md +36 -0
- package/agents/templates/coding/EVAL_PLAN_TEMPLATE.md +85 -0
- package/agents/templates/coding/EVAL_RUBRIC_TEMPLATE.md +57 -0
- package/agents/templates/coding/GOLDEN_DATASET_TEMPLATE.md +73 -0
- package/agents/templates/coding/PROPERTY_TEST_CHECKLIST.md +77 -0
- package/agents/templates/coding/SKETCH_TO_DIAGRAM_TEMPLATE.md +56 -0
- package/agents/templates/coding/TEST_CASE_TEMPLATE.md +28 -0
- package/agents/templates/coding/TEST_PLAN_TEMPLATE.md +36 -0
- package/agents/templates/coding/TIME_COMPLEXITY_REPORT_TEMPLATE.md +61 -0
- package/agents/templates/coordination/AGENTS_TEMPLATE.md +540 -0
- package/agents/templates/coordination/AGENT_HANDSHAKE_TEMPLATE.json +37 -0
- package/agents/templates/coordination/AGENT_PROFILE_TEMPLATE.md +28 -0
- package/agents/templates/coordination/SWARM_ARCHITECTURE_TEMPLATE.md +53 -0
- package/agents/templates/coordination/SWARM_CONFIG_TEMPLATE.json +45 -0
- package/agents/templates/core/AUDIT_LOG_TEMPLATE.md +75 -0
- package/agents/templates/core/BUG_REPORT_TEMPLATE.md +136 -0
- package/agents/templates/core/DEPENDENCY_GRAPH_TEMPLATE.md +63 -0
- package/agents/templates/core/ESTIMATION_TEMPLATE.md +70 -0
- package/agents/templates/core/MCP_CONFIG_TEMPLATE.json +807 -0
- package/agents/templates/core/PLAN_TEMPLATE.md +162 -0
- package/agents/templates/core/PROJECT_RETROSPECTIVE_TEMPLATE.md +67 -0
- package/agents/templates/core/SDLC_TRACKER.md +217 -0
- package/agents/templates/core/SESSION_SNAPSHOT_TEMPLATE.md +168 -0
- package/agents/templates/core/SYSTEM_ANALYSIS_TEMPLATE.md +52 -0
- package/agents/templates/core/TASKS_TEMPLATE.md +104 -0
- package/agents/templates/core/TRACEABILITY_MATRIX_TEMPLATE.md +18 -0
- package/agents/templates/core/WHITEBOARD_DESIGN_TEMPLATE.md +46 -0
- package/agents/templates/data/DATABASE_SCHEMA_TEMPLATE.md +38 -0
- package/agents/templates/data/DATA_PIPELINE_TEMPLATE.md +44 -0
- package/agents/templates/data/ONTOLOGY_TEMPLATE.md +52 -0
- package/agents/templates/industry/ENGINEERING_STANDARDS_REVIEW_TEMPLATE.md +43 -0
- package/agents/templates/industry/FHIR_INTEGRATION_TEMPLATE.md +53 -0
- package/agents/templates/industry/GLOBAL_STANDARDS_AUDIT_TEMPLATE.md +45 -0
- package/agents/templates/industry/IOT_TELEMETRY_TEMPLATE.md +59 -0
- package/agents/templates/industry/TELECOM_API_TEMPLATE.md +50 -0
- package/agents/templates/ops/BENCHMARK_REPORT_TEMPLATE.md +48 -0
- package/agents/templates/ops/CAPACITY_PLAN_TEMPLATE.md +41 -0
- package/agents/templates/ops/COST_OPTIMIZATION_REPORT_TEMPLATE.md +40 -0
- package/agents/templates/ops/DECOMMISSION_PLAN_TEMPLATE.md +102 -0
- package/agents/templates/ops/DEPLOY_CONFIG_TEMPLATE.md +88 -0
- package/agents/templates/ops/INCIDENT_POSTMORTEM_TEMPLATE.md +39 -0
- package/agents/templates/ops/INFRA_PLAN_TEMPLATE.md +37 -0
- package/agents/templates/ops/RELEASE_READINESS_REPORT.md +35 -0
- package/agents/templates/ops/RUNBOOK_TEMPLATE.md +103 -0
- package/agents/templates/ops/TECH_DEBT_TEMPLATE.md +102 -0
- package/agents/templates/product/BUSINESS_CASE_TEMPLATE.md +42 -0
- package/agents/templates/product/CHANGE_MANAGEMENT_PLAN_TEMPLATE.md +68 -0
- package/agents/templates/product/EMPATHY_MAP_TEMPLATE.md +47 -0
- package/agents/templates/product/EXPERIMENT_PLAN_TEMPLATE.md +84 -0
- package/agents/templates/product/GREEN_SOFTWARE_REPORT_TEMPLATE.md +42 -0
- package/agents/templates/product/IDEATION_LOG_TEMPLATE.md +42 -0
- package/agents/templates/product/NFR_TEMPLATE.md +31 -0
- package/agents/templates/product/PRD_TEMPLATE.md +222 -0
- package/agents/templates/product/PROBLEM_STATEMENT_TEMPLATE.md +32 -0
- package/agents/templates/product/REQUIREMENTS_REVIEW_TEMPLATE.md +173 -0
- package/agents/templates/product/SPEC_TEMPLATE.md +259 -0
- package/agents/templates/product/STAKEHOLDER_REGISTER_TEMPLATE.md +127 -0
- package/agents/templates/product/USER_STORY_MAP_TEMPLATE.md +35 -0
- package/agents/templates/product/VISUAL_SPEC_PACKAGE_TEMPLATE.md +207 -0
- package/agents/templates/security/DATASHEET_TEMPLATE.md +95 -0
- package/agents/templates/security/ETHICAL_IMPACT_ASSESSMENT.md +32 -0
- package/agents/templates/security/HAZARD_LOG.md +17 -0
- package/agents/templates/security/MODEL_CARD_TEMPLATE.md +120 -0
- package/agents/templates/security/SAFETY_CASE.md +39 -0
- package/agents/templates/security/SAFETY_POLICY.md +25 -0
- package/agents/templates/security/SECURITY_CHECKLIST.md +157 -0
- package/agents/templates/security/SYSTEM_CARD_TEMPLATE.md +139 -0
- package/agents/templates/security/THREAT_MODEL_TEMPLATE.md +171 -0
- package/bin/install.js +572 -0
- package/docs/CLI_REFERENCE.md +527 -0
- package/docs/INSTALL.md +68 -0
- package/docs/MCP_CONFIGURATIONS.md +1765 -0
- package/docs/PLATFORM_CONTROLS.md +324 -0
- package/docs/POST_INSTALL.md +75 -0
- package/docs/QUICK_COMMANDS.md +131 -0
- package/docs/QUICK_GUIDE.md +914 -0
- package/docs/README_FULL.md +1451 -0
- package/docs/SCHEMA.md +164 -0
- package/docs/V1_RELEASE_CHECKLIST.md +144 -0
- package/docs/VERIFICATION_GUIDE.md +283 -0
- package/docs/adr/0001-record-architecture-decisions.md +26 -0
- package/docs/adr/0002-brain-active-inference-runtime.md +27 -0
- package/docs/adr/0003-additive-only-emitter-schema-evolution.md +33 -0
- package/docs/assets/GABBE.png +0 -0
- package/docs/gabbe.config.example.json +9 -0
- package/docs/reports/duplication-report.md +30 -0
- package/install.ps1 +90 -0
- package/install.sh +89 -0
- package/package.json +49 -0
- package/scripts/init.py +1036 -0
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
"""Skill self-eval harness (Track E2).
|
|
4
|
+
|
|
5
|
+
Scores GABBE's own markdown skills against golden datasets stored beside them at
|
|
6
|
+
agents/skills/<category>/evals/<skill>.eval.yaml (promptfoo-compatible shape).
|
|
7
|
+
|
|
8
|
+
Two lanes — matching the eval philosophy that tests verify determinism while
|
|
9
|
+
evals score probabilistic quality:
|
|
10
|
+
|
|
11
|
+
* default / --check (deterministic, per-commit): lints every eval suite for a
|
|
12
|
+
valid structure and self-tests the deterministic assertion evaluators on
|
|
13
|
+
built-in fixtures. Needs NO model, so it is safe to gate per-commit.
|
|
14
|
+
|
|
15
|
+
* --live (GABBE_LIVE_LLM=1, nightly): renders each prompt, calls the model via
|
|
16
|
+
gabbe/llm.py (reused when the CLI is importable), evaluates the assertions,
|
|
17
|
+
and writes a JSON scorecard. Non-blocking by design.
|
|
18
|
+
|
|
19
|
+
HONESTY: evals SAMPLE the input space and raise confidence — they do not prove
|
|
20
|
+
correctness. LLM-as-judge assertions are biased-but-useful, never ground truth.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import argparse
|
|
26
|
+
import json
|
|
27
|
+
import re
|
|
28
|
+
import sys
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
from typing import Any
|
|
31
|
+
|
|
32
|
+
AGENTS_DIR = Path(__file__).resolve().parent.parent
|
|
33
|
+
SKILLS_DIR = AGENTS_DIR / "skills"
|
|
34
|
+
|
|
35
|
+
GREEN = "\033[0;32m"
|
|
36
|
+
RED = "\033[0;31m"
|
|
37
|
+
YELLOW = "\033[0;33m"
|
|
38
|
+
BLUE = "\033[0;34m"
|
|
39
|
+
NC = "\033[0m"
|
|
40
|
+
|
|
41
|
+
# Assertion types evaluable WITHOUT a model, given a candidate output.
|
|
42
|
+
DETERMINISTIC_TYPES = {
|
|
43
|
+
"equals",
|
|
44
|
+
"contains",
|
|
45
|
+
"icontains",
|
|
46
|
+
"not-contains",
|
|
47
|
+
"regex",
|
|
48
|
+
"is-json",
|
|
49
|
+
"min-length",
|
|
50
|
+
}
|
|
51
|
+
# Assertion types that require a model / embeddings (live lane only).
|
|
52
|
+
LIVE_ONLY_TYPES = {"similar", "llm-rubric"}
|
|
53
|
+
KNOWN_TYPES = DETERMINISTIC_TYPES | LIVE_ONLY_TYPES
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
# ---------------------------------------------------------------------------
|
|
57
|
+
# Deterministic assertion evaluators (pure functions — unit-testable, no model)
|
|
58
|
+
# ---------------------------------------------------------------------------
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def assert_output(assertion: dict[str, Any], output: str) -> tuple[bool, str]:
|
|
62
|
+
"""Evaluate a single deterministic assertion against a candidate output."""
|
|
63
|
+
t = assertion.get("type")
|
|
64
|
+
val = assertion.get("value")
|
|
65
|
+
if t == "equals":
|
|
66
|
+
return output.strip() == str(val).strip(), "equals"
|
|
67
|
+
if t == "contains":
|
|
68
|
+
return str(val) in output, "contains"
|
|
69
|
+
if t == "icontains":
|
|
70
|
+
return str(val).lower() in output.lower(), "icontains"
|
|
71
|
+
if t == "not-contains":
|
|
72
|
+
return str(val) not in output, "not-contains"
|
|
73
|
+
if t == "regex":
|
|
74
|
+
return re.search(str(val), output) is not None, "regex"
|
|
75
|
+
if t == "is-json":
|
|
76
|
+
try:
|
|
77
|
+
json.loads(output)
|
|
78
|
+
return True, "is-json"
|
|
79
|
+
except (ValueError, TypeError):
|
|
80
|
+
return False, "is-json"
|
|
81
|
+
if t == "min-length":
|
|
82
|
+
return len(output) >= int(str(val)), "min-length"
|
|
83
|
+
raise ValueError(f"not a deterministic assertion type: {t!r}")
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
# ---------------------------------------------------------------------------
|
|
87
|
+
# Suite loading + structural lint
|
|
88
|
+
# ---------------------------------------------------------------------------
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _load_yaml(path: Path) -> Any:
|
|
92
|
+
import yaml # PyYAML is a core GABBE dependency
|
|
93
|
+
|
|
94
|
+
return yaml.safe_load(path.read_text())
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def discover_suites(root: Path = SKILLS_DIR) -> list[Path]:
|
|
98
|
+
return sorted(root.glob("**/evals/*.eval.yaml"))
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def lint_suite(path: Path) -> list[str]:
|
|
102
|
+
"""Return a list of structural errors for one eval suite (empty == valid)."""
|
|
103
|
+
errors: list[str] = []
|
|
104
|
+
try:
|
|
105
|
+
suite = _load_yaml(path)
|
|
106
|
+
except Exception as e: # noqa: BLE001 - surface the YAML error
|
|
107
|
+
return [f"{path.name}: not valid YAML ({str(e).splitlines()[0]})"]
|
|
108
|
+
if not isinstance(suite, dict):
|
|
109
|
+
return [f"{path.name}: top-level must be a mapping"]
|
|
110
|
+
if not str(suite.get("description", "")).strip():
|
|
111
|
+
errors.append(f"{path.name}: missing 'description'")
|
|
112
|
+
tests = suite.get("tests")
|
|
113
|
+
if not isinstance(tests, list) or not tests:
|
|
114
|
+
return errors + [f"{path.name}: 'tests' must be a non-empty list"]
|
|
115
|
+
for i, test in enumerate(tests):
|
|
116
|
+
if not isinstance(test, dict):
|
|
117
|
+
errors.append(f"{path.name}: test #{i} must be a mapping")
|
|
118
|
+
continue
|
|
119
|
+
asserts = test.get("assert")
|
|
120
|
+
if not isinstance(asserts, list) or not asserts:
|
|
121
|
+
errors.append(f"{path.name}: test #{i} has no 'assert' list")
|
|
122
|
+
continue
|
|
123
|
+
for a in asserts:
|
|
124
|
+
if not isinstance(a, dict) or a.get("type") not in KNOWN_TYPES:
|
|
125
|
+
errors.append(f"{path.name}: test #{i} has an unknown assert type {a!r}")
|
|
126
|
+
elif a.get("type") in DETERMINISTIC_TYPES and "value" not in a:
|
|
127
|
+
errors.append(f"{path.name}: test #{i} {a.get('type')} assert needs a 'value'")
|
|
128
|
+
return errors
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
# Built-in fixtures proving the deterministic evaluators behave correctly. Run in
|
|
132
|
+
# the per-commit lane so a regression in the harness itself is caught with no model.
|
|
133
|
+
_SELF_TESTS: list[tuple[dict[str, Any], str, bool]] = [
|
|
134
|
+
({"type": "contains", "value": "?"}, "Do you mean X?", True),
|
|
135
|
+
({"type": "contains", "value": "?"}, "A statement.", False),
|
|
136
|
+
({"type": "icontains", "value": "ERROR"}, "fatal error here", True),
|
|
137
|
+
({"type": "not-contains", "value": "secret"}, "all clear", True),
|
|
138
|
+
({"type": "regex", "value": r"^\d{3}$"}, "404", True),
|
|
139
|
+
({"type": "regex", "value": r"^\d{3}$"}, "40x", False),
|
|
140
|
+
({"type": "is-json", "value": None}, '{"ok": true}', True),
|
|
141
|
+
({"type": "is-json", "value": None}, "not json", False),
|
|
142
|
+
({"type": "min-length", "value": 5}, "abcdef", True),
|
|
143
|
+
({"type": "min-length", "value": 5}, "abc", False),
|
|
144
|
+
({"type": "equals", "value": "yes"}, " yes ", True),
|
|
145
|
+
]
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def self_test_evaluators() -> list[str]:
|
|
149
|
+
failures = []
|
|
150
|
+
for assertion, output, expected in _SELF_TESTS:
|
|
151
|
+
got, _ = assert_output(assertion, output)
|
|
152
|
+
if got != expected:
|
|
153
|
+
failures.append(f"evaluator {assertion} on {output!r} -> {got}, expected {expected}")
|
|
154
|
+
return failures
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def run_check() -> int:
|
|
158
|
+
"""Per-commit deterministic lane: lint suites + self-test evaluators."""
|
|
159
|
+
print(f"{BLUE}== Eval harness self-check (deterministic, no model) =={NC}")
|
|
160
|
+
errors: list[str] = []
|
|
161
|
+
suites = discover_suites()
|
|
162
|
+
for path in suites:
|
|
163
|
+
errors.extend(lint_suite(path))
|
|
164
|
+
print(f" linted {len(suites)} eval suite(s)")
|
|
165
|
+
eval_failures = self_test_evaluators()
|
|
166
|
+
errors.extend(eval_failures)
|
|
167
|
+
print(f" self-tested {len(_SELF_TESTS)} deterministic evaluator cases")
|
|
168
|
+
if errors:
|
|
169
|
+
for e in errors:
|
|
170
|
+
print(f"{RED}x {e}{NC}")
|
|
171
|
+
print(f"{RED}eval self-check FAILED with {len(errors)} error(s){NC}")
|
|
172
|
+
return 1
|
|
173
|
+
print(f"{GREEN}eval self-check passed{NC}")
|
|
174
|
+
return 0
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
# ---------------------------------------------------------------------------
|
|
178
|
+
# Live lane (nightly): render prompts, call the model, score, write a scorecard
|
|
179
|
+
# ---------------------------------------------------------------------------
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _render(template: str, variables: dict[str, Any]) -> str:
|
|
183
|
+
out = template
|
|
184
|
+
for k, v in (variables or {}).items():
|
|
185
|
+
out = out.replace("{{" + k + "}}", str(v)).replace("{{ " + k + " }}", str(v))
|
|
186
|
+
return out
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def run_live(out_path: Path | None) -> int:
|
|
190
|
+
print(f"{BLUE}== Eval harness LIVE lane (calls the model) =={NC}")
|
|
191
|
+
try:
|
|
192
|
+
from gabbe.llm import call_llm # reuse the CLI's model adapter + caching
|
|
193
|
+
except Exception as e: # noqa: BLE001
|
|
194
|
+
print(f"{RED}live mode needs the gabbe CLI importable (gabbe.llm): {e}{NC}")
|
|
195
|
+
return 2
|
|
196
|
+
|
|
197
|
+
scorecard: dict[str, Any] = {"suites": [], "summary": {}}
|
|
198
|
+
total = passed = 0
|
|
199
|
+
for path in discover_suites():
|
|
200
|
+
suite = _load_yaml(path)
|
|
201
|
+
prompts = suite.get("prompts") or ["{{input}}"]
|
|
202
|
+
suite_rec: dict[str, Any] = {"suite": str(path.relative_to(AGENTS_DIR)), "cases": []}
|
|
203
|
+
for test in suite.get("tests", []):
|
|
204
|
+
rendered = _render(prompts[0], test.get("vars", {}))
|
|
205
|
+
output = call_llm(rendered) or ""
|
|
206
|
+
case_passed = True
|
|
207
|
+
details: list[dict[str, Any]] = []
|
|
208
|
+
for a in test.get("assert", []):
|
|
209
|
+
atype = a.get("type")
|
|
210
|
+
if atype in DETERMINISTIC_TYPES:
|
|
211
|
+
ok, label = assert_output(a, output)
|
|
212
|
+
elif atype == "llm-rubric":
|
|
213
|
+
ok, label = _judge(call_llm, a.get("value", ""), output)
|
|
214
|
+
else: # similar — needs embeddings we don't ship; SKIP honestly
|
|
215
|
+
# passed=None: recorded but NOT counted toward the pass rate
|
|
216
|
+
# (skipping must never silently inflate the scorecard).
|
|
217
|
+
details.append({"type": f"{atype} (skipped: needs embeddings)", "passed": None})
|
|
218
|
+
continue
|
|
219
|
+
details.append({"type": label, "passed": ok})
|
|
220
|
+
case_passed = case_passed and ok
|
|
221
|
+
total += 1
|
|
222
|
+
passed += 1 if case_passed else 0
|
|
223
|
+
suite_rec["cases"].append({"passed": case_passed, "asserts": details})
|
|
224
|
+
scorecard["suites"].append(suite_rec)
|
|
225
|
+
|
|
226
|
+
scorecard["summary"] = {
|
|
227
|
+
"cases": total,
|
|
228
|
+
"passed": passed,
|
|
229
|
+
"pass_rate": round(passed / total, 4) if total else None,
|
|
230
|
+
"note": "evals sample; pass_rate raises confidence, it does not prove correctness",
|
|
231
|
+
}
|
|
232
|
+
text = json.dumps(scorecard, indent=2)
|
|
233
|
+
if out_path:
|
|
234
|
+
out_path.write_text(text)
|
|
235
|
+
print(f" scorecard written to {out_path}")
|
|
236
|
+
else:
|
|
237
|
+
print(text)
|
|
238
|
+
print(f"{GREEN}live eval complete: {passed}/{total} cases passed{NC}")
|
|
239
|
+
return 0
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _judge(call_llm: Any, rubric: str, output: str) -> tuple[bool, str]:
|
|
243
|
+
"""Minimal LLM-as-judge: ask for a PASS/FAIL verdict against a rubric."""
|
|
244
|
+
prompt = (
|
|
245
|
+
"You are a strict evaluator. Given the RUBRIC and the CANDIDATE OUTPUT, "
|
|
246
|
+
"reply with exactly 'PASS' or 'FAIL' on the first line.\n\n"
|
|
247
|
+
f"RUBRIC:\n{rubric}\n\nCANDIDATE OUTPUT:\n{output}\n"
|
|
248
|
+
)
|
|
249
|
+
verdict = (call_llm(prompt) or "").strip().upper()
|
|
250
|
+
return verdict.startswith("PASS"), "llm-rubric"
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def main(argv: list[str] | None = None) -> int:
|
|
254
|
+
parser = argparse.ArgumentParser(description="GABBE skill eval harness (Track E2)")
|
|
255
|
+
parser.add_argument(
|
|
256
|
+
"--live",
|
|
257
|
+
action="store_true",
|
|
258
|
+
help="Run the model and score outputs (nightly; needs GABBE_LIVE_LLM=1 + an API key)",
|
|
259
|
+
)
|
|
260
|
+
parser.add_argument("--out", type=Path, default=None, help="Write the JSON scorecard here")
|
|
261
|
+
args = parser.parse_args(argv)
|
|
262
|
+
if args.live:
|
|
263
|
+
return run_live(args.out)
|
|
264
|
+
return run_check()
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
if __name__ == "__main__":
|
|
268
|
+
sys.exit(main())
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
#!/bin/sh
|
|
2
|
+
# preflight_summary.sh — print a compact knowledge + cost summary for preflight.skill.
|
|
3
|
+
#
|
|
4
|
+
# Python-independent on purpose: the Markdown preflight.skill is primary; this
|
|
5
|
+
# helper just prints the same SESSION_PREFLIGHT facts so any agent (or a human)
|
|
6
|
+
# can see the inventory + cost posture without the optional gabbe CLI.
|
|
7
|
+
#
|
|
8
|
+
# Usage: sh agents/scripts/preflight_summary.sh [KIT_ROOT]
|
|
9
|
+
# KIT_ROOT defaults to the directory two levels up from this script (repo root).
|
|
10
|
+
|
|
11
|
+
set -eu
|
|
12
|
+
|
|
13
|
+
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
|
14
|
+
KIT_ROOT=${1:-$(CDPATH= cd -- "$SCRIPT_DIR/../.." && pwd)}
|
|
15
|
+
AGENTS_DIR="$KIT_ROOT/agents"
|
|
16
|
+
MEM_DIR="$AGENTS_DIR/memory"
|
|
17
|
+
|
|
18
|
+
count() { find "$1" -name "$2" 2>/dev/null | wc -l | tr -d ' '; }
|
|
19
|
+
|
|
20
|
+
echo "## SESSION_PREFLIGHT (summary)"
|
|
21
|
+
echo ""
|
|
22
|
+
|
|
23
|
+
echo "### Inventory"
|
|
24
|
+
echo "- Skills: $(count "$AGENTS_DIR/skills" '*.skill.md')"
|
|
25
|
+
echo "- Guides: $(count "$AGENTS_DIR/guides" '*.md')"
|
|
26
|
+
echo "- Templates: $(count "$AGENTS_DIR/templates" '*')"
|
|
27
|
+
echo "- Personas: $(count "$AGENTS_DIR/personas" '*.md')"
|
|
28
|
+
echo ""
|
|
29
|
+
|
|
30
|
+
echo "### Cost posture"
|
|
31
|
+
# Autonomy: project/gabbe.config.json wins, then env, else default hybrid.
|
|
32
|
+
AUTONOMY="hybrid"
|
|
33
|
+
CFG="$KIT_ROOT/project/gabbe.config.json"
|
|
34
|
+
if [ -f "$CFG" ]; then
|
|
35
|
+
VAL=$(grep -o '"autonomy"[[:space:]]*:[[:space:]]*"[a-z]*"' "$CFG" 2>/dev/null | grep -o '[a-z]*"$' | tr -d '"' || true)
|
|
36
|
+
[ -n "${VAL:-}" ] && AUTONOMY="$VAL"
|
|
37
|
+
fi
|
|
38
|
+
[ -n "${GABBE_AUTONOMY:-}" ] && AUTONOMY="$GABBE_AUTONOMY"
|
|
39
|
+
echo "- Autonomy: $AUTONOMY (ask | auto | hybrid; default hybrid)"
|
|
40
|
+
echo "- Budget: markdown-enforced unless the optional gabbe CLI reports live limits"
|
|
41
|
+
echo "- Cost levers: prompt-caching, context-budgeting, model-tiering, batching"
|
|
42
|
+
echo ""
|
|
43
|
+
|
|
44
|
+
echo "### Memory headers"
|
|
45
|
+
if [ -f "$MEM_DIR/PROJECT_STATE.md" ]; then
|
|
46
|
+
PHASE=$(grep -i -m1 'phase' "$MEM_DIR/PROJECT_STATE.md" 2>/dev/null || true)
|
|
47
|
+
echo "- PROJECT_STATE: ${PHASE:-present}"
|
|
48
|
+
else
|
|
49
|
+
echo "- PROJECT_STATE: (none — fresh project)"
|
|
50
|
+
fi
|
|
51
|
+
if [ -f "$MEM_DIR/CONTINUITY.md" ]; then
|
|
52
|
+
echo "- CONTINUITY: present (read before acting — past failures to avoid)"
|
|
53
|
+
else
|
|
54
|
+
echo "- CONTINUITY: (none)"
|
|
55
|
+
fi
|
|
56
|
+
SNAP_DIR="$MEM_DIR/episodic/SESSION_SNAPSHOT"
|
|
57
|
+
if [ -d "$SNAP_DIR" ]; then
|
|
58
|
+
LATEST=$(ls -1t "$SNAP_DIR" 2>/dev/null | head -1 || true)
|
|
59
|
+
echo "- Latest snapshot: ${LATEST:-none}"
|
|
60
|
+
fi
|
|
61
|
+
echo ""
|
|
62
|
+
|
|
63
|
+
# New/changed skills vs last preflight (best-effort, stored under memory/).
|
|
64
|
+
echo "### New/changed since last preflight"
|
|
65
|
+
STAMP="$MEM_DIR/.preflight_skill_count"
|
|
66
|
+
NOW=$(count "$AGENTS_DIR/skills" '*.skill.md')
|
|
67
|
+
if [ -f "$STAMP" ]; then
|
|
68
|
+
PREV=$(cat "$STAMP" 2>/dev/null || echo 0)
|
|
69
|
+
if [ "$NOW" != "$PREV" ]; then
|
|
70
|
+
echo "- Skill count changed: $PREV -> $NOW (run update-scan.skill to review)"
|
|
71
|
+
else
|
|
72
|
+
echo "- No change in skill count ($NOW)"
|
|
73
|
+
fi
|
|
74
|
+
else
|
|
75
|
+
echo "- First preflight (baseline $NOW skills recorded)"
|
|
76
|
+
fi
|
|
77
|
+
# Record the current count for next time (best-effort; ignore failure on read-only fs).
|
|
78
|
+
printf '%s' "$NOW" > "$STAMP" 2>/dev/null || true
|
|
79
|
+
echo ""
|
|
80
|
+
echo "> Markdown preflight.skill is authoritative; this is a convenience summary."
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
"""R15: near-duplicate content report across the GABBE Markdown corpus.
|
|
4
|
+
|
|
5
|
+
Stdlib-only. Uses normalized line-shingle Jaccard similarity to find pairs of
|
|
6
|
+
skills/templates/guides that overlap heavily, so maintainers can consolidate
|
|
7
|
+
drifted or copy-pasted material. Report-only — it never edits or deletes files.
|
|
8
|
+
|
|
9
|
+
Usage:
|
|
10
|
+
python agents/scripts/report_duplicates.py [--threshold 0.6] [--top 20]
|
|
11
|
+
|
|
12
|
+
Writes docs/reports/duplication-report.md and prints a summary.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
import argparse
|
|
16
|
+
import re
|
|
17
|
+
import sys
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
ROOT = Path(__file__).resolve().parent.parent.parent
|
|
21
|
+
CORPUS_DIRS = [
|
|
22
|
+
ROOT / "agents" / "skills",
|
|
23
|
+
ROOT / "agents" / "templates",
|
|
24
|
+
ROOT / "agents" / "guides",
|
|
25
|
+
ROOT / "agents" / "personas",
|
|
26
|
+
]
|
|
27
|
+
OUT = ROOT / "docs" / "reports" / "duplication-report.md"
|
|
28
|
+
|
|
29
|
+
_WS = re.compile(r"\s+")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def normalize(text):
|
|
33
|
+
"""Strip frontmatter, lowercase, collapse whitespace, drop short lines."""
|
|
34
|
+
if text.startswith("---"):
|
|
35
|
+
end = text.find("---", 3)
|
|
36
|
+
if end != -1:
|
|
37
|
+
text = text[end + 3 :]
|
|
38
|
+
lines = []
|
|
39
|
+
for line in text.splitlines():
|
|
40
|
+
line = _WS.sub(" ", line.strip().lower())
|
|
41
|
+
if len(line) >= 12: # ignore trivial lines (headers, bullets)
|
|
42
|
+
lines.append(line)
|
|
43
|
+
return lines
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def shingles(lines, k=3):
|
|
47
|
+
"""Set of k-line shingles."""
|
|
48
|
+
return {" | ".join(lines[i : i + k]) for i in range(max(0, len(lines) - k + 1))}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def jaccard(a, b):
|
|
52
|
+
if not a or not b:
|
|
53
|
+
return 0.0
|
|
54
|
+
inter = len(a & b)
|
|
55
|
+
union = len(a | b)
|
|
56
|
+
return inter / union if union else 0.0
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def collect():
|
|
60
|
+
docs = {}
|
|
61
|
+
for d in CORPUS_DIRS:
|
|
62
|
+
if not d.exists():
|
|
63
|
+
continue
|
|
64
|
+
for p in sorted(d.rglob("*.md")):
|
|
65
|
+
if p.name.startswith("00-index"):
|
|
66
|
+
continue
|
|
67
|
+
try:
|
|
68
|
+
sh = shingles(normalize(p.read_text(encoding="utf-8")))
|
|
69
|
+
except Exception:
|
|
70
|
+
continue
|
|
71
|
+
if sh:
|
|
72
|
+
docs[str(p.relative_to(ROOT))] = sh
|
|
73
|
+
return docs
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def main():
|
|
77
|
+
ap = argparse.ArgumentParser(description=__doc__)
|
|
78
|
+
ap.add_argument("--threshold", type=float, default=0.6)
|
|
79
|
+
ap.add_argument("--top", type=int, default=20)
|
|
80
|
+
args = ap.parse_args()
|
|
81
|
+
|
|
82
|
+
docs = collect()
|
|
83
|
+
names = list(docs)
|
|
84
|
+
all_pairs = []
|
|
85
|
+
for i in range(len(names)):
|
|
86
|
+
for j in range(i + 1, len(names)):
|
|
87
|
+
sim = jaccard(docs[names[i]], docs[names[j]])
|
|
88
|
+
if sim > 0:
|
|
89
|
+
all_pairs.append((sim, names[i], names[j]))
|
|
90
|
+
all_pairs.sort(reverse=True)
|
|
91
|
+
over = [p for p in all_pairs if p[0] >= args.threshold]
|
|
92
|
+
|
|
93
|
+
OUT.parent.mkdir(parents=True, exist_ok=True)
|
|
94
|
+
lines = [
|
|
95
|
+
"# GABBE Duplication Report",
|
|
96
|
+
"",
|
|
97
|
+
f"Generated by `agents/scripts/report_duplicates.py` over {len(docs)} "
|
|
98
|
+
"corpus files (skills, templates, guides, personas).",
|
|
99
|
+
"Method: normalized 3-line shingle Jaccard similarity. Report-only — "
|
|
100
|
+
"no files were modified.",
|
|
101
|
+
"",
|
|
102
|
+
f"**{len(over)} pair(s)** at or above the consolidation threshold "
|
|
103
|
+
f"({args.threshold}). The corpus is largely non-duplicative; the "
|
|
104
|
+
f"top {min(args.top, len(all_pairs))} most-similar pairs are listed "
|
|
105
|
+
"below for review.",
|
|
106
|
+
"",
|
|
107
|
+
"| Similarity | File A | File B |",
|
|
108
|
+
"|---|---|---|",
|
|
109
|
+
]
|
|
110
|
+
for sim, a, b in all_pairs[: args.top]:
|
|
111
|
+
flag = " ⚠️" if sim >= args.threshold else ""
|
|
112
|
+
lines.append(f"| {sim:.2f}{flag} | `{a}` | `{b}` |")
|
|
113
|
+
if not all_pairs:
|
|
114
|
+
lines.append("| — | _no measurable overlap_ | — |")
|
|
115
|
+
lines.append("")
|
|
116
|
+
OUT.write_text("\n".join(lines) + "\n")
|
|
117
|
+
print(
|
|
118
|
+
f"Wrote {OUT.relative_to(ROOT)} — {len(over)} pair(s) >= "
|
|
119
|
+
f"{args.threshold} of {len(all_pairs)} with measurable overlap"
|
|
120
|
+
)
|
|
121
|
+
return 0
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
if __name__ == "__main__":
|
|
125
|
+
sys.exit(main())
|