@polymorphism-tech/morph-spec 8.47.0 → 9.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +51 -43
- package/README.md +39 -38
- package/bin/morph-spec.js +41 -113
- package/bin/validate.js +1 -147
- package/claude-plugin.json +1 -1
- package/docs/CHEATSHEET.md +1 -1
- package/docs/QUICKSTART.md +1 -1
- package/framework/CLAUDE.md +41 -101
- package/framework/MORPH.md +95 -252
- package/framework/agents.json +86 -21
- package/framework/commands/morph-apply.md +5 -28
- package/framework/commands/morph-archive.md +1 -1
- package/framework/commands/morph-hotfix.md +8 -8
- package/framework/commands/morph-infra.md +1 -1
- package/framework/commands/morph-preflight.md +2 -3
- package/framework/commands/morph-status.md +4 -4
- package/framework/docs/CLI.md +574 -286
- package/framework/evals/review-checklist.md +82 -41
- package/framework/hooks/README.md +10 -11
- package/framework/hooks/claude-code/core/block-destructive-worktree-removal.js +6 -49
- package/framework/hooks/claude-code/core/block-raw-worktree-add.js +2 -35
- package/framework/hooks/claude-code/core/chain.js +4 -65
- package/framework/hooks/claude-code/core/comment-guard.js +3 -17
- package/framework/hooks/claude-code/core/denylist.json +33 -3
- package/framework/hooks/claude-code/core/enforce-phase-writes.js +15 -63
- package/framework/hooks/claude-code/core/gate-guard.js +9 -78
- package/framework/hooks/claude-code/core/graph-refresh-kick.js +3 -24
- package/framework/hooks/claude-code/core/heavy-gate.js +3 -21
- package/framework/hooks/claude-code/core/heavy-release.js +2 -13
- package/framework/hooks/claude-code/core/loop-detect.js +2 -31
- package/framework/hooks/claude-code/core/pre-bash-denylist.js +36 -2
- package/framework/hooks/claude-code/core/protect-spec-files.js +25 -297
- package/framework/hooks/claude-code/core/state-sync.js +15 -85
- package/framework/hooks/claude-code/core/telemetry-log.js +15 -96
- package/framework/hooks/claude-code/core/validate-completion.js +37 -147
- package/framework/hooks/claude-code/core/warn-bash-source-write.js +23 -0
- package/framework/hooks/claude-code/core/worktree-context.js +15 -114
- package/framework/hooks/claude-code/core/worktree-create.js +6 -70
- package/framework/hooks/claude-code/core/worktree-env-inject.js +5 -56
- package/framework/hooks/claude-code/core/worktree-remove.js +5 -31
- package/framework/hooks/claude-code/statusline.py +55 -41
- package/framework/hooks/package.json +3 -0
- package/framework/hooks/shared/activity-logger.js +9 -89
- package/framework/hooks/shared/agent-reap.js +2 -17
- package/framework/hooks/shared/build-check.js +11 -76
- package/framework/hooks/shared/claim-registry.js +9 -119
- package/framework/hooks/shared/command-tokens.js +168 -44
- package/framework/hooks/shared/comment-rules.js +5 -20
- package/framework/hooks/shared/comment-scanner.js +11 -43
- package/framework/hooks/shared/comment-stripper.js +11 -85
- package/framework/hooks/shared/e2e-check.js +0 -0
- package/framework/hooks/shared/errata.js +59 -0
- package/framework/hooks/shared/feature-locator.js +11 -95
- package/framework/hooks/shared/frontend-locations.js +4 -0
- package/framework/hooks/shared/gate-question.js +12 -0
- package/framework/hooks/shared/git-exclude.js +3 -20
- package/framework/hooks/shared/git-paths.js +8 -104
- package/framework/hooks/shared/graph-freshness.js +22 -166
- package/framework/hooks/shared/graph-refresh.js +21 -149
- package/framework/hooks/shared/heavy-command.js +4 -12
- package/framework/hooks/shared/hook-response.js +8 -83
- package/framework/hooks/shared/morph-cli.js +8 -52
- package/framework/hooks/shared/payload-utils.js +10 -132
- package/framework/hooks/shared/phase-utils.js +14 -72
- package/framework/hooks/shared/redact-secrets.js +4 -15
- package/framework/hooks/shared/reparse-scan.js +6 -66
- package/framework/hooks/shared/slice-progress.js +99 -0
- package/framework/hooks/shared/source-files.js +4 -17
- package/framework/hooks/shared/spawn-command.js +23 -0
- package/framework/hooks/shared/state-reader.js +51 -247
- package/framework/hooks/shared/state-writer.js +9 -54
- package/framework/hooks/shared/stdin-reader.js +1 -11
- package/framework/hooks/shared/telemetry-logger.js +7 -67
- package/framework/hooks/shared/transcript-usage.js +7 -29
- package/framework/hooks/shared/worktree-root.js +7 -29
- package/framework/mcp/registry.json +3 -4
- package/framework/phases.json +23 -39
- package/framework/rules/csharp-standards.md +1 -1
- package/framework/rules/frontend-standards.md +1 -0
- package/framework/rules/hangfire-standards.md +1 -1
- package/framework/rules/neon-standards.md +1 -1
- package/framework/schemas/config.schema.json +14 -9
- package/framework/schemas/feature.schema.json +15 -50
- package/framework/skills/README.md +3 -4
- package/framework/skills/core/create-umbraco-lp/SKILL.md +1 -1
- package/framework/skills/core/migrate-lp-to-umbraco/SKILL.md +1 -1
- package/framework/skills/core/morph-apply/SKILL.md +149 -222
- package/framework/skills/core/morph-hotfix/SKILL.md +33 -42
- package/framework/skills/core/morph-machine/SKILL.md +2 -2
- package/framework/skills/core/morph-media/SKILL.md +1 -1
- package/framework/skills/core/morph-plan/SKILL.md +137 -299
- package/framework/skills/core/morph-review/SKILL.md +111 -136
- package/framework/skills/core/morph-standards/SKILL.md +27 -39
- package/framework/skills/core/morph-uiux/SKILL.md +94 -152
- package/framework/standards/STANDARDS.json +109 -1219
- package/framework/standards/STANDARDS.meta.json +1319 -0
- package/framework/standards/ai-agents/agent-archetypes.md +2 -2
- package/framework/standards/ai-agents/agent-session.md +1 -1
- package/framework/standards/ai-agents/conversational-agent-with-phases.md +4 -4
- package/framework/standards/ai-agents/durable-workflows-hitl.md +3 -3
- package/framework/standards/ai-agents/evals-with-cache.md +1 -2
- package/framework/standards/ai-agents/guardrails.md +1 -1
- package/framework/standards/ai-agents/multi-agent-patterns.md +2 -2
- package/framework/standards/ai-agents/observability-patterns.md +1 -1
- package/framework/standards/ai-agents/production.md +1 -1
- package/framework/standards/ai-agents/prompt-sources.md +2 -2
- package/framework/standards/ai-agents/providers/model-registry.md +1 -1
- package/framework/standards/ai-agents/rag-custom-pgvector.md +2 -2
- package/framework/standards/ai-agents/setup.md +3 -3
- package/framework/standards/ai-agents/sweet-spot.md +2 -2
- package/framework/standards/ai-agents/tool-adapters.md +0 -2
- package/framework/standards/ai-agents/vector-data-extensions.md +4 -4
- package/framework/standards/ai-agents/workflows.md +1 -1
- package/framework/standards/architecture/code-comments.md +7 -7
- package/framework/standards/architecture/vertical-slice/vertical-slice.md +1 -1
- package/framework/standards/backend/api/validation.md +1 -1
- package/framework/standards/backend/dotnet/vsa-handler-patterns.md +5 -5
- package/framework/standards/backend/integrations/hangfire/hangfire-jobs.md +1 -1
- package/framework/standards/backend/integrations/neon-auth/neon-auth.md +2 -2
- package/framework/standards/data/nosql/cache/redis.md +1 -1
- package/framework/standards/data/nosql/cosmos-db.md +3 -3
- package/framework/standards/frontend/admin-app/forms.md +1 -1
- package/framework/standards/frontend/design-system/art-direction.md +88 -0
- package/framework/standards/frontend/design-system/premium-finish.md +77 -2
- package/framework/standards/infrastructure/neon/neon-pgvector.md +5 -5
- package/framework/standards/integration/mcp/mcp-tools.md +7 -7
- package/framework/templates/CLAUDE.md.core.template +3 -3
- package/framework/templates/README.md +2 -3
- package/framework/templates/REGISTRY.json +15 -179
- package/framework/templates/docs/plan.md +72 -0
- package/framework/templates/docs/ui-mockups.md +4 -0
- package/framework/templates/dotnet/ai-kit/README.md +1 -9
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Morph.AiKit.Tests.csproj +0 -10
- package/framework/templates/meta-prompts/squad-leaders/backend-squad.md +1 -1
- package/framework/templates/meta-prompts/squad-leaders/frontend-squad.md +1 -1
- package/framework/templates/meta-prompts/squad-leaders/squad-leader.md +1 -1
- package/framework/templates/meta-prompts/validators/pre-commit-validator.md +3 -4
- package/package.json +1 -1
- package/src/commands/context/context-refresh.js +0 -16
- package/src/commands/dashboard/dashboard.js +2 -8
- package/src/commands/e2e/e2e.js +18 -105
- package/src/commands/graph/affected.js +0 -14
- package/src/commands/graph/build.js +11 -50
- package/src/commands/graph/explain.js +0 -16
- package/src/commands/graph/status.js +1 -12
- package/src/commands/mcp/mcp-setup.js +4 -62
- package/src/commands/observability/cost.js +17 -111
- package/src/commands/observability/telemetry.js +1 -14
- package/src/commands/project/branch.js +4 -45
- package/src/commands/project/claim.js +4 -48
- package/src/commands/project/doctor-procs.js +1 -1
- package/src/commands/project/doctor.js +53 -307
- package/src/commands/project/env.js +1 -27
- package/src/commands/project/fleet.js +6 -35
- package/src/commands/project/heavy.js +3 -3
- package/src/commands/project/init.js +8 -68
- package/src/commands/project/install-plugin-cmd.js +0 -1
- package/src/commands/project/setup-infra-cmd.js +2 -1
- package/src/commands/project/stack-scan.js +58 -0
- package/src/commands/project/standards.js +65 -72
- package/src/commands/project/status.js +28 -121
- package/src/commands/project/update.js +31 -223
- package/src/commands/project/worktree.js +109 -883
- package/src/commands/scaffold/scaffold.js +0 -25
- package/src/commands/state/advance.js +65 -120
- package/src/commands/state/approve.js +87 -229
- package/src/commands/state/archive.js +23 -139
- package/src/commands/state/auto-approval.js +65 -0
- package/src/commands/state/create.js +49 -109
- package/src/commands/state/delete.js +11 -63
- package/src/commands/state/finish.js +105 -494
- package/src/commands/state/gate-check.js +74 -183
- package/src/commands/state/gate-decision.js +8 -42
- package/src/commands/state/handoff.js +4 -31
- package/src/commands/state/list.js +21 -91
- package/src/commands/state/report-check.js +134 -338
- package/src/commands/state/retype.js +12 -28
- package/src/commands/validation/index.js +0 -3
- package/src/commands/validation/mutate.js +79 -19
- package/src/commands/validation/plan-check.js +81 -0
- package/src/commands/validation/validate-feature.js +52 -216
- package/src/commands/validation/verify.js +45 -160
- package/src/core/paths/output-schema.js +9 -155
- package/src/core/state/feature-identity.js +1 -25
- package/src/core/state/gate-order.js +2 -30
- package/src/core/state/lifecycle-audit.js +8 -100
- package/src/core/state/state-manager.js +32 -395
- package/src/core/templates/template-registry.js +2 -97
- package/src/core/templates/template-renderer.js +13 -93
- package/src/core/templates/template-seed.js +21 -0
- package/src/core/work-types.js +10 -45
- package/src/dashboard/app/App.jsx +1 -8
- package/src/dashboard/app/components/ActivityFeed.jsx +2 -8
- package/src/dashboard/app/components/CommandPalette.jsx +4 -11
- package/src/dashboard/app/components/DetailPanel.jsx +1 -5
- package/src/dashboard/app/components/EntityLink.jsx +0 -12
- package/src/dashboard/app/components/ErrorBoundary.jsx +1 -8
- package/src/dashboard/app/components/KnowledgeMarkdown.jsx +0 -11
- package/src/dashboard/app/components/Markdown.jsx +0 -9
- package/src/dashboard/app/components/PhaseRail.jsx +0 -11
- package/src/dashboard/app/components/SearchFilter.jsx +0 -4
- package/src/dashboard/app/components/Sidebar.jsx +0 -5
- package/src/dashboard/app/hooks/useMorphQuery.js +1 -11
- package/src/dashboard/app/hooks/usePref.js +1 -5
- package/src/dashboard/app/lib/format.js +0 -2
- package/src/dashboard/app/lib/phases.js +3 -9
- package/src/dashboard/app/lib/project.js +0 -12
- package/src/dashboard/app/pages/Activity.jsx +1 -1
- package/src/dashboard/app/pages/AgentMap.jsx +1 -8
- package/src/dashboard/app/pages/FeatureDetail.jsx +139 -133
- package/src/dashboard/app/pages/Features.jsx +9 -16
- package/src/dashboard/app/pages/Guide.jsx +2 -3
- package/src/dashboard/app/pages/HookInspector.jsx +2 -23
- package/src/dashboard/app/pages/PhasePipeline.jsx +3 -7
- package/src/dashboard/app/pages/Portfolio.jsx +0 -1
- package/src/dashboard/app/pages/RulesMap.jsx +1 -16
- package/src/dashboard/app/pages/SkillsLibrary.jsx +1 -16
- package/src/dashboard/app/pages/Standards.jsx +2 -17
- package/src/dashboard/scanner/activity-scanner.js +14 -83
- package/src/dashboard/scanner/agents-scanner.js +1 -23
- package/src/dashboard/scanner/docs-scanner.js +1 -7
- package/src/dashboard/scanner/feature-status.js +2 -5
- package/src/dashboard/scanner/features-scanner.js +92 -113
- package/src/dashboard/scanner/hooks-scanner.js +5 -28
- package/src/dashboard/scanner/index.js +0 -4
- package/src/dashboard/scanner/portfolio.js +5 -29
- package/src/dashboard/scanner/project-api.js +8 -54
- package/src/dashboard/scanner/rules-scanner.js +1 -2
- package/src/dashboard/scanner/skills-scanner.js +2 -12
- package/src/dashboard/scanner/standards-scanner.js +4 -7
- package/src/dashboard/vite-plugin-morph-data.js +5 -19
- package/src/lib/ai/ai-checks.js +7 -52
- package/src/lib/ai/ai-project-scan.js +14 -108
- package/src/lib/ai/code-audit.js +7 -63
- package/src/lib/ai/pin-drift.js +8 -51
- package/src/lib/ai/registry-audit.js +11 -90
- package/src/lib/ci/ci-verify.js +14 -13
- package/src/lib/ci/github-ci.js +9 -37
- package/src/lib/ci/workflow-scaffold.js +2 -10
- package/src/lib/detectors/claude-config-detector.js +13 -74
- package/src/lib/detectors/index.js +8 -59
- package/src/lib/e2e/compose-env.js +5 -66
- package/src/lib/e2e/stack-teardown.js +6 -40
- package/src/lib/generators/architecture-scanner.js +18 -156
- package/src/lib/generators/handoff-generator.js +37 -112
- package/src/lib/generators/scaffold-generator.js +5 -101
- package/src/lib/generators/settings-generator.js +11 -122
- package/src/lib/git/branch-classifier.js +11 -84
- package/src/lib/git/default-branch.js +14 -169
- package/src/lib/git/evidence-staleness.js +2 -1
- package/src/lib/git/gh.js +2 -17
- package/src/lib/git/pending-prs.js +6 -37
- package/src/lib/git/pr-files.js +3 -28
- package/src/lib/git/pr-index.js +3 -34
- package/src/lib/git/tree-fingerprint.js +17 -17
- package/src/lib/git/tree-state.js +6 -33
- package/src/lib/graph/god-nodes.js +2 -22
- package/src/lib/graph/graph-health.js +3 -24
- package/src/lib/graph/graph-loader.js +5 -56
- package/src/lib/graph/graphify-runtime.js +7 -37
- package/src/lib/graph/graphifyignore.js +8 -74
- package/src/lib/graph/sanitize.js +16 -136
- package/src/lib/heavy/cleanup.js +2 -6
- package/src/lib/installers/effects.js +8 -93
- package/src/lib/installers/mcp-installer.js +9 -169
- package/src/lib/installers/memory-installer.js +8 -33
- package/src/lib/installers/templates-reconcile.js +5 -60
- package/src/lib/mutation/commands.js +3 -2
- package/src/lib/mutation/dotnet-introspection.js +281 -0
- package/src/lib/mutation/plan.js +14 -6
- package/src/lib/mutation/runner.js +1 -1
- package/src/lib/mutation/subset.js +47 -13
- package/src/lib/plan/artifact-vocabulary.js +40 -0
- package/src/lib/plan/checks.js +208 -0
- package/src/lib/plan/plan-parser.js +204 -0
- package/src/lib/plan/plan-template.js +18 -0
- package/src/lib/projects/registry.js +7 -44
- package/src/lib/review-reports.js +204 -0
- package/src/lib/risk-detector.js +131 -274
- package/src/lib/specs/adr-index.js +38 -85
- package/src/lib/specs/design-seed.js +18 -0
- package/src/lib/specs/frontmatter.js +31 -0
- package/src/lib/specs/gfm-table.js +11 -0
- package/src/lib/specs/revocations.js +31 -102
- package/src/lib/specs/screen-checklist.js +56 -132
- package/src/lib/stack/dotnet-build-ignores.js +43 -0
- package/src/lib/stack/stack-profile.js +1 -28
- package/src/lib/stack/stack-scan.js +260 -0
- package/src/lib/stack-filter.js +4 -27
- package/src/lib/standards/registry.js +37 -0
- package/src/lib/standards/sections.js +47 -0
- package/src/lib/standards/verified-against.js +19 -191
- package/src/lib/tasks/eval-node.js +23 -179
- package/src/lib/tasks/linux-runner.js +18 -142
- package/src/lib/tasks/test-nodes.js +43 -313
- package/src/lib/tasks/test-runner.js +107 -447
- package/src/lib/telemetry/metrics.js +13 -103
- package/src/lib/telemetry/telemetry-reader.js +13 -238
- package/src/lib/telemetry/verify-event.js +4 -34
- package/src/lib/validators/code-budget/code-budget-validator.js +4 -44
- package/src/lib/validators/code-budget/index.js +0 -7
- package/src/lib/validators/contracts/contract-compliance-validator.js +4 -26
- package/src/lib/validators/contracts/index.js +0 -6
- package/src/lib/validators/nextjs/index.js +0 -5
- package/src/lib/validators/nextjs/next-component-validator.js +4 -81
- package/src/lib/validators/packages/index.js +0 -6
- package/src/lib/validators/packages/package-validator.js +6 -56
- package/src/lib/validators/reuse/index.js +0 -8
- package/src/lib/validators/reuse/reuse-validator.js +24 -137
- package/src/lib/validators/reuse/symbol-scanner.js +6 -62
- package/src/lib/validators/traceability-check.js +54 -156
- package/src/lib/validators/validation-runner.js +20 -125
- package/src/lib/validators/verify-runner.js +152 -575
- package/src/lib/work-type-detector.js +5 -67
- package/src/lib/worktree/docker-audit.js +17 -137
- package/src/lib/worktree/env-allocator.js +10 -98
- package/src/lib/worktree/feature-truth.js +12 -115
- package/src/lib/worktree/finish-backup.js +32 -242
- package/src/lib/worktree/fleet-scanner.js +31 -126
- package/src/lib/worktree/ignored-triage.js +15 -91
- package/src/lib/worktree/vscode-settings.js +2 -3
- package/src/lib/worktree/worktree-audit.js +21 -213
- package/src/lib/worktree/worktree-include.js +3 -5
- package/src/scripts/global-install.js +3 -9
- package/src/scripts/install-plugin.js +1 -13
- package/src/scripts/setup-infra.js +17 -138
- package/src/utils/agents-installer.js +92 -272
- package/src/utils/banner.js +4 -25
- package/src/utils/claude-md-injector.js +1 -37
- package/src/utils/claude-settings-manager.js +1 -87
- package/src/utils/file-copier.js +32 -90
- package/src/utils/hooks-installer.js +26 -296
- package/src/utils/skills-installer.js +4 -102
- package/src/utils/version-checker.js +4 -83
- package/framework/commands/morph-proposal.md +0 -57
- package/framework/evals/trace-schema.md +0 -67
- package/framework/hooks/claude-code/core/tasks-json-guard.js +0 -83
- package/framework/hooks/claude-code/core/trace-autogen.js +0 -277
- package/framework/hooks/git/pre-commit/specs.sh +0 -50
- package/framework/hooks/shared/tasks-validator.js +0 -111
- package/framework/rules/morph-task-tracking.md +0 -54
- package/framework/rules/morph-workflow.md +0 -244
- package/framework/schemas/tasks.schema.json +0 -79
- package/framework/skills/core/morph-brainstorming/SKILL.md +0 -161
- package/framework/skills/core/morph-implement/SKILL.md +0 -194
- package/framework/skills/core/morph-proposal/SKILL.md +0 -152
- package/framework/skills/core/morph-stack-scan/SKILL.md +0 -145
- package/framework/templates/docs/proposal.md +0 -192
- package/framework/templates/docs/spec.md +0 -57
- package/framework/templates/docs/ui-components.md +0 -52
- package/framework/templates/docs/ui-design-system.md +0 -76
- package/framework/templates/docs/ui-flows.md +0 -167
- package/framework/templates/examples/spec-examples.md +0 -91
- package/framework/templates/feature/decisions.md +0 -124
- package/framework/templates/feature/recap.md +0 -145
- package/framework/templates/tasks.template.json +0 -17
- package/src/commands/state/usecase.js +0 -143
- package/src/commands/tasks/dag.js +0 -181
- package/src/commands/utils/index.js +0 -3
- package/src/lib/agents/output-collector.js +0 -76
- package/src/lib/detectors/design-system-detector.js +0 -188
- package/src/lib/evaluator-report.js +0 -104
- package/src/lib/generators/design-system-generator.js +0 -508
- package/src/lib/specs/acervo-format.js +0 -70
- package/src/lib/specs/acervo-lifecycle.js +0 -264
- package/src/lib/specs/done-criteria-check.js +0 -200
- package/src/lib/specs/usecase-check.js +0 -301
- package/src/lib/specs/usecase-promoter.js +0 -301
- package/src/lib/specs/usecase-registry.js +0 -237
- package/src/lib/tasks/dispatch-planner.js +0 -319
- package/src/lib/tasks/dotnet-test-introspection.js +0 -632
- package/src/lib/tasks/mermaid-renderer.js +0 -132
- package/src/lib/tasks/task-classifier.js +0 -30
- package/src/lib/tasks/task-parser.js +0 -209
- package/src/lib/tasks/tasks-json-parser.js +0 -296
- package/src/lib/tasks/test-scope.js +0 -597
- package/src/utils/agent-id-mapper.js +0 -11
|
@@ -1,106 +1,29 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
*
|
|
4
|
-
* THE parser for a standard's verification header and for its age against the
|
|
5
|
-
* AI package pin. Single source of truth, on purpose.
|
|
6
|
-
*
|
|
7
|
-
* A standard declares what it was checked against, and when:
|
|
8
|
-
*
|
|
9
|
-
* **Verified against:** Microsoft.Agents.AI 1.20.0 (ai-pin 2026-09-08);
|
|
10
|
-
* provado por `templates/dotnet/ai-kit/.../AgentFactory.cs`. Last-verified: 2026-09-08.
|
|
11
|
-
*
|
|
12
|
-
* Three places need to read that same fact — `scripts/check-freshness.js`
|
|
13
|
-
* (the freshness gate), `test/framework/standard-header.test.js` (the header
|
|
14
|
-
* contract) and the "ai-agents vs ai-pin" report. Three copies of the regex
|
|
15
|
-
* would drift apart, which is exactly the failure mode CLAUDE.md's Dependency
|
|
16
|
-
* Matrix exists to catalogue. So the regex lives here, once.
|
|
17
|
-
*
|
|
18
|
-
* The age is measured against `framework/ai-pin.json`'s `pinnedAt`, never
|
|
19
|
-
* against "today": `pinnedAt` is the date a human checked the packages, so the
|
|
20
|
-
* obligation is tied to a human act. Measuring against today would make a
|
|
21
|
-
* parked repository change colour on its own.
|
|
22
|
-
*
|
|
23
|
-
* NOTE on the 90-day window: this module EXPOSES the age and the threshold; it
|
|
24
|
-
* enforces nothing — enforcement lives in its consumers. Issue #93 re-verified
|
|
25
|
-
* the `ai-agents` standards and turned the window into a HARD FAIL in
|
|
26
|
-
* scripts/check-freshness.js, so the number below is no longer only a warning.
|
|
27
|
-
* The name kept the `WARN` in it because renaming an export is a change to the
|
|
28
|
-
* module's surface, not to its documentation; scripts/check-freshness.js aliases
|
|
29
|
-
* it to `AI_PIN_STALE_DAYS` rather than re-declaring `90` — see that file's
|
|
30
|
-
* docblock and CONTRIBUTING.md's "two rulers" table.
|
|
31
|
-
*/
|
|
1
|
+
// Age is measured against `pinnedAt` of framework/ai-pin.json, never "today":
|
|
2
|
+
// a parked repository must not change colour on its own.
|
|
32
3
|
|
|
33
4
|
import { readFileSync, existsSync } from 'fs';
|
|
34
5
|
import { join, dirname } from 'path';
|
|
35
6
|
import { fileURLToPath } from 'url';
|
|
36
7
|
|
|
37
|
-
/** Repository root, resolved from this file: src/lib/standards → ../../.. */
|
|
38
8
|
const DEFAULT_ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..', '..');
|
|
39
9
|
|
|
40
|
-
/** Path of the pin relative to a root. */
|
|
41
10
|
const AI_PIN_RELATIVE = join('framework', 'ai-pin.json');
|
|
42
11
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
*
|
|
46
|
-
* Group 1 is the subject (what the standard was checked against), group 2 the
|
|
47
|
-
* ISO date. `[^\n]*?` keeps the match inside a single line: a `Last-verified:`
|
|
48
|
-
* two paragraphs below belongs to another claim, and stitching the two would
|
|
49
|
-
* date a header that never carried a date.
|
|
50
|
-
*
|
|
51
|
-
* @type {RegExp}
|
|
52
|
-
*/
|
|
12
|
+
// Group 1 = subject, group 2 = ISO date. `[^\n]*?` keeps it on one line: a
|
|
13
|
+
// `Last-verified:` further down belongs to another claim.
|
|
53
14
|
export const LAST_VERIFIED_RE = /\*\*Verified against:\*\*([^\n]*?)Last-verified:\s*(\d{4}-\d{2}-\d{2})/;
|
|
54
15
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
* against the installed package.
|
|
58
|
-
*
|
|
59
|
-
* Deliberately broader than the regex it replaces (`/^\s*\/\/\s*VERIFY:/`),
|
|
60
|
-
* which demanded line-start AND a colon. Every real marker in
|
|
61
|
-
* `framework/standards/ai-agents/` is an end-of-line comment without a colon —
|
|
62
|
-
*
|
|
63
|
-
* using OpenAI.Batch; // VERIFY namespace — confirme contra o pacote OpenAI 2.*
|
|
64
|
-
*
|
|
65
|
-
* — so the old counter reported 0 while two standards openly declared
|
|
66
|
-
* unconfirmed C# API. `\b` keeps `// VERIFYING` out; the fence requirement is
|
|
67
|
-
* NOT in the regex (a regex cannot see a fence) — it lives in
|
|
68
|
-
* {@link countVerifyMarkers}, so prose quoting `// VERIFY:` never counts.
|
|
69
|
-
*
|
|
70
|
-
* @type {RegExp}
|
|
71
|
-
*/
|
|
16
|
+
// Real markers are end-of-line and often lack a colon (`x; // VERIFY namespace`),
|
|
17
|
+
// so no line-start/colon anchor. The fence rule lives in countVerifyMarkers.
|
|
72
18
|
export const VERIFY_MARKER_RE = /(^|\s)\/\/\s*VERIFY\b/;
|
|
73
19
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
* due for re-verification.
|
|
77
|
-
*
|
|
78
|
-
* READ UNDER TWO NAMES, ONE VALUE. `scripts/check-freshness.js` imports it as
|
|
79
|
-
* `AI_PIN_STALE_DAYS` and **hard-fails** the run above it (issue #93), for
|
|
80
|
-
* standards under `framework/standards/ai-agents/` measured against `pinnedAt`.
|
|
81
|
-
* The alias exists so the number is declared once: a second literal `90` is the
|
|
82
|
-
* same drift that once blinded the `// VERIFY` counter.
|
|
83
|
-
*
|
|
84
|
-
* The `WARN` in the name is now historical — it shipped in issue #92, when the
|
|
85
|
-
* window was a measurement. Kept because renaming an export changes the module's
|
|
86
|
-
* surface; this comment changes only what the reader is told.
|
|
87
|
-
*
|
|
88
|
-
* @type {number}
|
|
89
|
-
*/
|
|
20
|
+
// scripts/check-freshness.js hard-fails above this (aliased as AI_PIN_STALE_DAYS):
|
|
21
|
+
// declare the number only here.
|
|
90
22
|
export const AI_AGENTS_WARN_DAYS = 90;
|
|
91
23
|
|
|
92
|
-
/** ISO calendar date, the only date shape this module accepts. */
|
|
93
24
|
const ISO_DATE_RE = /^(\d{4})-(\d{2})-(\d{2})$/;
|
|
94
25
|
|
|
95
|
-
/**
|
|
96
|
-
* Parse a standard's verification header.
|
|
97
|
-
*
|
|
98
|
-
* @param {string} text - Full markdown text of the standard.
|
|
99
|
-
* @returns {{ verifiedAgainst: string, lastVerified: string }|null}
|
|
100
|
-
* `null` when the standard carries no parseable header — the caller decides
|
|
101
|
-
* whether that is a warning (the backfill backlog) or a failure
|
|
102
|
-
* (`ai-agents/**`, where a new standard must be born dated).
|
|
103
|
-
*/
|
|
26
|
+
/** `null` when no parseable header: the caller decides if that is a warning or a failure. */
|
|
104
27
|
export function parseVerifiedAgainst(text) {
|
|
105
28
|
if (typeof text !== 'string' || text.length === 0) return null;
|
|
106
29
|
const m = LAST_VERIFIED_RE.exec(text);
|
|
@@ -108,80 +31,18 @@ export function parseVerifiedAgainst(text) {
|
|
|
108
31
|
return { verifiedAgainst: m[1].trim(), lastVerified: m[2] };
|
|
109
32
|
}
|
|
110
33
|
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
*
|
|
114
|
-
* THREE TYPESETTINGS EXIST IN THE ACERVO, and all three are real claims.
|
|
115
|
-
* Counted by re-deriving every one of the 78 standards on disk and summing the
|
|
116
|
-
* resulting paths — 27 paths across 16 entries:
|
|
117
|
-
*
|
|
118
|
-
* A ; provado por `templates/…/AgentFactory.cs` e `.../ConversationStore.cs`
|
|
119
|
-
* B ; `provado por templates/…/AgentFactory.cs` para a **ordem do pipeline**
|
|
120
|
-
* C . O timeout é **provado por** `.../AgentSpecOptions.cs` e `.../AgentFactory.cs`
|
|
121
|
-
*
|
|
122
|
-
* B opens the code span BEFORE the clause; C wraps the clause in bold and can
|
|
123
|
-
* appear MID-SENTENCE, after an earlier clause already closed. Each one was
|
|
124
|
-
* found by measuring, and each was missing from the rule that preceded it:
|
|
125
|
-
*
|
|
126
|
-
* - ignoring B dropped 4 of 16 claims (`middleware-patterns`,
|
|
127
|
-
* `observability-patterns`, `rag-custom-pgvector`, `structured-output`);
|
|
128
|
-
* - ignoring C dropped 2 paths from `llm-runtime-defaults` — 25 instead of 27.
|
|
129
|
-
*
|
|
130
|
-
* READ THE SHAPE OF THAT SECOND MISS, because it is why counting entries is not
|
|
131
|
-
* enough: `llm-runtime-defaults` had two OTHER paths from a clause of form A, so
|
|
132
|
-
* it stayed in the "16 entries with proof" bucket either way. No counter moved,
|
|
133
|
-
* `--check` stayed green, exit 0. The only ruler that saw it is the SUM of
|
|
134
|
-
* paths. A canary that pins entry counts would have gone green over it — which
|
|
135
|
-
* is why `standards-registry.test.js` pins the paths themselves, written out by
|
|
136
|
-
* hand, and not a total produced by this function.
|
|
137
|
-
*
|
|
138
|
-
* The DENIALS must stay empty, and they are the majority (61 of the 77 headers
|
|
139
|
-
* derive to `[]`; 16 say some variant of "sem cláusula `provado por`" or
|
|
140
|
-
* "NÃO carimba `provado por`"). Three properties keep them out without a
|
|
141
|
-
* blacklist of prose:
|
|
142
|
-
*
|
|
143
|
-
* 1. a path must contain a `/` — so "`provado por`. Os nomes" and
|
|
144
|
-
* "`provado por`: o ai-kit" match nothing;
|
|
145
|
-
* 2. a path must be CLOSED BY A BACKTICK — so prose that merely follows the
|
|
146
|
-
* clause ("para a **ordem do pipeline**") cannot be mistaken for a path;
|
|
147
|
-
* 3. the clause must not be the tail of a LONGER word — `aprovado por` and
|
|
148
|
-
* `comprovado por` are different claims. The corpus already contains
|
|
149
|
-
* "aprovado por lucas@…" inside a code sample, so this is a live shape,
|
|
150
|
-
* not a hypothetical.
|
|
151
|
-
*
|
|
152
|
-
* @type {RegExp}
|
|
153
|
-
*/
|
|
34
|
+
// Three real typesettings: backtick before the clause, after it, or the clause in
|
|
35
|
+
// bold mid-sentence. The non-letter prefix keeps `aprovado por` out.
|
|
154
36
|
export const PROVED_BY_TRIGGER_RE = /(?:^|[^a-zA-Z])provado por/gi;
|
|
155
37
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
* whitespace, backtick, `,`, `;`, the word `e`, and the markdown emphasis
|
|
159
|
-
* characters `*` and `_`. The emphasis characters are in the class because of
|
|
160
|
-
* typesetting C — without them, `**provado por**` armed the trigger and then
|
|
161
|
-
* matched no path at all, losing both of its files in silence.
|
|
162
|
-
*
|
|
163
|
-
* Anchored: it only ever reads from the cursor, so prose ends the scan instead
|
|
164
|
-
* of being searched through.
|
|
165
|
-
*
|
|
166
|
-
* @type {RegExp}
|
|
167
|
-
*/
|
|
38
|
+
// A path needs a `/` and a closing backtick, so denials ("sem cláusula `provado por`")
|
|
39
|
+
// derive nothing. `*`/`_` are separators for the bold form. Anchored: prose ends the scan.
|
|
168
40
|
export const PROVED_BY_PATH_RE =
|
|
169
41
|
/^[\s`,;*_]*(?:\be\b[\s`,;*_]*)?([A-Za-z0-9_.][A-Za-z0-9_.\-]*(?:\/[A-Za-z0-9_.\-]*)+)`/;
|
|
170
42
|
|
|
171
43
|
/**
|
|
172
|
-
*
|
|
173
|
-
*
|
|
174
|
-
* Reads the SAME header {@link parseVerifiedAgainst} reads — deliberately, and
|
|
175
|
-
* from this module, because a second regex over the same sentence in
|
|
176
|
-
* `scripts/generate-standards-registry.js` is precisely the drift the module
|
|
177
|
-
* docblock opens with.
|
|
178
|
-
*
|
|
179
|
-
* @param {string} verifiedAgainst - The subject captured by
|
|
180
|
-
* {@link parseVerifiedAgainst} (group 1), not the whole markdown file.
|
|
181
|
-
* @returns {string[]} Paths in declaration order, de-duplicated. `[]` means
|
|
182
|
-
* "the header was read and declares no proof" — a documental verification —
|
|
183
|
-
* never "not asked". The caller distinguishes the two by whether the header
|
|
184
|
-
* parsed at all.
|
|
44
|
+
* Takes the subject from parseVerifiedAgainst, not the whole file. `[]` means the
|
|
45
|
+
* header declares no proof, never "not asked".
|
|
185
46
|
*/
|
|
186
47
|
export function deriveProvedBy(verifiedAgainst) {
|
|
187
48
|
if (typeof verifiedAgainst !== 'string' || verifiedAgainst.length === 0) return [];
|
|
@@ -199,16 +60,7 @@ export function deriveProvedBy(verifiedAgainst) {
|
|
|
199
60
|
return paths;
|
|
200
61
|
}
|
|
201
62
|
|
|
202
|
-
/**
|
|
203
|
-
* Count unresolved `// VERIFY` markers that sit INSIDE a fenced code block.
|
|
204
|
-
*
|
|
205
|
-
* The fence rule is what separates a real marker from prose that merely quotes
|
|
206
|
-
* one (`marcados \`// VERIFY:\` no código`). It lives here, next to the regex,
|
|
207
|
-
* so every consumer counts the same thing.
|
|
208
|
-
*
|
|
209
|
-
* @param {string} text - Full markdown text of the standard.
|
|
210
|
-
* @returns {number}
|
|
211
|
-
*/
|
|
63
|
+
/** Only markers inside a fenced code block count: prose quoting one is not a marker. */
|
|
212
64
|
export function countVerifyMarkers(text) {
|
|
213
65
|
if (typeof text !== 'string' || text.length === 0) return 0;
|
|
214
66
|
let inFence = false;
|
|
@@ -220,20 +72,7 @@ export function countVerifyMarkers(text) {
|
|
|
220
72
|
return count;
|
|
221
73
|
}
|
|
222
74
|
|
|
223
|
-
/**
|
|
224
|
-
* Read and parse `framework/ai-pin.json`.
|
|
225
|
-
*
|
|
226
|
-
* Throws — never falls back to a default — when the pin is missing or invalid:
|
|
227
|
-
* without a pin there is no ruler, and pretending there is one is the original
|
|
228
|
-
* defect this feature removes. The message names the file so the failure is
|
|
229
|
-
* diagnosable from a CI log alone.
|
|
230
|
-
*
|
|
231
|
-
* @param {string} [root=DEFAULT_ROOT] - Repository root that contains `framework/`.
|
|
232
|
-
* @returns {{ pinnedAt: string, source: string, targetFramework: string, sdk: string,
|
|
233
|
-
* packages: Record<string, {version: string, published: string, role: string}>,
|
|
234
|
-
* toolchain: Record<string, {version: string}> }}
|
|
235
|
-
* @throws {Error} When the file is absent or is not valid JSON.
|
|
236
|
-
*/
|
|
75
|
+
/** Throws, never defaults, when the pin is missing or invalid: without a pin there is no ruler. */
|
|
237
76
|
export function readAiPin(root = DEFAULT_ROOT) {
|
|
238
77
|
const pinPath = join(root, AI_PIN_RELATIVE);
|
|
239
78
|
if (!existsSync(pinPath)) {
|
|
@@ -246,7 +85,6 @@ export function readAiPin(root = DEFAULT_ROOT) {
|
|
|
246
85
|
}
|
|
247
86
|
}
|
|
248
87
|
|
|
249
|
-
/** Parse an ISO calendar date into a UTC epoch, timezone-independent. */
|
|
250
88
|
function isoToUtc(value, label) {
|
|
251
89
|
const m = typeof value === 'string' ? ISO_DATE_RE.exec(value) : null;
|
|
252
90
|
if (!m) {
|
|
@@ -256,18 +94,8 @@ function isoToUtc(value, label) {
|
|
|
256
94
|
}
|
|
257
95
|
|
|
258
96
|
/**
|
|
259
|
-
*
|
|
260
|
-
*
|
|
261
|
-
* Both dates are read as UTC calendar days, so the result does not depend on
|
|
262
|
-
* the runner's timezone. The value is NEGATIVE when the standard is newer than
|
|
263
|
-
* the pin — the caller must treat that as a PIN error (a `pinnedAt` in the
|
|
264
|
-
* future), never as a fresh standard: otherwise every standard could be
|
|
265
|
-
* "renewed" by editing one date.
|
|
266
|
-
*
|
|
267
|
-
* @param {string} lastVerified - ISO date from the standard's header.
|
|
268
|
-
* @param {string} pinnedAt - ISO date from `framework/ai-pin.json`.
|
|
269
|
-
* @returns {number} Whole days between the two, possibly negative.
|
|
270
|
-
* @throws {TypeError} When either argument is not an ISO date.
|
|
97
|
+
* Negative when the standard is newer than the pin: the caller must treat that as a
|
|
98
|
+
* pin error, never as fresh, or any standard could be "renewed" by editing one date.
|
|
271
99
|
*/
|
|
272
100
|
export function ageInDaysAgainstPin(lastVerified, pinnedAt) {
|
|
273
101
|
const verified = isoToUtc(lastVerified, 'lastVerified');
|
|
@@ -1,36 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Eval Node — the `runner: "evals"` kind of `project.tests[]`.
|
|
3
|
-
*
|
|
4
|
-
* WHY THIS IS A RUNNER KIND AND NOT A SIXTH VERIFY NODE
|
|
5
|
-
*
|
|
6
|
-
* An eval IS `dotnet test` with a category filter: one command, one cwd, one
|
|
7
|
-
* clock, one exit code. Modelling it as something other than a test suite would
|
|
8
|
-
* mean a second copy of the executor, a second decision for `--filter`, a second
|
|
9
|
-
* decision for `--linux`, a second place for `expectedCount` — and a new line in
|
|
10
|
-
* a residency-capped charter that has zero slack. As a runner kind it inherits
|
|
11
|
-
* all of that for free, and the ONE thing the rollup cannot express (eval red vs
|
|
12
|
-
* unit-test red) is recovered by the verify stamp, which records `nodes.evals`
|
|
13
|
-
* separately.
|
|
14
|
-
*
|
|
15
|
-
* WHAT THIS MODULE OWNS
|
|
16
|
-
*
|
|
17
|
-
* - the STATES of a cache tree, judged BEFORE `dotnet test` is invoked. Paying
|
|
18
|
-
* for a whole suite to discover at the end that the cache did not serve is
|
|
19
|
-
* the worst possible ordering;
|
|
20
|
-
* - the `MORPH_EVAL_*` variables the CLI DICTATES, so the same xunit suite is
|
|
21
|
-
* runnable by `verify` without a second configuration path;
|
|
22
|
-
* - reading the summary the fixture writes — where **null is UNKNOWN, never
|
|
23
|
-
* zero**.
|
|
24
|
-
*
|
|
25
|
-
* WHAT IT DELIBERATELY DOES NOT OWN
|
|
26
|
-
*
|
|
27
|
-
* The pin. `readAiPin()` (src/lib/standards/verified-against.js) is the single
|
|
28
|
-
* reader of `framework/ai-pin.json`; a second parser here is how two truths
|
|
29
|
-
* about "which version is pinned" begin.
|
|
30
|
-
*
|
|
31
|
-
* @module eval-node
|
|
32
|
-
*/
|
|
33
|
-
|
|
34
1
|
import { existsSync, readFileSync, readdirSync, statSync } from 'fs';
|
|
35
2
|
import { join, resolve, sep } from 'path';
|
|
36
3
|
import { tmpdir } from 'os';
|
|
@@ -38,28 +5,19 @@ import { tmpdir } from 'os';
|
|
|
38
5
|
import { isInside } from './test-runner.js';
|
|
39
6
|
import { resolveAiPin } from '../ai/pin-drift.js';
|
|
40
7
|
|
|
41
|
-
/** The runner kind. One spelling, imported by every reader. */
|
|
42
8
|
export const EVAL_RUNNER = 'evals';
|
|
43
9
|
|
|
44
|
-
/**
|
|
10
|
+
/** The SAME name the .NET fixture writes. */
|
|
45
11
|
export const EVAL_STAMP_FILE = '.morph-eval-cache.json';
|
|
46
12
|
|
|
47
|
-
/**
|
|
13
|
+
/** Relative to the cache root's parent. */
|
|
48
14
|
export const DEFAULT_DATASET_DIR = 'eval-datasets';
|
|
49
15
|
|
|
50
|
-
/** Default credential file. Gitignored, like `e2e.auth.envFile`; never in the committed config. */
|
|
51
16
|
export const DEFAULT_ENV_FILE = '.env.evals';
|
|
52
17
|
|
|
53
18
|
/**
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
* Only the `Microsoft.Extensions.AI` family: the cache key is computed by that
|
|
57
|
-
* library, and its own docs say the hash is not guaranteed stable across
|
|
58
|
-
* releases. Bumping an unrelated provider package (OpenAI, Google.GenAI) cannot
|
|
59
|
-
* invalidate a cache tree, and a stamp that reacted to it would fail for
|
|
60
|
-
* nothing — a guardrail that cries wolf gets switched off.
|
|
61
|
-
*
|
|
62
|
-
* @param {Record<string, {version: string}>} packages
|
|
19
|
+
* Only the `Microsoft.Extensions.AI` family decides the cache format: that library
|
|
20
|
+
* computes the cache key. Reacting to unrelated provider bumps would fail for nothing.
|
|
63
21
|
* @returns {Record<string, string>} name → exact version
|
|
64
22
|
*/
|
|
65
23
|
export function cacheRelevantPackages(packages) {
|
|
@@ -72,13 +30,6 @@ export function cacheRelevantPackages(packages) {
|
|
|
72
30
|
return out;
|
|
73
31
|
}
|
|
74
32
|
|
|
75
|
-
/**
|
|
76
|
-
* Resolve every path an eval node names, against the project root.
|
|
77
|
-
*
|
|
78
|
-
* @param {import('./test-nodes.js').TestNode & {evals?: object}} node
|
|
79
|
-
* @param {string} projectPath
|
|
80
|
-
* @returns {{cacheRoot: string, datasetRoot: string, resultsRoot: string, envFile: string}}
|
|
81
|
-
*/
|
|
82
33
|
export function resolveEvalPaths(node, projectPath) {
|
|
83
34
|
const root = resolve(projectPath);
|
|
84
35
|
const evals = node?.evals ?? {};
|
|
@@ -88,25 +39,14 @@ export function resolveEvalPaths(node, projectPath) {
|
|
|
88
39
|
return {
|
|
89
40
|
cacheRoot,
|
|
90
41
|
datasetRoot: evals.datasetPath ? resolve(root, evals.datasetPath) : join(beside, DEFAULT_DATASET_DIR),
|
|
91
|
-
//
|
|
92
|
-
//
|
|
93
|
-
// is a guaranteed diff on every run with no signal in it.
|
|
42
|
+
// Outside the committed cache tree: results carry latency and token usage,
|
|
43
|
+
// so committing them is a diff on every run with no signal in it.
|
|
94
44
|
resultsRoot: join(beside, 'eval-results'),
|
|
95
45
|
envFile: resolve(root, evals.envFile ?? DEFAULT_ENV_FILE),
|
|
96
46
|
};
|
|
97
47
|
}
|
|
98
48
|
|
|
99
|
-
/**
|
|
100
|
-
* First problem with a node's `evals` block, or null when it is well formed.
|
|
101
|
-
*
|
|
102
|
-
* Fail-CLOSED, like the rest of `validateTestNodes`: a typo here would make the
|
|
103
|
-
* eval silently disappear, which is the exact defect this feature exists to
|
|
104
|
-
* prevent.
|
|
105
|
-
*
|
|
106
|
-
* @param {*} raw - the raw `tests[]` entry
|
|
107
|
-
* @param {string|null} projectPath
|
|
108
|
-
* @returns {string|null}
|
|
109
|
-
*/
|
|
49
|
+
/** Fail-CLOSED: a typo must never make the eval silently disappear. */
|
|
110
50
|
export function evalBlockProblem(raw, projectPath) {
|
|
111
51
|
const evals = raw?.evals;
|
|
112
52
|
|
|
@@ -129,7 +69,6 @@ export function evalBlockProblem(raw, projectPath) {
|
|
|
129
69
|
for (const key of ['cachePath', 'datasetPath']) {
|
|
130
70
|
const value = evals[key];
|
|
131
71
|
if (typeof value !== 'string' || !value.trim()) continue;
|
|
132
|
-
// Same rule as `cwd`: a path that leaves the root addresses ANOTHER tree.
|
|
133
72
|
if (!isInside(resolve(root, value.trim()), root)) {
|
|
134
73
|
return `evals.${key} "${value}" resolves outside the project root`;
|
|
135
74
|
}
|
|
@@ -138,7 +77,6 @@ export function evalBlockProblem(raw, projectPath) {
|
|
|
138
77
|
return null;
|
|
139
78
|
}
|
|
140
79
|
|
|
141
|
-
/** Does this cache tree hold at least one entry, beyond the stamp? */
|
|
142
80
|
function cacheHasEntries(cacheRoot) {
|
|
143
81
|
const walk = (dir) => {
|
|
144
82
|
let entries;
|
|
@@ -160,7 +98,7 @@ function cacheHasEntries(cacheRoot) {
|
|
|
160
98
|
return walk(cacheRoot);
|
|
161
99
|
}
|
|
162
100
|
|
|
163
|
-
/**
|
|
101
|
+
/** null for absent OR unreadable. */
|
|
164
102
|
export function readEvalStamp(cacheRoot) {
|
|
165
103
|
const path = join(cacheRoot, EVAL_STAMP_FILE);
|
|
166
104
|
if (!existsSync(path)) return null;
|
|
@@ -173,40 +111,18 @@ export function readEvalStamp(cacheRoot) {
|
|
|
173
111
|
}
|
|
174
112
|
|
|
175
113
|
/**
|
|
176
|
-
*
|
|
177
|
-
*
|
|
178
|
-
*
|
|
179
|
-
* The order is the order of the spec, and it is not arbitrary: the three cache
|
|
180
|
-
* states are judged before the command is invoked, because discovering at the
|
|
181
|
-
* END of a suite that the cache did not serve costs the whole clock and tells
|
|
182
|
-
* the reader nothing they could not have been told first.
|
|
183
|
-
*
|
|
184
|
-
* **There is no `skip` for a bad cache.** A skip here would recreate the hole
|
|
185
|
-
* this harness closed twice: a green gate over a verification that never
|
|
186
|
-
* happened. The only skip is scope — an eval suite does not belong in the
|
|
187
|
-
* per-task loop.
|
|
188
|
-
*
|
|
189
|
-
* @param {string} projectPath
|
|
190
|
-
* @param {object} node - a resolved test node with `runner === 'evals'`
|
|
191
|
-
* @param {{taskId?: string|null, refresh?: boolean}} [options]
|
|
192
|
-
* @returns {{status: 'skip'|'fail'|null, reason: string|null}} status null means RUN
|
|
114
|
+
* Judged BEFORE `dotnet test`: finding a bad cache at the end costs the whole clock.
|
|
115
|
+
* A bad cache is `fail`, never `skip` — a skip is a green gate over nothing.
|
|
116
|
+
* @returns {{status: 'fail'|null, reason: string|null}} status null means RUN
|
|
193
117
|
*/
|
|
194
118
|
export function evalPreflight(projectPath, node, options = {}) {
|
|
195
|
-
const {
|
|
119
|
+
const { refresh = false } = options;
|
|
196
120
|
|
|
197
121
|
if (node?.runner !== EVAL_RUNNER) return { status: null, reason: null };
|
|
198
122
|
|
|
199
|
-
// Scope first: an eval suite does not pay for itself in a per-task loop, and
|
|
200
|
-
// the result Gate 3 reads is the feature-scoped one. Same treatment `e2e: true`
|
|
201
|
-
// already gets.
|
|
202
|
-
if (taskId) {
|
|
203
|
-
return { status: 'skip', reason: 'eval node runs at feature scope only' };
|
|
204
|
-
}
|
|
205
|
-
|
|
206
123
|
const { cacheRoot, envFile } = resolveEvalPaths(node, projectPath);
|
|
207
124
|
|
|
208
|
-
//
|
|
209
|
-
// cache would make the FIRST cache impossible to create.
|
|
125
|
+
// Demanding a populated, stamped cache in refresh would make the FIRST cache impossible.
|
|
210
126
|
if (refresh) {
|
|
211
127
|
if (!existsSync(envFile)) {
|
|
212
128
|
return {
|
|
@@ -222,15 +138,12 @@ export function evalPreflight(projectPath, node, options = {}) {
|
|
|
222
138
|
}
|
|
223
139
|
|
|
224
140
|
if (!cacheHasEntries(cacheRoot)) {
|
|
225
|
-
// Distinct from `missing
|
|
226
|
-
// more likely a `.gitignore` mistake than a deliberate absence, and the two
|
|
227
|
-
// ask for different fixes.
|
|
141
|
+
// Distinct from `missing`: an empty committed directory is usually a `.gitignore` mistake.
|
|
228
142
|
return { status: 'fail', reason: `eval-cache-empty: ${cacheRoot} não contém nenhuma entrada de cache` };
|
|
229
143
|
}
|
|
230
144
|
|
|
231
145
|
const pin = resolveAiPin(projectPath);
|
|
232
146
|
if (!pin) {
|
|
233
|
-
// No pin in the project or in the CLI package: nothing to stamp against.
|
|
234
147
|
return { status: null, reason: null };
|
|
235
148
|
}
|
|
236
149
|
|
|
@@ -260,15 +173,7 @@ export function evalPreflight(projectPath, node, options = {}) {
|
|
|
260
173
|
}
|
|
261
174
|
|
|
262
175
|
/**
|
|
263
|
-
* The
|
|
264
|
-
*
|
|
265
|
-
* The summary path is chosen HERE, by the CLI, and not by the fixture: two eval
|
|
266
|
-
* nodes in the same `verify` must not collide, and the process that reads the
|
|
267
|
-
* file is the one that should decide where it lives.
|
|
268
|
-
*
|
|
269
|
-
* @param {object} node
|
|
270
|
-
* @param {string} projectPath
|
|
271
|
-
* @param {{refresh?: boolean}} [options]
|
|
176
|
+
* The CLI picks the summary path, not the fixture: two eval nodes in one verify must not collide.
|
|
272
177
|
* @returns {{env: Record<string,string>, summaryPath: string}}
|
|
273
178
|
*/
|
|
274
179
|
export function evalNodeEnv(node, projectPath, options = {}) {
|
|
@@ -287,31 +192,19 @@ export function evalNodeEnv(node, projectPath, options = {}) {
|
|
|
287
192
|
|
|
288
193
|
const pin = resolveAiPin(projectPath);
|
|
289
194
|
if (pin) {
|
|
290
|
-
// Compact, and only the packages that decide the cache format. The fixture
|
|
291
|
-
// stamps from THIS, so it never has to read (or parse) the pin itself.
|
|
292
195
|
env.MORPH_EVAL_PIN = JSON.stringify({
|
|
293
196
|
pinnedAt: pin.pinnedAt,
|
|
294
197
|
packages: cacheRelevantPackages(pin.packages),
|
|
295
198
|
});
|
|
296
199
|
}
|
|
297
200
|
|
|
298
|
-
//
|
|
299
|
-
// is NOT an error offline: that is the whole point of a warm cache.
|
|
201
|
+
// Absent credential file is NOT an error offline: that is the point of a warm cache.
|
|
300
202
|
Object.assign(env, readEnvFile(envFile));
|
|
301
203
|
|
|
302
204
|
return { env, summaryPath };
|
|
303
205
|
}
|
|
304
206
|
|
|
305
|
-
/**
|
|
306
|
-
* Parse a `KEY=value` file. Blank lines and `#` comments ignored; surrounding
|
|
307
|
-
* quotes stripped; the first `=` splits.
|
|
308
|
-
*
|
|
309
|
-
* Fail-open to `{}`: an unreadable credential file must not take down a run that
|
|
310
|
-
* did not need a credential in the first place.
|
|
311
|
-
*
|
|
312
|
-
* @param {string} path
|
|
313
|
-
* @returns {Record<string,string>}
|
|
314
|
-
*/
|
|
207
|
+
/** Fail-open to `{}`: an unreadable credential file must not take down a run that needed none. */
|
|
315
208
|
export function readEnvFile(path) {
|
|
316
209
|
if (!path || !existsSync(path)) return {};
|
|
317
210
|
let text;
|
|
@@ -337,14 +230,7 @@ export function readEnvFile(path) {
|
|
|
337
230
|
return out;
|
|
338
231
|
}
|
|
339
232
|
|
|
340
|
-
/**
|
|
341
|
-
* Read the summary the fixture wrote.
|
|
342
|
-
*
|
|
343
|
-
* @param {string} path
|
|
344
|
-
* @returns {object|null} **null is UNKNOWN, never zero.** A summary nobody could
|
|
345
|
-
* read is not a suite that evaluated zero scenarios: collapsing the two would
|
|
346
|
-
* manufacture a count drop against `expectedCount` out of a missing file.
|
|
347
|
-
*/
|
|
233
|
+
/** @returns {object|null} null is UNKNOWN, never zero: a missing file must not fabricate a count drop. */
|
|
348
234
|
export function readEvalSummary(path) {
|
|
349
235
|
if (!path || !existsSync(path)) return null;
|
|
350
236
|
try {
|
|
@@ -357,37 +243,15 @@ export function readEvalSummary(path) {
|
|
|
357
243
|
}
|
|
358
244
|
|
|
359
245
|
/**
|
|
360
|
-
*
|
|
361
|
-
*
|
|
362
|
-
* The ONLY source of an eval node's count. The `dotnet` extractor reads the same
|
|
363
|
-
* run but counts xunit cases — another unit (#110) — so an unreadable summary
|
|
364
|
-
* yields null: UNKNOWN, never 0, and never the extractor's number.
|
|
365
|
-
*
|
|
366
|
-
* @param {object|null} summary
|
|
367
|
-
* @returns {number|null}
|
|
246
|
+
* The ONLY source of an eval node's count. The `dotnet` extractor counts xunit cases,
|
|
247
|
+
* another unit, so an unreadable summary is null — never 0, never the extractor's number.
|
|
368
248
|
*/
|
|
369
249
|
export function evalScenarioCount(summary) {
|
|
370
250
|
const n = summary?.scenarios;
|
|
371
251
|
return Number.isInteger(n) ? n : null;
|
|
372
252
|
}
|
|
373
253
|
|
|
374
|
-
/**
|
|
375
|
-
* The NAMED failures of an eval run, one readable line each.
|
|
376
|
-
*
|
|
377
|
-
* The summary publishes `failures[]` (scenario, assertion, kind, reason) and
|
|
378
|
-
* without this reader the field would be written and never surfaced — a
|
|
379
|
-
* published field with no consumer, the same shape #91 closed on `superseded`.
|
|
380
|
-
* Worse here than usual: the whole point of `failures[]` is that a red eval
|
|
381
|
-
* names what broke, and a name nobody prints is not a name.
|
|
382
|
-
*
|
|
383
|
-
* Defensive by construction, because this parses a file produced by ANOTHER
|
|
384
|
-
* process: a summary that is absent, unreadable, or carries a malformed entry
|
|
385
|
-
* yields fewer lines, never a throw. A crash here would turn "the eval failed"
|
|
386
|
-
* into "the verify crashed", which reports the wrong defect.
|
|
387
|
-
*
|
|
388
|
-
* @param {object|null} summary
|
|
389
|
-
* @returns {string[]} readable lines; empty when there is nothing to name
|
|
390
|
-
*/
|
|
254
|
+
/** Never throws on a malformed summary: a crash would report "verify crashed" instead of "eval failed". */
|
|
391
255
|
export function evalFailureLines(summary) {
|
|
392
256
|
const failures = summary?.failures;
|
|
393
257
|
if (!Array.isArray(failures)) return [];
|
|
@@ -403,16 +267,7 @@ export function evalFailureLines(summary) {
|
|
|
403
267
|
});
|
|
404
268
|
}
|
|
405
269
|
|
|
406
|
-
/**
|
|
407
|
-
* Why an otherwise-green eval run must still be red, or null when it is fine.
|
|
408
|
-
*
|
|
409
|
-
* One case, and it is the guardrail-inert shape this repository has already paid
|
|
410
|
-
* to learn: a suite that evaluated ZERO scenarios and exited 0. The exit code
|
|
411
|
-
* alone cannot see it, because nothing failed — nothing ran.
|
|
412
|
-
*
|
|
413
|
-
* @param {object|null} summary
|
|
414
|
-
* @returns {string|null}
|
|
415
|
-
*/
|
|
270
|
+
/** Zero scenarios with exit 0 is red: the exit code cannot see a suite where nothing ran. */
|
|
416
271
|
export function evalEmptyRunProblem(summary) {
|
|
417
272
|
if (summary && summary.scenarios === 0) {
|
|
418
273
|
return 'eval-empty-run: a suíte de eval avaliou 0 cenários e saiu verde';
|
|
@@ -420,16 +275,7 @@ export function evalEmptyRunProblem(summary) {
|
|
|
420
275
|
return null;
|
|
421
276
|
}
|
|
422
277
|
|
|
423
|
-
/**
|
|
424
|
-
* Warning for a run whose cache went cold — the tree changed and must be committed.
|
|
425
|
-
*
|
|
426
|
-
* A warning and not a failure, deliberately: spending money nobody asked for is
|
|
427
|
-
* bad, but lying about the result is worse. Without the warning, a freshly
|
|
428
|
-
* written cache stays on the disk of whoever ran it and CI misses tomorrow.
|
|
429
|
-
*
|
|
430
|
-
* @param {object|null} summary
|
|
431
|
-
* @returns {string|null}
|
|
432
|
-
*/
|
|
278
|
+
/** A warning, not a failure: without it the fresh cache stays on one disk and CI misses tomorrow. */
|
|
433
279
|
export function evalColdCacheWarning(summary) {
|
|
434
280
|
if (summary?.cache === 'cold') {
|
|
435
281
|
const misses = Number.isInteger(summary.misses) ? summary.misses : '?';
|
|
@@ -438,14 +284,12 @@ export function evalColdCacheWarning(summary) {
|
|
|
438
284
|
return null;
|
|
439
285
|
}
|
|
440
286
|
|
|
441
|
-
/** Relative form of a path, for messages that should not shout an absolute path. */
|
|
442
287
|
export function relativeToProject(projectPath, target) {
|
|
443
288
|
const root = resolve(projectPath) + sep;
|
|
444
289
|
const full = resolve(target);
|
|
445
290
|
return full.startsWith(root) ? full.slice(root.length).replace(/\\/g, '/') : full;
|
|
446
291
|
}
|
|
447
292
|
|
|
448
|
-
/** Is `path` a directory? Fail-open to false. */
|
|
449
293
|
export function isDirectory(path) {
|
|
450
294
|
try {
|
|
451
295
|
return statSync(path).isDirectory();
|