@polymorphism-tech/morph-spec 8.34.2 → 8.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +20 -3
- package/README.md +2 -2
- package/bin/morph-spec.js +18 -4
- package/bin/validate.js +379 -379
- package/claude-plugin.json +1 -1
- package/docs/CHEATSHEET.md +5 -4
- package/docs/QUICKSTART.md +1 -1
- package/framework/MORPH.md +16 -17
- package/framework/agents.json +197 -44
- package/framework/ai-pin.json +22 -0
- package/framework/commands/morph-media.md +45 -0
- package/framework/docs/CLI.md +234 -9
- package/framework/evals/proof-rubrics/README.md +76 -0
- package/framework/evals/proof-rubrics/eval-de-modelo.md +64 -0
- package/framework/evals/proof-rubrics/fiacao.md +54 -0
- package/framework/evals/proof-rubrics/guard-cego.md +52 -0
- package/framework/evals/proof-rubrics/media-claims.md +109 -0
- package/framework/evals/proof-rubrics/mutation-proof.md +77 -0
- package/framework/evals/proof-rubrics/observabilidade-genai.md +119 -0
- package/framework/evals/proof-rubrics/verde-nao-e-coberto.md +48 -0
- package/framework/evals/rubrics/architecture.md +30 -0
- package/framework/evals/rubrics/code-quality.md +26 -0
- package/framework/evals/rubrics/test-coverage.md +20 -0
- package/framework/hooks/claude-code/core/protect-spec-files.js +41 -4
- package/framework/hooks/claude-code/statusline.sh +11 -11
- package/framework/hooks/git/commit-msg/conventional-commits.sh +33 -33
- package/framework/hooks/git/pre-commit/agents.sh +25 -25
- package/framework/hooks/git/pre-commit/orchestrator.sh +64 -64
- package/framework/hooks/git/pre-commit/specs.sh +50 -50
- package/framework/hooks/git/pre-push/run-tests.sh +44 -44
- package/framework/hooks/shared/e2e-check.js +135 -22
- package/framework/hooks/shared/stdin-reader.js +26 -26
- package/framework/rules/ai-agents-standards.md +14 -2
- package/framework/schemas/ai-pin.schema.json +97 -0
- package/framework/schemas/config.schema.json +107 -0
- package/framework/schemas/feature.schema.json +23 -4
- package/framework/skills/core/create-umbraco-lp/SKILL.md +1 -1
- package/framework/skills/core/migrate-lp-to-umbraco/SKILL.md +1 -1
- package/framework/skills/core/morph-apply/SKILL.md +26 -5
- package/framework/skills/core/morph-brainstorming/SKILL.md +1 -1
- package/framework/skills/core/morph-eval/SKILL.md +1 -1
- package/framework/skills/core/morph-hotfix/SKILL.md +2 -2
- package/framework/skills/core/morph-implement/SKILL.md +18 -2
- package/framework/skills/core/morph-media/SKILL.md +215 -0
- package/framework/skills/core/morph-plan/SKILL.md +33 -2
- package/framework/skills/core/morph-proposal/SKILL.md +1 -1
- package/framework/skills/core/morph-review/SKILL.md +166 -9
- package/framework/skills/core/morph-stack-scan/SKILL.md +1 -1
- package/framework/skills/core/morph-standards/SKILL.md +1 -1
- package/framework/skills/core/morph-uiux/SKILL.md +1 -1
- package/framework/standards/PROVENANCE.json +81 -3
- package/framework/standards/STANDARDS.json +1483 -839
- package/framework/standards/ai-agents/agent-archetypes.md +160 -0
- package/framework/standards/ai-agents/agent-session.md +164 -68
- package/framework/standards/ai-agents/agent-spec.md +196 -0
- package/framework/standards/ai-agents/batching.md +61 -37
- package/framework/standards/ai-agents/context-providers.md +97 -24
- package/framework/standards/ai-agents/conversational-agent-with-phases.md +239 -0
- package/framework/standards/ai-agents/cost-and-budget.md +256 -0
- package/framework/standards/ai-agents/durable-workflows-hitl.md +98 -5
- package/framework/standards/ai-agents/evals-with-cache.md +289 -0
- package/framework/standards/ai-agents/guardrails.md +222 -0
- package/framework/standards/ai-agents/llm-runtime-defaults.md +226 -0
- package/framework/standards/ai-agents/mcp-server.md +119 -18
- package/framework/standards/ai-agents/mcp-tools.md +85 -32
- package/framework/standards/ai-agents/media-pipeline.md +352 -0
- package/framework/standards/ai-agents/media-video.md +267 -0
- package/framework/standards/ai-agents/middleware-patterns.md +39 -5
- package/framework/standards/ai-agents/modalities-image-gen.md +354 -105
- package/framework/standards/ai-agents/modalities-speech-to-text.md +8 -2
- package/framework/standards/ai-agents/modalities-text-to-speech.md +10 -2
- package/framework/standards/ai-agents/modalities-vision.md +6 -2
- package/framework/standards/ai-agents/multi-agent-patterns.md +61 -5
- package/framework/standards/ai-agents/observability-patterns.md +227 -17
- package/framework/standards/ai-agents/production.md +38 -9
- package/framework/standards/ai-agents/prompt-sources.md +389 -0
- package/framework/standards/ai-agents/providers/model-registry.md +174 -39
- package/framework/standards/ai-agents/rag-custom-pgvector.md +69 -4
- package/framework/standards/ai-agents/service-tier-flex.md +27 -19
- package/framework/standards/ai-agents/setup.md +166 -50
- package/framework/standards/ai-agents/structured-output.md +34 -3
- package/framework/standards/ai-agents/sweet-spot.md +23 -3
- package/framework/standards/ai-agents/testing-ai.md +236 -0
- package/framework/standards/ai-agents/vector-data-extensions.md +50 -20
- package/framework/standards/ai-agents/workflows.md +27 -7
- package/framework/standards/frontend/design-system/ai-image-generation.md +12 -0
- package/framework/standards/frontend/design-system/ai-video-generation.md +13 -0
- package/framework/standards/infrastructure/docker/local-compose-isolation.md +3 -1
- package/framework/templates/.idea/morph-templates.xml +92 -0
- package/framework/templates/.vscode/morph-templates.code-snippets +186 -0
- package/framework/templates/IDE-SNIPPETS.md +262 -262
- package/framework/templates/REGISTRY.json +13 -0
- package/framework/templates/code/dotnet/abstractions/api-endpoint.cs +7 -7
- package/framework/templates/code/dotnet/ai-agents/AgentBootstrap.cs.template +39 -39
- package/framework/templates/code/dotnet/ai-agents/EmbeddingService.cs.template +68 -68
- package/framework/templates/code/dotnet/ai-agents/ModelRegistry.cs.template +117 -117
- package/framework/templates/code/dotnet/ai-agents/PromptRepository.cs.template +1 -1
- package/framework/templates/code/dotnet/ai-agents/ResilientToolMiddleware.cs.template +61 -61
- package/framework/templates/code/dotnet/ai-agents/VectorSearchTool.cs.template +107 -107
- package/framework/templates/code/dotnet/ai-agents/model-registry.json.template +58 -58
- package/framework/templates/code/dotnet/contracts/contracts-vsa.cs +281 -281
- package/framework/templates/code/dotnet/database/migration.cs +83 -83
- package/framework/templates/code/dotnet/feature/endpoint.cs.hbs +21 -21
- package/framework/templates/code/dotnet/feature/request.cs.hbs +4 -4
- package/framework/templates/code/dotnet/feature/response.cs.hbs +4 -4
- package/framework/templates/code/dotnet/feature/validator.cs.hbs +15 -15
- package/framework/templates/code/sql/ai-agents/agent-prompts-migration.sql.template +1 -1
- package/framework/templates/code/sql/ai-agents/pgvector-embeddings-migration.sql.template +91 -91
- package/framework/templates/code/typescript/contracts.ts +167 -167
- package/framework/templates/docs/clarifications.md +253 -253
- package/framework/templates/docs/schema-analysis.md +119 -119
- package/framework/templates/docs/ui-components.md +52 -52
- package/framework/templates/docs/ui-design-system.md +76 -76
- package/framework/templates/docs/ui-flows.md +167 -167
- package/framework/templates/docs/ui-mockups.md +98 -98
- package/framework/templates/dotnet/ai-kit/Directory.Build.props +25 -0
- package/framework/templates/dotnet/ai-kit/Directory.Packages.props +26 -0
- package/framework/templates/dotnet/ai-kit/Morph.AiKit.sln +56 -0
- package/framework/templates/dotnet/ai-kit/NuGet.config +12 -0
- package/framework/templates/dotnet/ai-kit/README.md +128 -0
- package/framework/templates/dotnet/ai-kit/global.json +9 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Agents/AgentApi.cs +65 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Agents/AgentFactory.cs +289 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Agents/AgentFactoryOptions.cs +91 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Agents/AgentSpec.cs +91 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Agents/AgentSpecOptions.cs +181 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/AiKitAssemblyMarker.cs +14 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Conversation/ConversationStore.cs +174 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Conversation/IConversationStore.cs +54 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Conversation/InMemoryConversationStore.cs +75 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Evals/CountingChatClient.cs +71 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Evals/EvalAssertion.cs +246 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Evals/EvalCacheMissException.cs +57 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Evals/EvalCacheOptions.cs +78 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Evals/EvalCacheStamp.cs +225 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Evals/EvalFixture.cs +463 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Evals/EvalMode.cs +32 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Evals/EvalQualityEvaluators.cs +49 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Evals/EvalScenario.cs +214 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Evals/EvalSummary.cs +168 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Evals/OfflineChatClient.cs +142 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Evals/ShortPathResponseCache.cs +349 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/IMediaGenerator.cs +42 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/IMediaProvider.cs +77 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MediaArtifact.cs +28 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MediaDisclosure.cs +107 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MediaGenerator.cs +253 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MediaIdempotencyKey.cs +103 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MediaJob.cs +259 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MediaJobId.cs +21 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MediaJobState.cs +47 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MediaPricing.cs +102 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MediaProvenance.cs +44 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MediaRequest.cs +192 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MediaResult.cs +94 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MediaReview.cs +99 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MediaSeams.cs +97 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MediaStore.cs +133 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MorphMedia.cs +93 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/Providers/GoogleImageProvider.cs +112 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/Providers/OpenAIImageProvider.cs +288 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/Providers/VeoVideoProvider.cs +158 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Middleware/IUsageCostSink.cs +29 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Middleware/UsageCostMiddleware.cs +208 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Middleware/UsageCostRecord.cs +95 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Morph.AiKit.csproj +38 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Observability/MorphOpenTelemetryExtensions.cs +119 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Observability/MorphTelemetryOptions.cs +82 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Providers/IProviderClientFactory.cs +33 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Providers/ModelAlias.cs +267 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Providers/ModelRegistry.cs +474 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Providers/model-registry.v2.json +87 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Rag/DeltaIngest.cs +169 -0
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Rag/DeltaPlan.cs +49 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/AgentFactoryTests.cs +495 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/AgentSpecTests.cs +260 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/ConversationStoreTests.cs +405 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/DeltaIngestTests.cs +379 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Evals/EvalAssertionTests.cs +197 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Evals/EvalCacheMissTests.cs +139 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Evals/EvalCacheStampTests.cs +157 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Evals/EvalFailuresTests.cs +178 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Evals/EvalFixtureTests.cs +153 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Evals/EvalRefreshTests.cs +194 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Evals/EvalWorkspace.cs +86 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Evals/KitEvals.cs +213 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Evals/OfflineChatClientTests.cs +95 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Fakes/FakeChatClient.cs +225 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Fakes/FakeChatClientTests.cs +215 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Fakes/FakeClock.cs +33 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Fakes/FakeMediaProvider.cs +89 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Fakes/FakeProviderClientFactory.cs +55 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Fakes/FakeUsageCostSink.cs +60 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Fakes/InMemoryMediaStore.cs +131 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/MediaAccessibilityTests.cs +133 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/MediaAliasIsNotAnAgentTests.cs +127 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/MediaDisclosureTests.cs +216 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/MediaGeneratorTests.cs +264 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/MediaIdempotencyKeyTests.cs +148 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/MediaJobStateTests.cs +184 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/MediaReviewTests.cs +89 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/MediaStoreTests.cs +98 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/ModelRegistryTests.cs +451 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Morph.AiKit.Tests.csproj +76 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/SkeletonSmokeTests.cs +33 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/TelemetryTests.cs +261 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/UsageCostMiddlewareTests.cs +374 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/eval-cache/.morph-eval-cache.json +12 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/eval-cache/cache/orcamento.acme.a-resposta-estruturada/1/CF70A7CE27BBD66A/contents.data +13 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/eval-cache/cache/orcamento.acme.a-resposta-estruturada/1/CF70A7CE27BBD66A/entry.json +7 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/eval-cache/cache/triagem.acme.a-saudacao/1/19F631228C6E00BA/contents.data +13 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/eval-cache/cache/triagem.acme.a-saudacao/1/19F631228C6E00BA/entry.json +7 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/eval-cache/cache/triagem.acme.b-escalonamento/1/A24A7C9BDD8239D8/contents.data +13 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/eval-cache/cache/triagem.acme.b-escalonamento/1/A24A7C9BDD8239D8/entry.json +7 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/eval-cache/cache/triagem.globex.a-saudacao/1/E38CD37DF003EDB7/contents.data +13 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/eval-cache/cache/triagem.globex.a-saudacao/1/E38CD37DF003EDB7/entry.json +7 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/eval-datasets/orcamento/acme/a-resposta-estruturada.json +24 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/eval-datasets/triagem/acme/a-saudacao.json +23 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/eval-datasets/triagem/acme/b-escalonamento.json +24 -0
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/eval-datasets/triagem/globex/a-saudacao.json +23 -0
- package/framework/templates/feature/decisions.md +122 -122
- package/framework/templates/frontend/nextjs/Dockerfile.nextjs.hbs +43 -43
- package/framework/templates/frontend/nextjs/client-component.tsx.hbs +26 -26
- package/framework/templates/frontend/nextjs/env.mjs.hbs +32 -32
- package/framework/templates/frontend/nextjs/feature-form.tsx.hbs +56 -56
- package/framework/templates/frontend/nextjs/page.tsx.hbs +22 -22
- package/framework/templates/frontend/nextjs/tsconfig.json.hbs +26 -26
- package/framework/templates/frontend/nextjs/use-feature.ts.hbs +54 -54
- package/framework/templates/infrastructure/docker/Dockerfile.template +58 -58
- package/framework/templates/infrastructure/docker/dockerfile-api.dockerfile +38 -38
- package/framework/templates/infrastructure/docker/dockerfile-web.dockerfile +48 -48
- package/framework/templates/infrastructure/github/README.md +56 -56
- package/framework/templates/infrastructure/github/actions/docker-build-push/action.yml.hbs +45 -45
- package/framework/templates/infrastructure/github/actions/health-check/action.yml.hbs +27 -27
- package/framework/templates/infrastructure/github/workflows/docker-build-push.yml.hbs +59 -59
- package/framework/templates/infrastructure/github/workflows/dotnet-build.yml.hbs +39 -39
- package/framework/templates/integrations/asaas-client.cs +387 -387
- package/framework/templates/integrations/asaas-webhook.cs +351 -351
- package/framework/templates/meta-prompts/fusion/fusion-agent.md +76 -76
- package/framework/templates/meta-prompts/fusion/fusion-aggregator.md +100 -100
- package/framework/templates/meta-prompts/validators/pre-commit-validator.md +95 -95
- package/framework/templates/project/validate.js +120 -120
- package/framework/templates/state.template.json +17 -17
- package/framework/templates/ui/design-system.css +226 -226
- package/package.json +3 -3
- package/src/commands/dashboard/dashboard.js +53 -53
- package/src/commands/e2e/e2e.js +15 -2
- package/src/commands/index.js +13 -13
- package/src/commands/project/checkpoint.js +209 -209
- package/src/commands/project/doctor.js +140 -5
- package/src/commands/project/update.js +469 -153
- package/src/commands/project/worktree.js +8 -1
- package/src/commands/state/advance.js +24 -2
- package/src/commands/state/approve.js +33 -1
- package/src/commands/state/create.js +72 -11
- package/src/commands/state/finish.js +333 -21
- package/src/commands/state/gate-check.js +190 -6
- package/src/commands/state/score.js +36 -1
- package/src/commands/validation/verify.js +134 -25
- package/src/core/state/phase-state-machine.js +197 -197
- package/src/core/state/state-manager.js +9 -1
- package/src/core/templates/template-renderer.js +477 -477
- package/src/dashboard/app/components/DetailPanel.jsx +11 -11
- package/src/dashboard/app/components/FlowGraph.jsx +17 -17
- package/src/dashboard/app/components/SearchFilter.jsx +22 -22
- package/src/dashboard/app/hooks/useDetailPanel.js +15 -15
- package/src/dashboard/app/main.jsx +13 -13
- package/src/dashboard/app/pages/AgentMap.jsx +370 -370
- package/src/dashboard/app/pages/PhasePipeline.jsx +408 -408
- package/src/dashboard/app/pages/RulesMap.jsx +273 -273
- package/src/dashboard/app/pages/SkillsLibrary.jsx +366 -366
- package/src/dashboard/app/pages/Standards.jsx +332 -332
- package/src/dashboard/app/styles/index.css +47 -47
- package/src/dashboard/index.html +12 -12
- package/src/dashboard/package.json +24 -24
- package/src/dashboard/scanner/phases-scanner.js +15 -15
- package/src/dashboard/scanner/rubrics-scanner.js +12 -12
- package/src/dashboard/scanner/rules-scanner.js +37 -37
- package/src/dashboard/scanner/skills-scanner.js +43 -43
- package/src/dashboard/scanner/squads-scanner.js +13 -13
- package/src/dashboard/vite-plugin-morph-data.js +17 -17
- package/src/dashboard/vite.config.js +24 -24
- package/src/lib/ai/ai-checks.js +182 -0
- package/src/lib/ai/ai-project-scan.js +347 -0
- package/src/lib/ai/code-audit.js +181 -0
- package/src/lib/ai/comment-stripper.js +184 -0
- package/src/lib/ai/pin-drift.js +138 -0
- package/src/lib/ai/registry-audit.js +285 -0
- package/src/lib/detectors/design-system-detector.js +188 -188
- package/src/lib/detectors/index.js +217 -217
- package/src/lib/e2e/compose-env.js +50 -4
- package/src/lib/evaluator-report.js +269 -15
- package/src/lib/generators/design-system-generator.js +508 -508
- package/src/lib/generators/settings-generator.js +209 -209
- package/src/lib/git/default-branch.js +92 -4
- package/src/lib/hooks/hook-executor.js +167 -167
- package/src/lib/installers/effects.js +193 -0
- package/src/lib/installers/mcp-installer.js +719 -719
- package/src/lib/installers/templates-reconcile.js +3 -3
- package/src/lib/risk-detector.js +284 -0
- package/src/lib/standards/verified-against.js +276 -0
- package/src/lib/tasks/eval-node.js +485 -0
- package/src/lib/tasks/linux-runner.js +297 -0
- package/src/lib/tasks/task-classifier.js +30 -30
- package/src/lib/tasks/task-parser.js +209 -209
- package/src/lib/tasks/test-nodes.js +486 -0
- package/src/lib/tasks/test-runner.js +410 -33
- package/src/lib/validators/checkpoint-persistence.js +39 -39
- package/src/lib/validators/css/css-validator.js +340 -340
- package/src/lib/validators/shared/emit-validator-dispatch.js +14 -14
- package/src/lib/validators/shared/issue-counter.js +14 -14
- package/src/lib/validators/verify-runner.js +583 -28
- package/src/lib/worktree/docker-audit.js +287 -0
- package/src/lib/worktree/env-allocator.js +24 -0
- package/src/lib/worktree/worktree-audit.js +30 -1
- package/src/scripts/setup-infra.js +24 -11
- package/src/utils/agent-id-mapper.js +11 -11
- package/src/utils/agents-installer.js +18 -8
- package/src/utils/claude-md-injector.js +10 -5
- package/src/utils/claude-settings-manager.js +300 -300
- package/src/utils/file-copier.js +11 -4
- package/src/utils/hooks-installer.js +29 -26
- package/src/utils/skills-installer.js +36 -29
- package/src/utils/version-checker.js +68 -3
- package/framework/standards/ai-agents/prompts-in-db.md +0 -214
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
using System.Diagnostics;
|
|
2
|
+
using Microsoft.Agents.AI;
|
|
3
|
+
using Microsoft.Extensions.AI;
|
|
4
|
+
using Morph.AiKit.Agents;
|
|
5
|
+
using Morph.AiKit.Observability;
|
|
6
|
+
using Morph.AiKit.Providers;
|
|
7
|
+
using Morph.AiKit.Tests.Fakes;
|
|
8
|
+
using Xunit;
|
|
9
|
+
|
|
10
|
+
namespace Morph.AiKit.Tests;
|
|
11
|
+
|
|
12
|
+
/// <summary>
|
|
13
|
+
/// Canarios da telemetria (T12). O arquivo inteiro mede <b>spans</b>, com
|
|
14
|
+
/// <see cref="ActivityListener"/> — nao propriedades de configuracao. Um guard contra span
|
|
15
|
+
/// duplicado so pode ser provado contando spans.
|
|
16
|
+
/// </summary>
|
|
17
|
+
/// <remarks>
|
|
18
|
+
/// Cada caso usa um <c>SourceName</c> unico, para que um teste nunca conte o span do outro.
|
|
19
|
+
/// </remarks>
|
|
20
|
+
public sealed class TelemetryTests
|
|
21
|
+
{
|
|
22
|
+
private const string Document = """
|
|
23
|
+
{
|
|
24
|
+
"version": "2.0",
|
|
25
|
+
"defaultAlias": "text",
|
|
26
|
+
"aliases": {
|
|
27
|
+
"text": {
|
|
28
|
+
"provider": "openai",
|
|
29
|
+
"model": "gpt-5.4-mini",
|
|
30
|
+
"pricing": { "inputPerMTok": 0.25, "outputPerMTok": 2.0 }
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
""";
|
|
35
|
+
|
|
36
|
+
private static AgentSpec Spec(AgentSpecOptions? options = null)
|
|
37
|
+
=> new()
|
|
38
|
+
{
|
|
39
|
+
Alias = "text",
|
|
40
|
+
Instructions = "voce e um agente de teste",
|
|
41
|
+
Name = "agente-instrumentado",
|
|
42
|
+
Options = options ?? new AgentSpecOptions(),
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
// ---------------------------------------------------------------------------------
|
|
46
|
+
// O caso normal: um span de agente, um span de chat
|
|
47
|
+
// ---------------------------------------------------------------------------------
|
|
48
|
+
|
|
49
|
+
/// <summary>
|
|
50
|
+
/// A forma que as GenAI conventions descrevem: <c>invoke_agent</c> como raiz e
|
|
51
|
+
/// <c>chat</c> como filho. Este teste tambem e a prova de que a escolha da T10 de montar
|
|
52
|
+
/// o proprio pipeline nao custou o span de chat — ele so existe porque a T12 o coloca.
|
|
53
|
+
/// </summary>
|
|
54
|
+
[Fact]
|
|
55
|
+
public async Task AnInstrumentedAgentEmitsOneInvokeAgentSpanAndOneChatSpan()
|
|
56
|
+
{
|
|
57
|
+
var source = NewSourceName();
|
|
58
|
+
|
|
59
|
+
var spans = await CollectAsync(source, () => BuildAgent(
|
|
60
|
+
new MorphTelemetryOptions { SourceName = source },
|
|
61
|
+
leaf: new FakeChatClient().EnqueueText("pronto")));
|
|
62
|
+
|
|
63
|
+
Assert.Equal(1, CountOf(spans, "invoke_agent"));
|
|
64
|
+
Assert.Equal(1, CountOf(spans, "chat"));
|
|
65
|
+
Assert.Null(Single(spans, "invoke_agent").Parent);
|
|
66
|
+
Assert.NotNull(Single(spans, "chat").Parent);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// ---------------------------------------------------------------------------------
|
|
70
|
+
// O guard — as duas metades da mesma afirmacao
|
|
71
|
+
// ---------------------------------------------------------------------------------
|
|
72
|
+
|
|
73
|
+
/// <summary>
|
|
74
|
+
/// <b>O guard, no default.</b> Um pipeline que ja carrega instrumentacao (o caso real:
|
|
75
|
+
/// um <c>IChatClient</c> que veio do DI ja instrumentado) continua com UM span de chat.
|
|
76
|
+
/// </summary>
|
|
77
|
+
[Fact]
|
|
78
|
+
public async Task TheDefaultDoesNotAddASecondChatSpanOverAnAlreadyInstrumentedPipeline()
|
|
79
|
+
{
|
|
80
|
+
var source = NewSourceName();
|
|
81
|
+
|
|
82
|
+
var spans = await CollectAsync(source, () => BuildAgent(
|
|
83
|
+
new MorphTelemetryOptions { SourceName = source },
|
|
84
|
+
leaf: AlreadyInstrumented(source)));
|
|
85
|
+
|
|
86
|
+
Assert.Equal(1, CountOf(spans, "invoke_agent"));
|
|
87
|
+
Assert.Equal(1, CountOf(spans, "chat"));
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/// <summary>
|
|
91
|
+
/// <b>O guard, desligado.</b> Forcar <c>InstrumentChatClient = true</c> sobre o MESMO
|
|
92
|
+
/// pipeline ja instrumentado produz o par duplicado. E o canario que documenta o risco:
|
|
93
|
+
/// sem o ramo automatico, este e o resultado.
|
|
94
|
+
/// </summary>
|
|
95
|
+
[Fact]
|
|
96
|
+
public async Task ForcingInstrumentChatClientProducesTheDuplicateChatSpan()
|
|
97
|
+
{
|
|
98
|
+
var source = NewSourceName();
|
|
99
|
+
|
|
100
|
+
var spans = await CollectAsync(source, () => BuildAgent(
|
|
101
|
+
new MorphTelemetryOptions { SourceName = source, InstrumentChatClient = true },
|
|
102
|
+
leaf: AlreadyInstrumented(source)));
|
|
103
|
+
|
|
104
|
+
Assert.Equal(1, CountOf(spans, "invoke_agent"));
|
|
105
|
+
Assert.Equal(2, CountOf(spans, "chat"));
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/// <summary>
|
|
109
|
+
/// O mesmo guard no nivel do agente: aplicar a telemetria duas vezes no
|
|
110
|
+
/// <see cref="AIAgentBuilder"/> nao produz dois <c>invoke_agent</c>.
|
|
111
|
+
/// </summary>
|
|
112
|
+
[Fact]
|
|
113
|
+
public async Task TheAgentGuardKeepsASingleInvokeAgentSpanWhenTelemetryIsAppliedTwice()
|
|
114
|
+
{
|
|
115
|
+
var source = NewSourceName();
|
|
116
|
+
var telemetry = new MorphTelemetryOptions { SourceName = source };
|
|
117
|
+
|
|
118
|
+
var spans = await CollectAsync(source, () =>
|
|
119
|
+
{
|
|
120
|
+
var agent = BuildAgent(telemetry, new FakeChatClient().EnqueueText("pronto"));
|
|
121
|
+
return agent.AsBuilder().UseMorphTelemetry(telemetry).Build();
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
Assert.Equal(1, CountOf(spans, "invoke_agent"));
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/// <summary>
|
|
128
|
+
/// <c>false</c> nao e o mesmo que o default. Com a <c>AgentFactory</c> — que monta o
|
|
129
|
+
/// proprio pipeline — desligar a instrumentacao de chat deixa a invocacao com o span do
|
|
130
|
+
/// agente e <b>nenhum</b> span de chat. O teste existe para que essa consequencia seja
|
|
131
|
+
/// um fato escrito, e nao uma surpresa em producao.
|
|
132
|
+
/// </summary>
|
|
133
|
+
[Fact]
|
|
134
|
+
public async Task TurningTheChatInstrumentationOffLeavesTheAgentSpanAndNoChatSpan()
|
|
135
|
+
{
|
|
136
|
+
var source = NewSourceName();
|
|
137
|
+
|
|
138
|
+
var spans = await CollectAsync(source, () => BuildAgent(
|
|
139
|
+
new MorphTelemetryOptions { SourceName = source, InstrumentChatClient = false },
|
|
140
|
+
leaf: new FakeChatClient().EnqueueText("pronto")));
|
|
141
|
+
|
|
142
|
+
Assert.Equal(1, CountOf(spans, "invoke_agent"));
|
|
143
|
+
Assert.Equal(0, CountOf(spans, "chat"));
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// ---------------------------------------------------------------------------------
|
|
147
|
+
// Dado sensivel
|
|
148
|
+
// ---------------------------------------------------------------------------------
|
|
149
|
+
|
|
150
|
+
/// <summary>
|
|
151
|
+
/// Conteudo sensivel nasce desligado nos dois niveis — o standard exige <c>false</c>
|
|
152
|
+
/// fora de dev/staging, porque prompt e resposta sao PII do cliente.
|
|
153
|
+
/// </summary>
|
|
154
|
+
[Fact]
|
|
155
|
+
public void SensitiveDataIsOffByDefaultAtBothLevels()
|
|
156
|
+
{
|
|
157
|
+
var agent = BuildAgent(
|
|
158
|
+
new MorphTelemetryOptions { SourceName = NewSourceName() },
|
|
159
|
+
new FakeChatClient());
|
|
160
|
+
|
|
161
|
+
Assert.False(Assert.IsType<OpenTelemetryAgent>(agent).EnableSensitiveData);
|
|
162
|
+
Assert.False(Assert.IsType<OpenTelemetryChatClient>(
|
|
163
|
+
agent.GetService(typeof(OpenTelemetryChatClient))).EnableSensitiveData);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/// <summary>
|
|
167
|
+
/// A spec pode ligar o dado sensivel para UM agente sem mexer na configuracao global —
|
|
168
|
+
/// e o destino que a T7 declarou para <c>EnableSensitiveTelemetry</c>. Nulo na spec
|
|
169
|
+
/// ("nao opinei") preserva o valor global; preenchido, vence.
|
|
170
|
+
/// </summary>
|
|
171
|
+
[Fact]
|
|
172
|
+
public void TheSpecCanOverrideSensitiveDataForASingleAgent()
|
|
173
|
+
{
|
|
174
|
+
var telemetry = new MorphTelemetryOptions { SourceName = NewSourceName() };
|
|
175
|
+
|
|
176
|
+
var loud = BuildAgent(
|
|
177
|
+
telemetry,
|
|
178
|
+
new FakeChatClient(),
|
|
179
|
+
spec: Spec(new AgentSpecOptions { EnableSensitiveTelemetry = true }));
|
|
180
|
+
|
|
181
|
+
Assert.True(Assert.IsType<OpenTelemetryAgent>(loud).EnableSensitiveData);
|
|
182
|
+
Assert.True(Assert.IsType<OpenTelemetryChatClient>(
|
|
183
|
+
loud.GetService(typeof(OpenTelemetryChatClient))).EnableSensitiveData);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// ---------------------------------------------------------------------------------
|
|
187
|
+
// Configuracao invalida
|
|
188
|
+
// ---------------------------------------------------------------------------------
|
|
189
|
+
|
|
190
|
+
/// <summary>
|
|
191
|
+
/// Source em branco e recusado na construcao: um <c>ActivitySource</c> sem nome nao pode
|
|
192
|
+
/// ser registrado em <c>AddSource(...)</c>, e a telemetria sumiria em silencio.
|
|
193
|
+
/// </summary>
|
|
194
|
+
[Fact]
|
|
195
|
+
public void ABlankSourceNameIsRefusedAtConstruction()
|
|
196
|
+
=> Assert.Throws<ArgumentException>(() => new MorphTelemetryOptions { SourceName = " " });
|
|
197
|
+
|
|
198
|
+
// ---------------------------------------------------------------------------------
|
|
199
|
+
// Apoio dos testes
|
|
200
|
+
// ---------------------------------------------------------------------------------
|
|
201
|
+
|
|
202
|
+
private static string NewSourceName() => "morph.aikit.test." + Guid.CreateVersion7().ToString("N");
|
|
203
|
+
|
|
204
|
+
/// <summary>Um pipeline que ja chega instrumentado — o caso real do cliente vindo do DI.</summary>
|
|
205
|
+
private static IChatClient AlreadyInstrumented(string sourceName)
|
|
206
|
+
=> new FakeChatClient()
|
|
207
|
+
.EnqueueText("pronto")
|
|
208
|
+
.AsBuilder()
|
|
209
|
+
.UseOpenTelemetry(sourceName: sourceName)
|
|
210
|
+
.Build();
|
|
211
|
+
|
|
212
|
+
private static AIAgent BuildAgent(MorphTelemetryOptions telemetry, IChatClient leaf, AgentSpec? spec = null)
|
|
213
|
+
{
|
|
214
|
+
using var registry = ModelRegistry.FromJson(Document, new FakeProviderClientFactory());
|
|
215
|
+
var factory = new AgentFactory(
|
|
216
|
+
registry,
|
|
217
|
+
new AgentFactoryOptions { ChatClientSelector = (_, _) => leaf }.WithMorphTelemetry(telemetry));
|
|
218
|
+
|
|
219
|
+
return factory.Create(spec ?? Spec());
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/// <summary>
|
|
223
|
+
/// Coleta os spans de UMA invocacao. O listener e registrado antes de o agente existir e
|
|
224
|
+
/// removido no fim — telemetria e estado global do processo, e um listener vazado
|
|
225
|
+
/// contaminaria os outros casos.
|
|
226
|
+
/// </summary>
|
|
227
|
+
private static async Task<IReadOnlyList<Activity>> CollectAsync(string sourceName, Func<AIAgent> build)
|
|
228
|
+
{
|
|
229
|
+
var spans = new List<Activity>();
|
|
230
|
+
|
|
231
|
+
using var listener = new ActivityListener
|
|
232
|
+
{
|
|
233
|
+
ShouldListenTo = candidate => candidate.Name == sourceName,
|
|
234
|
+
Sample = (ref ActivityCreationOptions<ActivityContext> _) => ActivitySamplingResult.AllDataAndRecorded,
|
|
235
|
+
ActivityStopped = activity =>
|
|
236
|
+
{
|
|
237
|
+
lock (spans)
|
|
238
|
+
{
|
|
239
|
+
spans.Add(activity);
|
|
240
|
+
}
|
|
241
|
+
},
|
|
242
|
+
};
|
|
243
|
+
|
|
244
|
+
ActivitySource.AddActivityListener(listener);
|
|
245
|
+
|
|
246
|
+
var agent = build();
|
|
247
|
+
var session = await agent.CreateSessionAsync(TestContext.Current.CancellationToken);
|
|
248
|
+
_ = await agent.RunAsync("oi", session, cancellationToken: TestContext.Current.CancellationToken);
|
|
249
|
+
|
|
250
|
+
lock (spans)
|
|
251
|
+
{
|
|
252
|
+
return [.. spans];
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
private static int CountOf(IReadOnlyList<Activity> spans, string operation)
|
|
257
|
+
=> spans.Count(span => Equals(span.GetTagItem("gen_ai.operation.name"), operation));
|
|
258
|
+
|
|
259
|
+
private static Activity Single(IReadOnlyList<Activity> spans, string operation)
|
|
260
|
+
=> spans.Single(span => Equals(span.GetTagItem("gen_ai.operation.name"), operation));
|
|
261
|
+
}
|
|
@@ -0,0 +1,374 @@
|
|
|
1
|
+
using Microsoft.Extensions.AI;
|
|
2
|
+
using Morph.AiKit.Agents;
|
|
3
|
+
using Morph.AiKit.Middleware;
|
|
4
|
+
using Morph.AiKit.Providers;
|
|
5
|
+
using Morph.AiKit.Tests.Fakes;
|
|
6
|
+
using Xunit;
|
|
7
|
+
|
|
8
|
+
namespace Morph.AiKit.Tests;
|
|
9
|
+
|
|
10
|
+
/// <summary>
|
|
11
|
+
/// Canarios do <see cref="UsageCostMiddleware"/> (T11). A afirmacao central da task e uma
|
|
12
|
+
/// so: <b><c>cost_usd</c> nao sai nulo, e o zero nunca e ambiguo</b> — ou e medicao, ou e
|
|
13
|
+
/// dispensa escrita, ou e lacuna declarada.
|
|
14
|
+
/// </summary>
|
|
15
|
+
public sealed class UsageCostMiddlewareTests
|
|
16
|
+
{
|
|
17
|
+
private static readonly DateTimeOffset FixedNow = new(2026, 9, 8, 12, 0, 0, TimeSpan.Zero);
|
|
18
|
+
|
|
19
|
+
/// <summary>Tarifa ilustrativa com desconto de cache — a que exercita a aritmetica inteira.</summary>
|
|
20
|
+
private static ModelPricing Tariff()
|
|
21
|
+
=> new(InputPerMTok: 1.25m, OutputPerMTok: 10.0m, CachedInputPerMTok: 0.125m);
|
|
22
|
+
|
|
23
|
+
private static ModelAlias PricedAlias(ModelPricing? pricing = null)
|
|
24
|
+
=> new("openai", "gpt-5.4", AgentApi.Responses, pricing ?? Tariff(), Endpoint: null)
|
|
25
|
+
{
|
|
26
|
+
Name = "text-heavy",
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
private static ModelAlias WaivedAlias()
|
|
30
|
+
=> new("ollama", "qwen3:8b", AgentApi.Responses, Pricing: null, Endpoint: "http://localhost:11434")
|
|
31
|
+
{
|
|
32
|
+
Name = "local-dev",
|
|
33
|
+
PricingWaiver = "modelo local: a conta e a maquina, nao o token.",
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
private static (IChatClient Client, FakeChatClient Inner, FakeUsageCostSink Sink) Pipeline(
|
|
37
|
+
ModelAlias alias,
|
|
38
|
+
Action<FakeChatClient> script)
|
|
39
|
+
{
|
|
40
|
+
var inner = new FakeChatClient();
|
|
41
|
+
script(inner);
|
|
42
|
+
|
|
43
|
+
var sink = new FakeUsageCostSink();
|
|
44
|
+
|
|
45
|
+
// A extensao publica e o caminho de registro: se ela quebrar, o teste quebra junto.
|
|
46
|
+
var client = inner.AsBuilder().UseUsageCost(alias, sink, new FakeClock(FixedNow)).Build();
|
|
47
|
+
|
|
48
|
+
return (client, inner, sink);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// ---------------------------------------------------------------------------------
|
|
52
|
+
// doneCriteria 1: Usage conhecido + ModelPricing conhecido => CostUsd EXATO
|
|
53
|
+
// ---------------------------------------------------------------------------------
|
|
54
|
+
|
|
55
|
+
/// <summary>
|
|
56
|
+
/// O valor esperado esta escrito a mao, com a semantica correta de token cacheado:
|
|
57
|
+
/// <b>cacheado e SUBCONJUNTO da entrada</b>. 1000 de entrada com 400 servidos do cache
|
|
58
|
+
/// custam 600 na tarifa cheia mais 400 na de cache — nunca 1000 mais 400. E por isso que
|
|
59
|
+
/// este caso, e nao um sem cache, e o que denuncia uma segunda aritmetica.
|
|
60
|
+
/// </summary>
|
|
61
|
+
[Fact]
|
|
62
|
+
public async Task AKnownUsageAgainstAKnownTariffProducesTheExactCostInUsd()
|
|
63
|
+
{
|
|
64
|
+
var (client, _, sink) = Pipeline(
|
|
65
|
+
PricedAlias(),
|
|
66
|
+
fake => fake.EnqueueTextWithUsage("ok", inputTokens: 1000, outputTokens: 200, cachedInputTokens: 400));
|
|
67
|
+
|
|
68
|
+
await client.GetResponseAsync(
|
|
69
|
+
[new ChatMessage(ChatRole.User, "oi")],
|
|
70
|
+
cancellationToken: TestContext.Current.CancellationToken);
|
|
71
|
+
|
|
72
|
+
var record = Assert.Single(sink.Records);
|
|
73
|
+
|
|
74
|
+
// (600 * 1.25) + (400 * 0.125) + (200 * 10.0) = 750 + 50 + 2000 = 2800 por 1M tokens.
|
|
75
|
+
Assert.Equal(0.0028m, record.CostUsd);
|
|
76
|
+
Assert.Equal(UsageCostStatus.Computed, record.Status);
|
|
77
|
+
Assert.True(record.IsMeasuredCost);
|
|
78
|
+
Assert.Equal("text-heavy", record.Alias);
|
|
79
|
+
Assert.Equal(1000, record.InputTokens);
|
|
80
|
+
Assert.Equal(200, record.OutputTokens);
|
|
81
|
+
Assert.Equal(400, record.CachedInputTokens);
|
|
82
|
+
Assert.Null(record.PricingWaiver);
|
|
83
|
+
Assert.Equal(FixedNow, record.At);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/// <summary>
|
|
87
|
+
/// O canario que torna "nao existe uma segunda conta do custo" verificavel em vez de
|
|
88
|
+
/// prometido: o custo publicado tem de ser <b>identico</b> ao de
|
|
89
|
+
/// <see cref="ModelPricing.ComputeCostUsd"/> — inclusive nos casos que separam as duas
|
|
90
|
+
/// formulas plausiveis (cacheado sem tarifa de cache, cacheado igual a entrada, saida
|
|
91
|
+
/// zerada).
|
|
92
|
+
/// </summary>
|
|
93
|
+
[Theory]
|
|
94
|
+
[InlineData(1000, 200, 400L)]
|
|
95
|
+
[InlineData(1000, 200, null)]
|
|
96
|
+
[InlineData(1000, 200, 1000L)]
|
|
97
|
+
[InlineData(1000, 200, 0L)]
|
|
98
|
+
[InlineData(7, 3, 2L)]
|
|
99
|
+
[InlineData(1_000_000, 0, 999_999L)]
|
|
100
|
+
public async Task ThePublishedCostIsTheOneComputeCostUsdWouldGive(long input, long output, long? cached)
|
|
101
|
+
{
|
|
102
|
+
var pricing = Tariff();
|
|
103
|
+
var (client, _, sink) = Pipeline(
|
|
104
|
+
PricedAlias(pricing),
|
|
105
|
+
fake => fake.EnqueueTextWithUsage("ok", input, output, cached));
|
|
106
|
+
|
|
107
|
+
await client.GetResponseAsync(
|
|
108
|
+
[new ChatMessage(ChatRole.User, "oi")],
|
|
109
|
+
cancellationToken: TestContext.Current.CancellationToken);
|
|
110
|
+
|
|
111
|
+
Assert.Equal(pricing.ComputeCostUsd(input, output, cached), sink.Records[0].CostUsd);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/// <summary>
|
|
115
|
+
/// Tarifa sem desconto de cache declarado: nulo em <c>CachedInputPerMTok</c> significa
|
|
116
|
+
/// "sem desconto", <b>nunca</b> "de graca" — a entrada inteira e cobrada cheia mesmo com
|
|
117
|
+
/// tokens vindos do cache.
|
|
118
|
+
/// </summary>
|
|
119
|
+
[Fact]
|
|
120
|
+
public async Task WithoutADeclaredCacheTariffTheWholeInputIsChargedAsInput()
|
|
121
|
+
{
|
|
122
|
+
var (client, _, sink) = Pipeline(
|
|
123
|
+
PricedAlias(new ModelPricing(1.25m, 10.0m, CachedInputPerMTok: null)),
|
|
124
|
+
fake => fake.EnqueueTextWithUsage("ok", inputTokens: 1000, outputTokens: 200, cachedInputTokens: 400));
|
|
125
|
+
|
|
126
|
+
await client.GetResponseAsync(
|
|
127
|
+
[new ChatMessage(ChatRole.User, "oi")],
|
|
128
|
+
cancellationToken: TestContext.Current.CancellationToken);
|
|
129
|
+
|
|
130
|
+
// (1000 * 1.25) + (200 * 10.0) = 3250 por 1M tokens. O cacheado nao rende desconto.
|
|
131
|
+
Assert.Equal(0.00325m, sink.Records[0].CostUsd);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// ---------------------------------------------------------------------------------
|
|
135
|
+
// doneCriteria 2: alias sem Pricing => CostUsd == 0, ausencia sinalizada, SEM excecao
|
|
136
|
+
// ---------------------------------------------------------------------------------
|
|
137
|
+
|
|
138
|
+
/// <summary>
|
|
139
|
+
/// Alias sem tarifa e caminho previsto, nao erro. E o zero nao e anonimo: o motivo
|
|
140
|
+
/// escrito no <c>pricingWaiver</c> do registry viaja no record. Nao ha excecao — e nao
|
|
141
|
+
/// ha nulo em <c>CostUsd</c>.
|
|
142
|
+
/// </summary>
|
|
143
|
+
[Fact]
|
|
144
|
+
public async Task AnAliasWithoutATariffRecordsTokensWithZeroCostAndSaysWhy()
|
|
145
|
+
{
|
|
146
|
+
var alias = WaivedAlias();
|
|
147
|
+
var (client, _, sink) = Pipeline(
|
|
148
|
+
alias,
|
|
149
|
+
fake => fake.EnqueueTextWithUsage("ok", inputTokens: 900, outputTokens: 100));
|
|
150
|
+
|
|
151
|
+
var response = await client.GetResponseAsync(
|
|
152
|
+
[new ChatMessage(ChatRole.User, "oi")],
|
|
153
|
+
cancellationToken: TestContext.Current.CancellationToken);
|
|
154
|
+
|
|
155
|
+
Assert.Equal("ok", response.Text);
|
|
156
|
+
|
|
157
|
+
var record = Assert.Single(sink.Records);
|
|
158
|
+
|
|
159
|
+
Assert.Equal(0m, record.CostUsd); // zero, jamais nulo
|
|
160
|
+
Assert.Equal(UsageCostStatus.Waived, record.Status);
|
|
161
|
+
Assert.False(record.IsMeasuredCost); // logo, nao entra na soma
|
|
162
|
+
Assert.Equal(alias.PricingWaiver, record.PricingWaiver);
|
|
163
|
+
Assert.Equal(900, record.InputTokens); // os tokens continuam medidos
|
|
164
|
+
Assert.Equal(100, record.OutputTokens);
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/// <summary>
|
|
168
|
+
/// A regra que sustenta a anterior mora no registry (T8): um alias sem tarifa <b>e</b>
|
|
169
|
+
/// sem dispensa nao carrega. Por isso um record <c>Waived</c> sempre tem motivo — a
|
|
170
|
+
/// ausencia por esquecimento morre antes de chegar ao middleware.
|
|
171
|
+
/// </summary>
|
|
172
|
+
[Fact]
|
|
173
|
+
public void AWaivedRecordAlwaysHasAReasonBecauseTheRegistryRefusesAnAliasWithNeither()
|
|
174
|
+
{
|
|
175
|
+
var orphan = new ModelAlias("openai", "gpt-5.4", AgentApi.Responses, Pricing: null, Endpoint: null)
|
|
176
|
+
{
|
|
177
|
+
Name = "sem-decisao",
|
|
178
|
+
};
|
|
179
|
+
|
|
180
|
+
var refusal = Assert.Throws<ModelRegistryException>(() => ModelAlias.ValidatePricingDecision(orphan));
|
|
181
|
+
Assert.Contains("sem-decisao", refusal.Message, StringComparison.Ordinal);
|
|
182
|
+
|
|
183
|
+
ModelAlias.ValidatePricingDecision(WaivedAlias()); // com dispensa escrita, passa
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/// <summary>
|
|
187
|
+
/// A soma so junta o que foi medido. Uma volta tarifada mais uma dispensada somam o
|
|
188
|
+
/// custo da primeira — o zero da dispensa nao "abaixa a media" nem some do relatorio.
|
|
189
|
+
/// </summary>
|
|
190
|
+
[Fact]
|
|
191
|
+
public async Task OnlyMeasuredCostsEnterTheSum()
|
|
192
|
+
{
|
|
193
|
+
var sink = new FakeUsageCostSink();
|
|
194
|
+
var clock = new FakeClock(FixedNow);
|
|
195
|
+
|
|
196
|
+
using var paid = new FakeChatClient().EnqueueTextWithUsage("a", 1000, 200, 400);
|
|
197
|
+
using var free = new FakeChatClient().EnqueueTextWithUsage("b", 900, 100);
|
|
198
|
+
|
|
199
|
+
var paidClient = paid.AsBuilder().UseUsageCost(PricedAlias(), sink, clock).Build();
|
|
200
|
+
var freeClient = free.AsBuilder().UseUsageCost(WaivedAlias(), sink, clock).Build();
|
|
201
|
+
|
|
202
|
+
await paidClient.GetResponseAsync(
|
|
203
|
+
[new ChatMessage(ChatRole.User, "1")],
|
|
204
|
+
cancellationToken: TestContext.Current.CancellationToken);
|
|
205
|
+
await freeClient.GetResponseAsync(
|
|
206
|
+
[new ChatMessage(ChatRole.User, "2")],
|
|
207
|
+
cancellationToken: TestContext.Current.CancellationToken);
|
|
208
|
+
|
|
209
|
+
Assert.Equal(2, sink.Count);
|
|
210
|
+
Assert.Equal(0.0028m, sink.MeasuredTotalUsd);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
// ---------------------------------------------------------------------------------
|
|
214
|
+
// O terceiro zero: o provider nao reportou uso
|
|
215
|
+
// ---------------------------------------------------------------------------------
|
|
216
|
+
|
|
217
|
+
/// <summary>
|
|
218
|
+
/// Resposta sem <c>Usage</c>: a linha sai marcada como lacuna. Nao publicar nada faria o
|
|
219
|
+
/// custo desaparecer do relatorio, que e a mesma dor por outro angulo; publicar como
|
|
220
|
+
/// <c>Computed</c> mentiria que a volta custou zero.
|
|
221
|
+
/// </summary>
|
|
222
|
+
[Fact]
|
|
223
|
+
public async Task AResponseWithoutUsageIsRecordedAsAGapNotAsFree()
|
|
224
|
+
{
|
|
225
|
+
var (client, _, sink) = Pipeline(PricedAlias(), fake => fake.EnqueueText("sem contador"));
|
|
226
|
+
|
|
227
|
+
await client.GetResponseAsync(
|
|
228
|
+
[new ChatMessage(ChatRole.User, "oi")],
|
|
229
|
+
cancellationToken: TestContext.Current.CancellationToken);
|
|
230
|
+
|
|
231
|
+
var record = Assert.Single(sink.Records);
|
|
232
|
+
|
|
233
|
+
Assert.Equal(UsageCostStatus.UsageMissing, record.Status);
|
|
234
|
+
Assert.Equal(0m, record.CostUsd);
|
|
235
|
+
Assert.False(record.IsMeasuredCost);
|
|
236
|
+
Assert.Equal(0m, sink.MeasuredTotalUsd);
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/// <summary>
|
|
240
|
+
/// <c>cached > input</c> e o provider se contradizendo. A conta do T8 recusa esse par
|
|
241
|
+
/// com excecao — e recusar aqui derrubaria a resposta do usuario por causa de um
|
|
242
|
+
/// contador. O middleware degrada para lacuna declarada, sem lancar.
|
|
243
|
+
/// </summary>
|
|
244
|
+
[Fact]
|
|
245
|
+
public async Task InconsistentTokenCountsBecomeADeclaredGapInsteadOfAThrownRequest()
|
|
246
|
+
{
|
|
247
|
+
var pricing = Tariff();
|
|
248
|
+
Assert.Throws<ArgumentException>(() => pricing.ComputeCostUsd(10, 5, cachedInputTokens: 11));
|
|
249
|
+
|
|
250
|
+
var (client, _, sink) = Pipeline(
|
|
251
|
+
PricedAlias(pricing),
|
|
252
|
+
fake => fake.EnqueueTextWithUsage("ok", inputTokens: 10, outputTokens: 5, cachedInputTokens: 11));
|
|
253
|
+
|
|
254
|
+
var response = await client.GetResponseAsync(
|
|
255
|
+
[new ChatMessage(ChatRole.User, "oi")],
|
|
256
|
+
cancellationToken: TestContext.Current.CancellationToken);
|
|
257
|
+
|
|
258
|
+
Assert.Equal("ok", response.Text);
|
|
259
|
+
Assert.Equal(UsageCostStatus.UsageMissing, Assert.Single(sink.Records).Status);
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
// ---------------------------------------------------------------------------------
|
|
263
|
+
// Uma linha por volta ao modelo — inclusive no streaming
|
|
264
|
+
// ---------------------------------------------------------------------------------
|
|
265
|
+
|
|
266
|
+
/// <summary>
|
|
267
|
+
/// Duas <b>chamadas</b> a <c>GetResponseAsync</c> publicam <b>duas</b> linhas: uma por
|
|
268
|
+
/// ida ao modelo, cada uma com o seu proprio numero.
|
|
269
|
+
/// </summary>
|
|
270
|
+
/// <remarks>
|
|
271
|
+
/// O que este teste prova, e so isto: <b>uma linha por chamada</b>, sem acumulo de
|
|
272
|
+
/// estado entre chamadas. Ele NAO prova nada sobre a posicao do middleware no pipeline
|
|
273
|
+
/// — sao duas chamadas separadas, nao uma chamada com volta de tool, e aqui o
|
|
274
|
+
/// middleware e montado sozinho, sem <c>FunctionInvokingChatClient</c> nenhum. A
|
|
275
|
+
/// posicao real e o oposto do que uma leitura apressada deste teste sugeriria: o
|
|
276
|
+
/// middleware fica <b>ACIMA</b> da invocacao de funcoes, e quem prova isso e
|
|
277
|
+
/// <c>AgentFactoryTests.ThePipelineOrderIsUsageCostAboveFunctionInvocationAndTelemetryBelowIt</c>
|
|
278
|
+
/// (com <c>...SeesTheUsageOfEveryToolRoundSummedOnce</c> mostrando o uso ja somado das
|
|
279
|
+
/// duas voltas). Acima do FICC, uma conversa com tool call e <b>uma</b> chamada com o
|
|
280
|
+
/// uso ja totalizado — nao duas.
|
|
281
|
+
/// </remarks>
|
|
282
|
+
[Fact]
|
|
283
|
+
public async Task EveryTripToTheModelPublishesItsOwnLine()
|
|
284
|
+
{
|
|
285
|
+
var (client, inner, sink) = Pipeline(
|
|
286
|
+
PricedAlias(),
|
|
287
|
+
fake => fake
|
|
288
|
+
.EnqueueTextWithUsage("primeira", 1000, 200, 400)
|
|
289
|
+
.EnqueueTextWithUsage("segunda", 2000, 100, null));
|
|
290
|
+
|
|
291
|
+
await client.GetResponseAsync(
|
|
292
|
+
[new ChatMessage(ChatRole.User, "a")],
|
|
293
|
+
cancellationToken: TestContext.Current.CancellationToken);
|
|
294
|
+
await client.GetResponseAsync(
|
|
295
|
+
[new ChatMessage(ChatRole.User, "b")],
|
|
296
|
+
cancellationToken: TestContext.Current.CancellationToken);
|
|
297
|
+
|
|
298
|
+
Assert.Equal(2, inner.CallCount);
|
|
299
|
+
Assert.Equal(2, sink.Count);
|
|
300
|
+
Assert.Equal(0.0028m, sink.Records[0].CostUsd);
|
|
301
|
+
Assert.Equal(0.0035m, sink.Records[1].CostUsd); // (2000*1.25 + 100*10) / 1M
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/// <summary>
|
|
305
|
+
/// No streaming o uso chega como <c>UsageContent</c> avulso. A linha sai <b>uma</b> vez,
|
|
306
|
+
/// ao fim da enumeracao, com o mesmo numero do caminho nao-streaming.
|
|
307
|
+
/// </summary>
|
|
308
|
+
[Fact]
|
|
309
|
+
public async Task StreamingPublishesOneLineAtTheEndWithTheSameNumber()
|
|
310
|
+
{
|
|
311
|
+
var (client, _, sink) = Pipeline(
|
|
312
|
+
PricedAlias(),
|
|
313
|
+
fake => fake.EnqueueTextWithUsage("streamado", 1000, 200, 400));
|
|
314
|
+
|
|
315
|
+
var updates = 0;
|
|
316
|
+
await foreach (var _ in client.GetStreamingResponseAsync(
|
|
317
|
+
[new ChatMessage(ChatRole.User, "oi")],
|
|
318
|
+
cancellationToken: TestContext.Current.CancellationToken))
|
|
319
|
+
{
|
|
320
|
+
Assert.Equal(0, sink.Count); // nada publicado no meio da enumeracao
|
|
321
|
+
updates++;
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
Assert.True(updates > 0);
|
|
325
|
+
Assert.Equal(0.0028m, Assert.Single(sink.Records).CostUsd);
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
// ---------------------------------------------------------------------------------
|
|
329
|
+
// Higiene de construcao
|
|
330
|
+
// ---------------------------------------------------------------------------------
|
|
331
|
+
|
|
332
|
+
/// <summary>Alias ou sink nulos sao defeito de composicao e falham na construcao.</summary>
|
|
333
|
+
[Fact]
|
|
334
|
+
public void TheMiddlewareRefusesToBeBuiltWithoutAnAliasOrASink()
|
|
335
|
+
{
|
|
336
|
+
using var inner = new FakeChatClient();
|
|
337
|
+
|
|
338
|
+
Assert.Throws<ArgumentNullException>(
|
|
339
|
+
() => new UsageCostMiddleware(inner, null!, new FakeUsageCostSink()));
|
|
340
|
+
|
|
341
|
+
Assert.Throws<ArgumentNullException>(
|
|
342
|
+
() => new UsageCostMiddleware(inner, PricedAlias(), null!));
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
/// <summary>
|
|
346
|
+
/// O instante do record vem do <see cref="TimeProvider"/> injetado, nao do relogio do
|
|
347
|
+
/// processo — sem isso o campo <c>At</c> seria intestavel (standard
|
|
348
|
+
/// <c>backend-dotnet-testing</c>).
|
|
349
|
+
/// </summary>
|
|
350
|
+
[Fact]
|
|
351
|
+
public async Task TheTimestampComesFromTheInjectedClock()
|
|
352
|
+
{
|
|
353
|
+
var clock = new FakeClock(FixedNow);
|
|
354
|
+
var sink = new FakeUsageCostSink();
|
|
355
|
+
using var inner = new FakeChatClient()
|
|
356
|
+
.EnqueueTextWithUsage("a", 10, 10)
|
|
357
|
+
.EnqueueTextWithUsage("b", 10, 10);
|
|
358
|
+
|
|
359
|
+
var client = inner.AsBuilder().UseUsageCost(PricedAlias(), sink, clock).Build();
|
|
360
|
+
|
|
361
|
+
await client.GetResponseAsync(
|
|
362
|
+
[new ChatMessage(ChatRole.User, "1")],
|
|
363
|
+
cancellationToken: TestContext.Current.CancellationToken);
|
|
364
|
+
|
|
365
|
+
clock.Advance(TimeSpan.FromMinutes(5));
|
|
366
|
+
|
|
367
|
+
await client.GetResponseAsync(
|
|
368
|
+
[new ChatMessage(ChatRole.User, "2")],
|
|
369
|
+
cancellationToken: TestContext.Current.CancellationToken);
|
|
370
|
+
|
|
371
|
+
Assert.Equal(FixedNow, sink.Records[0].At);
|
|
372
|
+
Assert.Equal(FixedNow.AddMinutes(5), sink.Records[1].At);
|
|
373
|
+
}
|
|
374
|
+
}
|
package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/eval-cache/.morph-eval-cache.json
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
{
|
|
2
|
+
"pinnedAt": "2026-09-08",
|
|
3
|
+
"packages": {
|
|
4
|
+
"Microsoft.Extensions.AI": "10.9.0",
|
|
5
|
+
"Microsoft.Extensions.AI.Evaluation": "10.9.0",
|
|
6
|
+
"Microsoft.Extensions.AI.Evaluation.Quality": "10.9.0",
|
|
7
|
+
"Microsoft.Extensions.AI.Evaluation.Reporting": "10.9.0"
|
|
8
|
+
},
|
|
9
|
+
"provider": "morph-fake",
|
|
10
|
+
"model": "morph-kit-exemplar",
|
|
11
|
+
"writtenBy": "morph-eval-fixture"
|
|
12
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"scenarioName": "orcamento.acme.a-resposta-estruturada",
|
|
3
|
+
"iterationName": "1",
|
|
4
|
+
"key": "CF70A7CE27BBD66AC1E097FB07773A83302651F432FAC98DA349B93B2AC2204DB504344EF1C220390AE6504FFA3BCB81",
|
|
5
|
+
"creation": "2026-09-09T04:54:34.7288625+00:00",
|
|
6
|
+
"expiration": "2036-09-06T04:54:34.7288625+00:00"
|
|
7
|
+
}
|