@polymorphism-tech/morph-spec 8.38.0 → 8.40.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +19 -8
- package/README.md +2 -2
- package/bin/morph-spec.js +60 -43
- package/claude-plugin.json +1 -1
- package/docs/CHEATSHEET.md +1 -1
- package/docs/QUICKSTART.md +1 -1
- package/framework/CLAUDE.md +3 -2
- package/framework/MORPH.md +1 -1
- package/framework/agents.json +36 -25
- package/framework/docs/CLI.md +2 -2
- package/framework/evals/rubrics/code-quality.md +4 -4
- package/framework/evals/rubrics/condensed.md +2 -2
- package/framework/mcp/registry.json +33 -2
- package/framework/rules/infrastructure-standards.md +0 -1
- package/framework/rules/morph-workflow.md +9 -4
- package/framework/skills/core/create-umbraco-lp/SKILL.md +1 -1
- package/framework/skills/core/migrate-lp-to-umbraco/SKILL.md +1 -1
- package/framework/skills/core/morph-apply/SKILL.md +5 -1
- package/framework/skills/core/morph-brainstorming/SKILL.md +1 -1
- package/framework/skills/core/morph-eval/SKILL.md +1 -1
- package/framework/skills/core/morph-hotfix/SKILL.md +1 -1
- package/framework/skills/core/morph-implement/SKILL.md +6 -1
- package/framework/skills/core/morph-media/SKILL.md +1 -1
- package/framework/skills/core/morph-plan/SKILL.md +4 -3
- package/framework/skills/core/morph-proposal/SKILL.md +1 -1
- package/framework/skills/core/morph-review/SKILL.md +1 -1
- package/framework/skills/core/morph-stack-scan/SKILL.md +1 -1
- package/framework/skills/core/morph-standards/SKILL.md +1 -1
- package/framework/skills/core/morph-uiux/SKILL.md +6 -5
- package/framework/standards/STANDARDS.json +3 -3
- package/framework/standards/ai-agents/context-providers.md +16 -26
- package/framework/standards/ai-agents/durable-workflows-hitl.md +15 -21
- package/framework/standards/ai-agents/modalities-image-gen.md +21 -25
- package/framework/standards/ai-agents/modalities-speech-to-text.md +4 -15
- package/framework/standards/ai-agents/modalities-text-to-speech.md +2 -11
- package/framework/standards/ai-agents/modalities-vision.md +1 -18
- package/framework/standards/ai-agents/prompt-sources.md +0 -4
- package/framework/standards/architecture/code-comments.md +52 -16
- package/framework/standards/backend/integrations/neon-auth/neon-auth.md +63 -50
- package/framework/standards/frontend/design-system/ai-image-generation.md +5 -5
- package/framework/standards/integration/mcp/mcp-tools.md +48 -117
- package/framework/templates/code/dotnet/abstractions/result.cs +0 -2
- package/framework/templates/code/dotnet/ai-agents/AgentBootstrap.cs.template +26 -39
- package/framework/templates/code/dotnet/ai-agents/EmbeddingService.cs.template +28 -68
- package/framework/templates/code/dotnet/ai-agents/ModelRegistry.cs.template +99 -117
- package/framework/templates/code/dotnet/ai-agents/PromptRepository.cs.template +2 -44
- package/framework/templates/code/dotnet/ai-agents/ResilientToolMiddleware.cs.template +40 -61
- package/framework/templates/code/dotnet/ai-agents/VectorSearchTool.cs.template +54 -107
- package/framework/templates/code/dotnet/ai-agents/model-registry.json.template +53 -58
- package/framework/templates/code/dotnet/contracts/contracts-vsa.cs +192 -281
- package/framework/templates/code/dotnet/database/migration.cs +49 -83
- package/framework/templates/code/dotnet/database/vector-entity.cs +2 -46
- package/framework/templates/code/dotnet/extensions/result-extensions.cs +0 -2
- package/framework/templates/code/dotnet/feature/validator.cs.hbs +11 -15
- package/framework/templates/code/dotnet/jobs/agent.cs +3 -51
- package/framework/templates/code/dotnet/jobs/job.cs +2 -97
- package/framework/templates/code/dotnet/test.cs +2 -87
- package/framework/templates/code/typescript/contracts.ts +140 -167
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Agents/AgentApi.cs +1 -48
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Agents/AgentFactory.cs +3 -79
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Agents/AgentFactoryOptions.cs +1 -70
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Agents/AgentSpec.cs +1 -47
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Agents/AgentSpecOptions.cs +2 -99
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/AiKitAssemblyMarker.cs +0 -6
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Conversation/ConversationStore.cs +3 -88
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Conversation/IConversationStore.cs +1 -41
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Conversation/InMemoryConversationStore.cs +0 -21
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Evals/CountingChatClient.cs +1 -25
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Evals/EvalAssertion.cs +2 -63
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Evals/EvalCacheMissException.cs +0 -30
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Evals/EvalCacheOptions.cs +1 -52
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Evals/EvalCacheStamp.cs +3 -78
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Evals/EvalFixture.cs +4 -109
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Evals/EvalMode.cs +0 -22
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Evals/EvalQualityEvaluators.cs +1 -36
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Evals/EvalScenario.cs +2 -57
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Evals/EvalSummary.cs +0 -75
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Evals/OfflineChatClient.cs +2 -59
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Evals/ShortPathResponseCache.cs +4 -111
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/IMediaGenerator.cs +0 -34
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/IMediaProvider.cs +1 -57
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MediaArtifact.cs +0 -19
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MediaDisclosure.cs +2 -74
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MediaGenerator.cs +2 -33
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MediaIdempotencyKey.cs +1 -43
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MediaJob.cs +1 -96
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MediaJobId.cs +0 -13
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MediaJobState.cs +0 -27
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MediaPricing.cs +0 -33
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MediaProvenance.cs +0 -31
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MediaRequest.cs +3 -75
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MediaResult.cs +0 -42
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MediaReview.cs +0 -39
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MediaSeams.cs +1 -55
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MediaStore.cs +3 -59
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/MorphMedia.cs +0 -46
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/Providers/GoogleImageProvider.cs +3 -28
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/Providers/OpenAIImageProvider.cs +7 -61
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Media/Providers/VeoVideoProvider.cs +2 -37
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Middleware/IUsageCostSink.cs +1 -23
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Middleware/UsageCostMiddleware.cs +2 -82
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Middleware/UsageCostRecord.cs +1 -68
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Observability/MorphOpenTelemetryExtensions.cs +1 -62
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Observability/MorphTelemetryOptions.cs +2 -62
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Providers/IProviderClientFactory.cs +1 -25
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Providers/ModelAlias.cs +0 -124
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Providers/ModelRegistry.cs +1 -73
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Rag/DeltaIngest.cs +1 -66
- package/framework/templates/dotnet/ai-kit/src/Morph.AiKit/Rag/DeltaPlan.cs +0 -41
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/AgentFactoryTests.cs +2 -112
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/AgentSpecTests.cs +2 -43
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/ConversationStoreTests.cs +5 -110
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/DeltaIngestTests.cs +7 -103
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Evals/EvalAssertionTests.cs +1 -42
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Evals/EvalCacheMissTests.cs +1 -30
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Evals/EvalCacheStampTests.cs +1 -19
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Evals/EvalFailuresTests.cs +0 -48
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Evals/EvalFixtureTests.cs +1 -29
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Evals/EvalRefreshTests.cs +2 -38
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Evals/EvalWorkspace.cs +1 -14
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Evals/KitEvals.cs +4 -69
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Evals/OfflineChatClientTests.cs +1 -10
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Fakes/FakeChatClient.cs +1 -57
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Fakes/FakeChatClientTests.cs +0 -26
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Fakes/FakeClock.cs +1 -19
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Fakes/FakeMediaProvider.cs +0 -32
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Fakes/FakeProviderClientFactory.cs +0 -14
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Fakes/FakeUsageCostSink.cs +0 -24
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Fakes/InMemoryMediaStore.cs +0 -32
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/MediaAccessibilityTests.cs +0 -37
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/MediaAliasIsNotAnAgentTests.cs +0 -24
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/MediaDisclosureTests.cs +0 -78
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/MediaGeneratorTests.cs +0 -53
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/MediaIdempotencyKeyTests.cs +0 -44
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/MediaJobStateTests.cs +0 -44
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/MediaReviewTests.cs +0 -19
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/MediaStoreTests.cs +0 -24
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/ModelRegistryTests.cs +6 -48
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/SkeletonSmokeTests.cs +1 -13
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/TelemetryTests.cs +2 -71
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/UsageCostMiddlewareTests.cs +5 -100
- package/framework/templates/frontend/nextjs/Dockerfile.nextjs.hbs +39 -43
- package/framework/templates/frontend/nextjs/env.mjs.hbs +29 -32
- package/framework/templates/frontend/nextjs/page.tsx.hbs +21 -22
- package/framework/templates/frontend/nextjs/use-feature.ts.hbs +42 -54
- package/framework/templates/integrations/asaas-client.cs +332 -387
- package/framework/templates/integrations/asaas-webhook.cs +290 -351
- package/framework/templates/integrations/neon-auth-config.cs +9 -105
- package/framework/templates/project/validate.js +97 -120
- package/package.json +2 -2
- package/src/commands/dashboard/dashboard.js +80 -53
- package/src/commands/mcp/mcp-setup.js +37 -9
- package/src/commands/project/doctor.js +3 -5
- package/src/commands/project/init.js +6 -3
- package/src/commands/project/install-plugin-cmd.js +1 -1
- package/src/commands/project/update.js +5 -0
- package/src/dashboard/app/App.jsx +80 -16
- package/src/dashboard/app/components/ActivityFeed.jsx +172 -0
- package/src/dashboard/app/components/CommandPalette.jsx +155 -0
- package/src/dashboard/app/components/DetailPanel.jsx +47 -11
- package/src/dashboard/app/components/EntityLink.jsx +55 -0
- package/src/dashboard/app/components/ErrorBoundary.jsx +54 -0
- package/src/dashboard/app/components/FlowGraph.jsx +19 -17
- package/src/dashboard/app/components/KnowledgeMarkdown.jsx +53 -0
- package/src/dashboard/app/components/Markdown.jsx +47 -0
- package/src/dashboard/app/components/PhaseRail.jsx +111 -0
- package/src/dashboard/app/components/SearchFilter.jsx +40 -22
- package/src/dashboard/app/components/Sidebar.jsx +146 -37
- package/src/dashboard/app/hooks/useMorphQuery.js +54 -0
- package/src/dashboard/app/hooks/usePref.js +38 -0
- package/src/dashboard/app/lib/format.js +69 -0
- package/src/dashboard/app/lib/phases.js +56 -0
- package/src/dashboard/app/lib/project.js +50 -0
- package/src/dashboard/app/pages/Activity.jsx +24 -0
- package/src/dashboard/app/pages/AgentMap.jsx +371 -370
- package/src/dashboard/app/pages/FeatureDetail.jsx +604 -0
- package/src/dashboard/app/pages/Features.jsx +400 -0
- package/src/dashboard/app/pages/Guide.jsx +146 -0
- package/src/dashboard/app/pages/HookInspector.jsx +357 -322
- package/src/dashboard/app/pages/PhasePipeline.jsx +277 -408
- package/src/dashboard/app/pages/Portfolio.jsx +156 -0
- package/src/dashboard/app/pages/Rubrics.jsx +101 -0
- package/src/dashboard/app/pages/RulesMap.jsx +300 -273
- package/src/dashboard/app/pages/SkillsLibrary.jsx +483 -366
- package/src/dashboard/app/pages/Standards.jsx +407 -332
- package/src/dashboard/app/styles/index.css +187 -47
- package/src/dashboard/index.html +14 -12
- package/src/dashboard/package.json +32 -24
- package/src/dashboard/scanner/activity-scanner.js +351 -0
- package/src/dashboard/scanner/agents-scanner.js +59 -1
- package/src/dashboard/scanner/docs-scanner.js +59 -0
- package/src/dashboard/scanner/feature-status.js +10 -0
- package/src/dashboard/scanner/features-scanner.js +360 -0
- package/src/dashboard/scanner/index.js +14 -7
- package/src/dashboard/scanner/phases-scanner.js +14 -15
- package/src/dashboard/scanner/portfolio.js +153 -0
- package/src/dashboard/scanner/project-api.js +195 -0
- package/src/dashboard/scanner/rules-scanner.js +42 -37
- package/src/dashboard/scanner/skills-scanner.js +105 -43
- package/src/dashboard/scanner/standards-scanner.js +2 -0
- package/src/dashboard/vite-plugin-morph-data.js +134 -17
- package/src/lib/detectors/claude-config-detector.js +2 -7
- package/src/lib/detectors/index.js +5 -1
- package/src/lib/evaluator-report.js +22 -0
- package/src/lib/generators/settings-generator.js +208 -209
- package/src/lib/installers/mcp-installer.js +724 -719
- package/src/lib/installers/memory-installer.js +1 -2
- package/src/lib/projects/registry.js +154 -0
- package/src/lib/validators/code-budget/code-budget-validator.js +32 -4
- package/src/scripts/install-plugin.js +1 -1
- package/src/scripts/setup-infra.js +2 -4
- package/src/utils/agents-installer.js +57 -12
- package/src/utils/banner.js +1 -1
- package/src/commands/index.js +0 -13
- package/src/commands/project/checkpoint.js +0 -209
- package/src/core/state/phase-state-machine.js +0 -197
- package/src/dashboard/app/hooks/useDetailPanel.js +0 -15
- package/src/dashboard/app/pages/LiveState.jsx +0 -206
- package/src/dashboard/scanner/squads-scanner.js +0 -13
- package/src/dashboard/scanner/state-scanner.js +0 -85
- package/src/lib/generators/index.js +0 -10
- package/src/lib/hooks/hook-executor.js +0 -167
- package/src/lib/validators/checkpoint-persistence.js +0 -39
- package/src/lib/validators/shared/emit-validator-dispatch.js +0 -14
- package/src/lib/validators/shared/index.js +0 -12
- package/src/lib/validators/shared/issue-counter.js +0 -14
- package/src/lib/validators/shared/result-formatter.js +0 -124
|
@@ -3,47 +3,23 @@ using Xunit;
|
|
|
3
3
|
|
|
4
4
|
namespace Morph.AiKit.Tests;
|
|
5
5
|
|
|
6
|
-
/// <summary>
|
|
7
|
-
/// Canarios do <see cref="DeltaIngest"/> (T14). A afirmacao central e uma so, e ela e
|
|
8
|
-
/// <b>demonstrada</b> contra um acervo, nao afirmada: <b>ingerir a mesma coisa duas vezes nao
|
|
9
|
-
/// duplica nada, e mudar um campo atualiza em vez de acrescentar</b>.
|
|
10
|
-
/// </summary>
|
|
11
6
|
public sealed class DeltaIngestTests
|
|
12
7
|
{
|
|
13
|
-
/// <summary>Um documento de acervo: identidade estavel + conteudo que muda.</summary>
|
|
14
8
|
private sealed record Doc(string Slug, string Body);
|
|
15
9
|
|
|
16
10
|
private static string Key(Doc d) => d.Slug;
|
|
17
11
|
|
|
18
|
-
/// <summary>
|
|
19
|
-
/// Hash de conteudo do teste. Nao precisa ser criptografico — precisa mudar quando o
|
|
20
|
-
/// conteudo muda, que e a unica propriedade que o <see cref="DeltaIngest"/> usa.
|
|
21
|
-
/// </summary>
|
|
22
12
|
private static string Hash(Doc d) => $"h:{d.Body.Length}:{d.Body.GetHashCode(StringComparison.Ordinal)}";
|
|
23
13
|
|
|
24
|
-
/// <summary>
|
|
25
|
-
/// Um acervo minusculo que <b>aplica</b> o plano — e a unica forma de a idempotencia ser
|
|
26
|
-
/// medida no que ficou guardado, em vez de nos nomes das listas do plano.
|
|
27
|
-
/// </summary>
|
|
28
|
-
/// <remarks>
|
|
29
|
-
/// Modela um acervo de verdade: <c>Added</c> vira <b>INSERT</b> numa lista de linhas,
|
|
30
|
-
/// <c>Updated</c> vira UPDATE da linha existente e <c>Removed</c> vira DELETE. O indice de
|
|
31
|
-
/// hashes e mantido a parte, como um indice de banco. Isso importa: se o acervo fizesse
|
|
32
|
-
/// upsert de tudo, uma re-ingestao que classificasse errado sairia invisivel — aqui ela
|
|
33
|
-
/// aparece como linha duplicada, que e o defeito real que esta task combate.
|
|
34
|
-
/// </remarks>
|
|
35
14
|
private sealed class Corpus
|
|
36
15
|
{
|
|
37
16
|
private readonly List<Doc> _rows = [];
|
|
38
17
|
private readonly Dictionary<string, string> _index = new(StringComparer.Ordinal);
|
|
39
18
|
|
|
40
|
-
/// <summary>Quantos embeddings foram pagos ao todo. Nao sobe numa ingestao no-op.</summary>
|
|
41
19
|
public int EmbeddingCalls { get; private set; }
|
|
42
20
|
|
|
43
|
-
/// <summary>As linhas guardadas, na ordem em que entraram.</summary>
|
|
44
21
|
public IReadOnlyList<Doc> Rows => _rows;
|
|
45
22
|
|
|
46
|
-
/// <summary>O indice lookup-key para hash, que alimenta o proximo plano.</summary>
|
|
47
23
|
public IReadOnlyDictionary<string, string> Hashes => _index;
|
|
48
24
|
|
|
49
25
|
public DeltaPlan<Doc> Ingest(IEnumerable<Doc> incoming)
|
|
@@ -52,21 +28,21 @@ public sealed class DeltaIngestTests
|
|
|
52
28
|
|
|
53
29
|
foreach (var doc in plan.Added)
|
|
54
30
|
{
|
|
55
|
-
_rows.Add(doc);
|
|
31
|
+
_rows.Add(doc);
|
|
56
32
|
_index[doc.Slug] = Hash(doc);
|
|
57
33
|
EmbeddingCalls++;
|
|
58
34
|
}
|
|
59
35
|
|
|
60
36
|
foreach (var doc in plan.Updated)
|
|
61
37
|
{
|
|
62
|
-
_rows[_rows.FindIndex(r => r.Slug == doc.Slug)] = doc;
|
|
38
|
+
_rows[_rows.FindIndex(r => r.Slug == doc.Slug)] = doc;
|
|
63
39
|
_index[doc.Slug] = Hash(doc);
|
|
64
40
|
EmbeddingCalls++;
|
|
65
41
|
}
|
|
66
42
|
|
|
67
43
|
foreach (var slug in plan.Removed)
|
|
68
44
|
{
|
|
69
|
-
_rows.RemoveAll(r => r.Slug == slug);
|
|
45
|
+
_rows.RemoveAll(r => r.Slug == slug);
|
|
70
46
|
_index.Remove(slug);
|
|
71
47
|
}
|
|
72
48
|
|
|
@@ -81,14 +57,6 @@ public sealed class DeltaIngestTests
|
|
|
81
57
|
new("garantia", "Garantia de 12 meses."),
|
|
82
58
|
];
|
|
83
59
|
|
|
84
|
-
// ---------------------------------------------------------------------------------
|
|
85
|
-
// doneCriteria: os quatro baldes
|
|
86
|
-
// ---------------------------------------------------------------------------------
|
|
87
|
-
|
|
88
|
-
/// <summary>
|
|
89
|
-
/// Os quatro baldes numa unica passada: um item novo, um com hash mudado, um com hash
|
|
90
|
-
/// igual e uma chave que sumiu da origem.
|
|
91
|
-
/// </summary>
|
|
92
60
|
[Fact]
|
|
93
61
|
public void TheFourBucketsAreSortedInASinglePass()
|
|
94
62
|
{
|
|
@@ -109,7 +77,6 @@ public sealed class DeltaIngestTests
|
|
|
109
77
|
Assert.Equal(3, plan.WriteCount);
|
|
110
78
|
}
|
|
111
79
|
|
|
112
|
-
/// <summary>Acervo vazio: tudo e novo, nada e removido.</summary>
|
|
113
80
|
[Fact]
|
|
114
81
|
public void AnEmptyCorpusMakesEverythingNew()
|
|
115
82
|
{
|
|
@@ -121,12 +88,6 @@ public sealed class DeltaIngestTests
|
|
|
121
88
|
Assert.Empty(plan.Removed);
|
|
122
89
|
}
|
|
123
90
|
|
|
124
|
-
/// <summary>
|
|
125
|
-
/// Remessa vazia com acervo cheio: <b>tudo</b> sai em <c>Removed</c>. E o comportamento
|
|
126
|
-
/// correto e tambem o mais perigoso — e por isso o plano e um plano: quem decide se
|
|
127
|
-
/// aplica um <c>Removed</c> de 100% e o projeto, que sabe se a origem realmente esvaziou
|
|
128
|
-
/// ou se o crawler falhou.
|
|
129
|
-
/// </summary>
|
|
130
91
|
[Fact]
|
|
131
92
|
public void AnEmptyShipmentAgainstAFullCorpusRemovesEverything()
|
|
132
93
|
{
|
|
@@ -140,15 +101,6 @@ public sealed class DeltaIngestTests
|
|
|
140
101
|
Assert.Empty(plan.Updated);
|
|
141
102
|
}
|
|
142
103
|
|
|
143
|
-
// ---------------------------------------------------------------------------------
|
|
144
|
-
// doneCriteria: IDEMPOTENCIA, medida no acervo
|
|
145
|
-
// ---------------------------------------------------------------------------------
|
|
146
|
-
|
|
147
|
-
/// <summary>
|
|
148
|
-
/// <b>A prova principal.</b> Ingerir a mesma remessa duas vezes: na segunda o plano e
|
|
149
|
-
/// vazio nos tres baldes de escrita, o acervo tem exatamente os mesmos documentos e
|
|
150
|
-
/// <b>nenhum embedding a mais foi pago</b>.
|
|
151
|
-
/// </summary>
|
|
152
104
|
[Fact]
|
|
153
105
|
public void IngestingTheSameShipmentTwiceLeavesTheCorpusUntouched()
|
|
154
106
|
{
|
|
@@ -168,21 +120,14 @@ public sealed class DeltaIngestTests
|
|
|
168
120
|
Assert.Equal(0, second.WriteCount);
|
|
169
121
|
Assert.Equal(3, second.Unchanged.Count);
|
|
170
122
|
|
|
171
|
-
|
|
172
|
-
Assert.Equal(3, corpus.Rows.Count); // <- a linha que morre se a mesma coisa duplicar
|
|
123
|
+
Assert.Equal(3, corpus.Rows.Count);
|
|
173
124
|
Assert.Equal(before, corpus.Rows.OrderBy(d => d.Slug, StringComparer.Ordinal));
|
|
174
125
|
Assert.Equal(3, corpus.EmbeddingCalls);
|
|
175
126
|
|
|
176
|
-
// E uma terceira, para que "duas vezes" nao pareca coincidencia.
|
|
177
127
|
Assert.True(corpus.Ingest(shipment).IsNoOp);
|
|
178
128
|
Assert.Equal(3, corpus.EmbeddingCalls);
|
|
179
129
|
}
|
|
180
130
|
|
|
181
|
-
/// <summary>
|
|
182
|
-
/// <b>Mudar um campo atualiza, nao acrescenta.</b> A contagem do acervo nao sobe, o slug
|
|
183
|
-
/// continua unico e o corpo e o novo — que e a diferenca entre uma ingestao delta e a
|
|
184
|
-
/// ingestao apaga-tudo que este tipo existe para substituir.
|
|
185
|
-
/// </summary>
|
|
186
131
|
[Fact]
|
|
187
132
|
public void ChangingOneFieldUpdatesInPlaceInsteadOfAppending()
|
|
188
133
|
{
|
|
@@ -192,7 +137,7 @@ public sealed class DeltaIngestTests
|
|
|
192
137
|
Doc[] revised =
|
|
193
138
|
[
|
|
194
139
|
new("politica-troca", "Trocas em ate 7 dias."),
|
|
195
|
-
new("prazo-entrega", "Entrega em 1 dia util."),
|
|
140
|
+
new("prazo-entrega", "Entrega em 1 dia util."),
|
|
196
141
|
new("garantia", "Garantia de 12 meses."),
|
|
197
142
|
];
|
|
198
143
|
|
|
@@ -203,22 +148,15 @@ public sealed class DeltaIngestTests
|
|
|
203
148
|
Assert.Empty(plan.Removed);
|
|
204
149
|
Assert.Equal(2, plan.Unchanged.Count);
|
|
205
150
|
|
|
206
|
-
Assert.Equal(3, corpus.Rows.Count);
|
|
207
|
-
Assert.Single(corpus.Rows, d => d.Slug == "prazo-entrega");
|
|
151
|
+
Assert.Equal(3, corpus.Rows.Count);
|
|
152
|
+
Assert.Single(corpus.Rows, d => d.Slug == "prazo-entrega");
|
|
208
153
|
Assert.Equal("Entrega em 1 dia util.", corpus.Rows.Single(d => d.Slug == "prazo-entrega").Body);
|
|
209
154
|
Assert.Equal(4, corpus.EmbeddingCalls); // 3 iniciais + 1 mudanca
|
|
210
155
|
|
|
211
|
-
// E a re-ingestao da remessa revisada volta a ser no-op.
|
|
212
156
|
Assert.True(corpus.Ingest(revised).IsNoOp);
|
|
213
157
|
Assert.Equal(4, corpus.EmbeddingCalls);
|
|
214
158
|
}
|
|
215
159
|
|
|
216
|
-
/// <summary>
|
|
217
|
-
/// A mesma afirmacao dita como <b>contagem de linhas</b>, e antes de qualquer outra
|
|
218
|
-
/// asserçao: re-ingerir nao insere uma segunda copia. E a linha que fica vermelha com
|
|
219
|
-
/// "6" no lugar de "3" quando a comparacao por lookup-key se quebra — a duplicacao
|
|
220
|
-
/// literal, e nao um nome de balde trocado.
|
|
221
|
-
/// </summary>
|
|
222
160
|
[Fact]
|
|
223
161
|
public void ReIngestingDoesNotInsertASecondCopyOfTheSameDocument()
|
|
224
162
|
{
|
|
@@ -233,10 +171,6 @@ public sealed class DeltaIngestTests
|
|
|
233
171
|
Assert.Equal(3, corpus.EmbeddingCalls);
|
|
234
172
|
}
|
|
235
173
|
|
|
236
|
-
/// <summary>
|
|
237
|
-
/// Idempotencia sobrevive a reordenacao: a mesma remessa embaralhada continua sendo
|
|
238
|
-
/// no-op. A identidade e a chave, nunca a posicao na lista.
|
|
239
|
-
/// </summary>
|
|
240
174
|
[Fact]
|
|
241
175
|
public void OrderDoesNotMakeAShipmentLookDifferent()
|
|
242
176
|
{
|
|
@@ -249,15 +183,6 @@ public sealed class DeltaIngestTests
|
|
|
249
183
|
Assert.Equal(3, corpus.EmbeddingCalls);
|
|
250
184
|
}
|
|
251
185
|
|
|
252
|
-
// ---------------------------------------------------------------------------------
|
|
253
|
-
// doneCriteria: chave duplicada falha de forma explicita
|
|
254
|
-
// ---------------------------------------------------------------------------------
|
|
255
|
-
|
|
256
|
-
/// <summary>
|
|
257
|
-
/// Chave repetida derruba o plano dizendo a chave <b>e as duas posicoes</b>. Ficar com o
|
|
258
|
-
/// ultimo gravaria um dos dois ao acaso; deduplicar sozinho esconderia o defeito da
|
|
259
|
-
/// origem, que e onde ele tem conserto.
|
|
260
|
-
/// </summary>
|
|
261
186
|
[Fact]
|
|
262
187
|
public void ADuplicateLookupKeyFailsNamingTheKeyAndBothPositions()
|
|
263
188
|
{
|
|
@@ -277,7 +202,6 @@ public sealed class DeltaIngestTests
|
|
|
277
202
|
Assert.Contains("deixa de ser deterministico", failure.Message, StringComparison.Ordinal);
|
|
278
203
|
}
|
|
279
204
|
|
|
280
|
-
/// <summary>Chave em branco colapsaria itens distintos num so — e recusada.</summary>
|
|
281
205
|
[Theory]
|
|
282
206
|
[InlineData("")]
|
|
283
207
|
[InlineData(" ")]
|
|
@@ -289,10 +213,6 @@ public sealed class DeltaIngestTests
|
|
|
289
213
|
Assert.Contains("posicao 0", failure.Message, StringComparison.Ordinal);
|
|
290
214
|
}
|
|
291
215
|
|
|
292
|
-
/// <summary>
|
|
293
|
-
/// Hash em branco e recusado: sem versao nao ha como separar "nao mudou" de "mudou", e
|
|
294
|
-
/// tratar a ausencia como "mudou" reintroduz o re-embedding de tudo — o defeito original.
|
|
295
|
-
/// </summary>
|
|
296
216
|
[Fact]
|
|
297
217
|
public void ABlankContentHashIsRefusedInsteadOfMeaningChanged()
|
|
298
218
|
{
|
|
@@ -302,7 +222,6 @@ public sealed class DeltaIngestTests
|
|
|
302
222
|
Assert.Contains("politica-troca", failure.Message, StringComparison.Ordinal);
|
|
303
223
|
}
|
|
304
224
|
|
|
305
|
-
/// <summary>Argumento nulo e defeito de chamada, e falha na chamada.</summary>
|
|
306
225
|
[Fact]
|
|
307
226
|
public void TheRequiredArgumentsAreNotOptional()
|
|
308
227
|
{
|
|
@@ -314,15 +233,6 @@ public sealed class DeltaIngestTests
|
|
|
314
233
|
Assert.Throws<ArgumentNullException>(() => DeltaIngest.Plan(ThreeDocs(), Key, Hash, null!));
|
|
315
234
|
}
|
|
316
235
|
|
|
317
|
-
// ---------------------------------------------------------------------------------
|
|
318
|
-
// Identidade e versao sao perguntas diferentes
|
|
319
|
-
// ---------------------------------------------------------------------------------
|
|
320
|
-
|
|
321
|
-
/// <summary>
|
|
322
|
-
/// A comparacao de chave e <b>ordinal</b>: <c>Politica</c> e <c>politica</c> sao dois
|
|
323
|
-
/// documentos. Deixar a cultura decidir faria o mesmo corpus produzir planos diferentes
|
|
324
|
-
/// conforme a maquina que roda a ingestao.
|
|
325
|
-
/// </summary>
|
|
326
236
|
[Fact]
|
|
327
237
|
public void KeysAreComparedOrdinallySoThePlanDoesNotDependOnTheMachine()
|
|
328
238
|
{
|
|
@@ -341,11 +251,6 @@ public sealed class DeltaIngestTests
|
|
|
341
251
|
Assert.Equal(["politica-troca"], plan.Removed);
|
|
342
252
|
}
|
|
343
253
|
|
|
344
|
-
/// <summary>
|
|
345
|
-
/// O plano nao le o corpo do item: um documento cuja chave e hash nao mudaram e
|
|
346
|
-
/// <c>Unchanged</c> ainda que o objeto seja outra instancia. E o que permite ao acervo
|
|
347
|
-
/// reprocessar so o que mudou de verdade.
|
|
348
|
-
/// </summary>
|
|
349
254
|
[Fact]
|
|
350
255
|
public void EqualityIsDecidedByKeyAndHashNotByInstance()
|
|
351
256
|
{
|
|
@@ -357,7 +262,6 @@ public sealed class DeltaIngestTests
|
|
|
357
262
|
Assert.True(corpus.Ingest(reconstructed).IsNoOp);
|
|
358
263
|
}
|
|
359
264
|
|
|
360
|
-
/// <summary>A remessa e enumerada uma unica vez — importa quando ela vem de um stream.</summary>
|
|
361
265
|
[Fact]
|
|
362
266
|
public void TheShipmentIsEnumeratedExactlyOnce()
|
|
363
267
|
{
|
package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Evals/EvalAssertionTests.cs
CHANGED
|
@@ -4,35 +4,11 @@ using Xunit;
|
|
|
4
4
|
|
|
5
5
|
namespace Morph.AiKit.Tests.Evals;
|
|
6
6
|
|
|
7
|
-
/// <summary>
|
|
8
|
-
/// A REGUA, testada contra a resposta ERRADA.
|
|
9
|
-
/// </summary>
|
|
10
|
-
/// <remarks>
|
|
11
|
-
/// <para>
|
|
12
|
-
/// Este arquivo nasceu de um achado do Gate 3: das seis <see cref="EvalAssertionKind"/>,
|
|
13
|
-
/// afrouxar CINCO para sempre-passar nao matava um unico teste. A suite continuava
|
|
14
|
-
/// "152 total, 0 falhou" com o motor de assercao praticamente inerte — e duas das cinco
|
|
15
|
-
/// (<c>NoPii</c> e <c>JsonHasKey</c>) sao usadas pelo dataset commitado, ou seja, os quatro
|
|
16
|
-
/// cenarios exemplares dependiam de reguas que nunca haviam sido provadas.
|
|
17
|
-
/// </para>
|
|
18
|
-
/// <para>
|
|
19
|
-
/// A causa e a mesma de todo canario inerte desta base: testava-se que a assercao APROVA o
|
|
20
|
-
/// que deve aprovar, nunca que ela REPROVA o que deve reprovar. Uma regua so separa duas
|
|
21
|
-
/// coisas se for exercitada dos DOIS lados — e a que so viu o lado bom e indistinguivel de
|
|
22
|
-
/// <c>return true</c>.
|
|
23
|
-
/// </para>
|
|
24
|
-
/// <para>
|
|
25
|
-
/// Por isso cada teste aqui carrega o PAR: a resposta que passa e a resposta que deveria
|
|
26
|
-
/// reprovar. Afrouxar qualquer <see cref="EvalAssertionKind"/> mata o teste que a nomeia.
|
|
27
|
-
/// </para>
|
|
28
|
-
/// </remarks>
|
|
29
7
|
public sealed class EvalAssertionTests
|
|
30
8
|
{
|
|
31
|
-
/// <summary>Resposta so de texto — a superficie que as assercoes de texto leem.</summary>
|
|
32
9
|
private static ChatResponse Text(string text)
|
|
33
10
|
=> new(new ChatMessage(ChatRole.Assistant, text));
|
|
34
11
|
|
|
35
|
-
/// <summary>Resposta que escolheu uma ferramenta, com argumentos.</summary>
|
|
36
12
|
private static ChatResponse Tool(string name, params (string Key, object? Value)[] args)
|
|
37
13
|
{
|
|
38
14
|
var arguments = new Dictionary<string, object?>(StringComparer.Ordinal);
|
|
@@ -45,8 +21,6 @@ public sealed class EvalAssertionTests
|
|
|
45
21
|
return new ChatResponse(message);
|
|
46
22
|
}
|
|
47
23
|
|
|
48
|
-
// ---------------------------------------------------------------- Contains
|
|
49
|
-
|
|
50
24
|
[Fact]
|
|
51
25
|
public void ContainsSeparatesTheAnswerThatCarriesTheTextFromTheOneThatDoesNot()
|
|
52
26
|
{
|
|
@@ -68,8 +42,6 @@ public sealed class EvalAssertionTests
|
|
|
68
42
|
Assert.False(assertion.Evaluate(Text("ola")).Passed);
|
|
69
43
|
}
|
|
70
44
|
|
|
71
|
-
// ------------------------------------------------------------- NotContains
|
|
72
|
-
|
|
73
45
|
[Fact]
|
|
74
46
|
public void NotContainsSeparatesTheCleanAnswerFromTheOneThatLeaksTheForbiddenText()
|
|
75
47
|
{
|
|
@@ -82,8 +54,6 @@ public sealed class EvalAssertionTests
|
|
|
82
54
|
Assert.Contains("desconto", reproved.Reason, StringComparison.Ordinal);
|
|
83
55
|
}
|
|
84
56
|
|
|
85
|
-
// -------------------------------------------------------------- JsonHasKey
|
|
86
|
-
|
|
87
57
|
[Fact]
|
|
88
58
|
public void JsonHasKeySeparatesTheObjectThatCarriesTheKeyFromTheOneThatDoesNot()
|
|
89
59
|
{
|
|
@@ -109,15 +79,12 @@ public sealed class EvalAssertionTests
|
|
|
109
79
|
[Fact]
|
|
110
80
|
public void JsonHasKeyReprovesAJsonArrayEvenWhenItParses()
|
|
111
81
|
{
|
|
112
|
-
// Um array
|
|
113
|
-
// aprovaria uma resposta em que a chave nao pode nem existir.
|
|
82
|
+
// Um array e JSON valido: a regua tem de cobrar a chave, nao so o parse.
|
|
114
83
|
var assertion = new EvalAssertion { Name = "tem-total", Kind = EvalAssertionKind.JsonHasKey, Target = "total" };
|
|
115
84
|
|
|
116
85
|
Assert.False(assertion.Evaluate(Text("""[1, 2, 3]""")).Passed);
|
|
117
86
|
}
|
|
118
87
|
|
|
119
|
-
// ------------------------------------------------------------------- NoPii
|
|
120
|
-
|
|
121
88
|
[Theory]
|
|
122
89
|
[InlineData("Entre em contato pelo lucas@example.com quando puder.", "e-mail")]
|
|
123
90
|
[InlineData("O CPF do titular e 123.456.789-01.", "CPF")]
|
|
@@ -139,8 +106,6 @@ public sealed class EvalAssertionTests
|
|
|
139
106
|
Assert.True(assertion.Evaluate(Text("Bom dia! Como posso ajudar voce hoje?")).Passed);
|
|
140
107
|
}
|
|
141
108
|
|
|
142
|
-
// -------------------------------------------------------------- ToolCalled
|
|
143
|
-
|
|
144
109
|
[Fact]
|
|
145
110
|
public void ToolCalledSeparatesTheRightToolFromAnotherToolAndFromNoToolAtAll()
|
|
146
111
|
{
|
|
@@ -155,8 +120,6 @@ public sealed class EvalAssertionTests
|
|
|
155
120
|
Assert.False(assertion.Evaluate(Text("vou escalar para um humano")).Passed);
|
|
156
121
|
}
|
|
157
122
|
|
|
158
|
-
// ------------------------------------------------------------ ToolArgument
|
|
159
|
-
|
|
160
123
|
[Fact]
|
|
161
124
|
public void ToolArgumentSeparatesTheRightArgumentFromTheWrongValueAndTheMissingKey()
|
|
162
125
|
{
|
|
@@ -179,13 +142,9 @@ public sealed class EvalAssertionTests
|
|
|
179
142
|
"ToolArgument aceitou uma chamada sem a CHAVE exigida");
|
|
180
143
|
}
|
|
181
144
|
|
|
182
|
-
// ------------------------------------- o veredito diz QUEM julgou (B2)
|
|
183
|
-
|
|
184
145
|
[Fact]
|
|
185
146
|
public void EveryOutcomeCarriesTheNameAndTheKindOfTheAssertionThatProducedIt()
|
|
186
147
|
{
|
|
187
|
-
// Sem este carimbo, uma falha chega a jusante como contador e o sumario nao consegue
|
|
188
|
-
// nomear nada — foi assim que o eval mordia anonimamente.
|
|
189
148
|
var assertion = new EvalAssertion { Name = "saudacao", Kind = EvalAssertionKind.Contains, Value = "Bom dia" };
|
|
190
149
|
|
|
191
150
|
foreach (var outcome in new[] { assertion.Evaluate(Text("Bom dia")), assertion.Evaluate(Text("nada")) })
|
package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Evals/EvalCacheMissTests.cs
CHANGED
|
@@ -4,16 +4,6 @@ using Xunit;
|
|
|
4
4
|
|
|
5
5
|
namespace Morph.AiKit.Tests.Evals;
|
|
6
6
|
|
|
7
|
-
/// <summary>
|
|
8
|
-
/// Miss de cache sem credencial REPROVA, nomeado. Jamais <c>skip</c>.
|
|
9
|
-
/// </summary>
|
|
10
|
-
/// <remarks>
|
|
11
|
-
/// A tentacao do <c>skip</c> e real ("nao da para avaliar, entao nao avalio") e recria
|
|
12
|
-
/// exatamente o buraco que este harness fechou duas vezes: gate verde sobre verificacao que
|
|
13
|
-
/// nao aconteceu. E ha razao positiva: em CI, com cache commitada, um miss significa UMA
|
|
14
|
-
/// coisa so — o prompt ou o modelo mudou e ninguem regravou. O miss e o veredito, nao o
|
|
15
|
-
/// obstaculo.
|
|
16
|
-
/// </remarks>
|
|
17
7
|
public sealed class EvalCacheMissTests
|
|
18
8
|
{
|
|
19
9
|
private static async Task<EvalWorkspace> SeededAsync(params string[] scenarioNames)
|
|
@@ -41,21 +31,16 @@ public sealed class EvalCacheMissTests
|
|
|
41
31
|
{
|
|
42
32
|
using var ws = await SeededAsync("a-primeiro");
|
|
43
33
|
|
|
44
|
-
//
|
|
45
|
-
// ninguem regravou". O prefixo "b-" e deliberado: os cenarios rodam em ordem ordinal
|
|
46
|
-
// de caminho, e o teste abaixo depende de o cenario BOM ter rodado antes do miss.
|
|
34
|
+
// Prefixo "b-": os cenarios rodam em ordem ordinal de caminho, e o miss tem de vir depois do cenario bom.
|
|
47
35
|
ws.WriteScenario("triagem", "acme", "b-novo", "pergunta inedita", """[ { "kind": "contains", "value": "resposta" } ]""");
|
|
48
36
|
|
|
49
37
|
await using var offline = EvalFixture.Create(ws.Options(EvalMode.Offline));
|
|
50
38
|
var error = await Assert.ThrowsAsync<EvalCacheMissException>(
|
|
51
39
|
() => offline.RunAllAsync(TestContext.Current.CancellationToken));
|
|
52
40
|
|
|
53
|
-
// O cenario que faltou, e nao "algum cenario".
|
|
54
41
|
Assert.Equal("triagem.acme.b-novo", error.ScenarioName);
|
|
55
42
|
Assert.Contains("eval-cache-miss", error.Message, StringComparison.Ordinal);
|
|
56
43
|
|
|
57
|
-
// P3: a mensagem carrega o comando de regravacao e o arquivo de credencial esperado.
|
|
58
|
-
// Um "cache miss" seco mandaria o leitor procurar os dois.
|
|
59
44
|
Assert.Contains(EvalCacheMissException.RefreshCommand, error.Message, StringComparison.Ordinal);
|
|
60
45
|
Assert.Contains(".env.evals", error.Message, StringComparison.Ordinal);
|
|
61
46
|
Assert.Contains(ws.CacheRoot, error.Message, StringComparison.Ordinal);
|
|
@@ -67,33 +52,22 @@ public sealed class EvalCacheMissTests
|
|
|
67
52
|
using var ws = await SeededAsync("a-primeiro");
|
|
68
53
|
ws.WriteScenario("triagem", "acme", "b-novo", "pergunta inedita", """[ { "kind": "contains", "value": "resposta" } ]""");
|
|
69
54
|
|
|
70
|
-
// O sumario da SEMEADURA, byte a byte. Ele e a linha de base: o que este teste cobra
|
|
71
|
-
// e que o run que morreu por miss nao escreva NADA por cima dele.
|
|
72
55
|
var summaryBeforeTheMiss = File.ReadAllBytes(ws.SummaryPath);
|
|
73
56
|
|
|
74
57
|
await using var offline = EvalFixture.Create(ws.Options(EvalMode.Offline));
|
|
75
58
|
await Assert.ThrowsAsync<EvalCacheMissException>(
|
|
76
59
|
() => offline.RunAllAsync(TestContext.Current.CancellationToken));
|
|
77
60
|
|
|
78
|
-
// Nao ha caminho pelo qual o cenario ausente apareca como avaliado: a contagem so tem
|
|
79
|
-
// o que de fato resolveu, e o sumario NAO foi reescrito. Um sumario dizendo
|
|
80
|
-
// "1 cenario, 1 passou, cache warm" aqui seria exatamente a mentira que a excecao
|
|
81
|
-
// existe para impedir — e o CLI, que le o ARQUIVO, acreditaria nela.
|
|
82
61
|
Assert.Single(offline.Results);
|
|
83
62
|
Assert.Equal("triagem.acme.a-primeiro", offline.Results[0].Scenario);
|
|
84
63
|
Assert.Equal(summaryBeforeTheMiss, File.ReadAllBytes(ws.SummaryPath));
|
|
85
64
|
|
|
86
|
-
// E o contador do cliente interno registra a tentativa: a arvore de cache nao
|
|
87
|
-
// respondeu, e isso e um FATO medido, nao uma inferencia.
|
|
88
65
|
Assert.Equal(1, offline.Offline!.Invocations);
|
|
89
66
|
}
|
|
90
67
|
|
|
91
68
|
[Fact]
|
|
92
69
|
public async Task AFailedAssertionMakesTheScenarioREDWithoutAnyCacheMiss()
|
|
93
70
|
{
|
|
94
|
-
// A outra metade do contrato: reprovar por conteudo e diferente de reprovar por
|
|
95
|
-
// cache. Uma so mensagem para os dois casos mandaria o leitor procurar no lugar
|
|
96
|
-
// errado metade das vezes.
|
|
97
71
|
using var ws = new EvalWorkspace();
|
|
98
72
|
ws.WriteScenario("triagem", "acme", "saudacao", "bom dia", """[ { "kind": "contains", "value": "jamais aparece" } ]""");
|
|
99
73
|
|
|
@@ -130,9 +104,6 @@ public sealed class EvalCacheMissTests
|
|
|
130
104
|
await using var offline = EvalFixture.Create(ws.Options(EvalMode.Offline));
|
|
131
105
|
var summary = await offline.RunAllAsync(TestContext.Current.CancellationToken);
|
|
132
106
|
|
|
133
|
-
// Um cenario que nao cobra nada nao pode contar como cobertura. Deixa-lo verde e o
|
|
134
|
-
// mesmo defeito de um teste vazio — pior aqui, porque ele apareceria na contagem
|
|
135
|
-
// como se tivesse verificado algo.
|
|
136
107
|
Assert.Equal(1, summary.Failed);
|
|
137
108
|
Assert.Equal(0, summary.Passed);
|
|
138
109
|
}
|
package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Evals/EvalCacheStampTests.cs
CHANGED
|
@@ -4,14 +4,6 @@ using Xunit;
|
|
|
4
4
|
|
|
5
5
|
namespace Morph.AiKit.Tests.Evals;
|
|
6
6
|
|
|
7
|
-
/// <summary>
|
|
8
|
-
/// O carimbo: por que a arvore de cache precisa dizer sob quais versoes foi gravada.
|
|
9
|
-
/// </summary>
|
|
10
|
-
/// <remarks>
|
|
11
|
-
/// A funcao de hash que produz a chave nao promete estabilidade entre versoes da biblioteca.
|
|
12
|
-
/// Sem carimbo, subir o pin chega como N misses inexplicaveis num dia em que ninguem mexeu
|
|
13
|
-
/// em nada; com carimbo, chega como uma frase que nomeia as DUAS versoes.
|
|
14
|
-
/// </remarks>
|
|
15
7
|
public sealed class EvalCacheStampTests
|
|
16
8
|
{
|
|
17
9
|
private static readonly FakeClock Today = new(new DateTimeOffset(2026, 9, 8, 12, 0, 0, TimeSpan.Zero));
|
|
@@ -51,8 +43,6 @@ public sealed class EvalCacheStampTests
|
|
|
51
43
|
using var ws = new EvalWorkspace();
|
|
52
44
|
Stamp(ws.CacheRoot);
|
|
53
45
|
|
|
54
|
-
// Ler de VOLTA do disco, e nao comparar o objeto com ele mesmo: o carimbo e um
|
|
55
|
-
// artefato, e um teste que so olhasse a instancia em memoria seria um espelho.
|
|
56
46
|
var read = EvalCacheStamp.Read(ws.CacheRoot);
|
|
57
47
|
|
|
58
48
|
Assert.NotNull(read);
|
|
@@ -72,7 +62,6 @@ public sealed class EvalCacheStampTests
|
|
|
72
62
|
|
|
73
63
|
Assert.Equal(EvalStampVerdict.PinMismatch, check.Verdict);
|
|
74
64
|
Assert.Contains("eval-cache-stale-pin", check.Reason, StringComparison.Ordinal);
|
|
75
|
-
// Dizer so uma das versoes obriga o leitor a ir buscar a outra.
|
|
76
65
|
Assert.Contains("gravada 10.9.0", check.Reason, StringComparison.Ordinal);
|
|
77
66
|
Assert.Contains("pin 10.11.0", check.Reason, StringComparison.Ordinal);
|
|
78
67
|
}
|
|
@@ -89,9 +78,6 @@ public sealed class EvalCacheStampTests
|
|
|
89
78
|
Assert.Equal(EvalStampVerdict.Missing, missing.Verdict);
|
|
90
79
|
Assert.Contains("eval-cache-unstamped", missing.Reason, StringComparison.Ordinal);
|
|
91
80
|
|
|
92
|
-
// As duas situacoes pedem acoes diferentes: "regrave" versus "regrave PORQUE o
|
|
93
|
-
// pacote mudou". Um veredito unico devolveria ao leitor a pergunta que o carimbo
|
|
94
|
-
// existia para responder.
|
|
95
81
|
Assert.NotEqual(missing.Verdict, divergent.Verdict);
|
|
96
82
|
Assert.DoesNotContain("eval-cache-stale-pin", missing.Reason, StringComparison.Ordinal);
|
|
97
83
|
}
|
|
@@ -116,8 +102,6 @@ public sealed class EvalCacheStampTests
|
|
|
116
102
|
|
|
117
103
|
var check = EvalCacheStamp.Check(ws.CacheRoot, Pin(pinnedAt: "2027-01-01"), Today);
|
|
118
104
|
|
|
119
|
-
// O carimbo CASA com o pin — e mesmo assim reprova, porque a regua esta adulterada.
|
|
120
|
-
// Uma data no futuro renovaria sozinha qualquer janela medida contra ela.
|
|
121
105
|
Assert.Equal(EvalStampVerdict.FuturePin, check.Verdict);
|
|
122
106
|
Assert.Contains("eval-pin-future", check.Reason, StringComparison.Ordinal);
|
|
123
107
|
}
|
|
@@ -135,8 +119,6 @@ public sealed class EvalCacheStampTests
|
|
|
135
119
|
["Microsoft.Extensions.AI.Evaluation"] = "10.11.0",
|
|
136
120
|
});
|
|
137
121
|
|
|
138
|
-
// A recusa acontece na CONSTRUCAO — antes de qualquer cenario. Nao faz sentido pagar
|
|
139
|
-
// o relogio de uma suite para descobrir no fim que a cache nao servia.
|
|
140
122
|
var error = Assert.Throws<InvalidOperationException>(
|
|
141
123
|
() => EvalFixture.Create(ws.Options(EvalMode.Offline, pin), timeProvider: Today));
|
|
142
124
|
|
|
@@ -148,7 +130,7 @@ public sealed class EvalCacheStampTests
|
|
|
148
130
|
{
|
|
149
131
|
using var ws = new EvalWorkspace();
|
|
150
132
|
|
|
151
|
-
// Cobrar carimbo
|
|
133
|
+
// Cobrar carimbo na escrita impediria criar a primeira cache.
|
|
152
134
|
var fixture = EvalFixture.Create(ws.Options(EvalMode.Refresh), new FakeChatClient());
|
|
153
135
|
|
|
154
136
|
Assert.True(fixture.StampCheck.IsOk);
|
package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Evals/EvalFailuresTests.cs
CHANGED
|
@@ -4,37 +4,8 @@ using Xunit;
|
|
|
4
4
|
|
|
5
5
|
namespace Morph.AiKit.Tests.Evals;
|
|
6
6
|
|
|
7
|
-
/// <summary>
|
|
8
|
-
/// A camada que NOMEIA a falha, testada pelo conteudo do que ela produz.
|
|
9
|
-
/// </summary>
|
|
10
|
-
/// <remarks>
|
|
11
|
-
/// <para>
|
|
12
|
-
/// Este arquivo fecha a setima ocorrencia da mesma classe nesta epic, e a mais ironica: o
|
|
13
|
-
/// mecanismo de <c>Failures</c> nasceu para consertar um eval que mordia ANONIMAMENTE, e
|
|
14
|
-
/// nasceu ele proprio sem par. As tres partes podiam ser desligadas sem matar um teste —
|
|
15
|
-
/// <c>CollectFailures</c> devolvendo lista vazia, o ramo da vacuidade nunca disparando, e o
|
|
16
|
-
/// nome da assercao trocado por um literal — e a suite seguia "165 total, 0 falhou".
|
|
17
|
-
/// </para>
|
|
18
|
-
/// <para>
|
|
19
|
-
/// A causa foi uma unica testemunha automatizada: <c>Assert.Empty(summary.Failures)</c> num
|
|
20
|
-
/// run VERDE. Essa asserção é verdadeira exatamente quando o mecanismo esta morto — ela nao
|
|
21
|
-
/// distingue "nao houve falha" de "falha nenhuma e registrada". Uma lista vazia prova a
|
|
22
|
-
/// ausencia de conteudo, jamais a presenca de capacidade.
|
|
23
|
-
/// </para>
|
|
24
|
-
/// <para>
|
|
25
|
-
/// Por isso aqui nao se assere sobre <c>Failures</c> VAZIA: assere-se sobre o CONTEUDO de
|
|
26
|
-
/// <c>Failures</c> num run que falha de verdade — o par, a mesma forma que fechou o motor de
|
|
27
|
-
/// assercao um andar abaixo.
|
|
28
|
-
/// </para>
|
|
29
|
-
/// </remarks>
|
|
30
7
|
public sealed class EvalFailuresTests
|
|
31
8
|
{
|
|
32
|
-
/// <summary>Grava a cache do cenario com o texto dado, e devolve o fixture offline pronto.</summary>
|
|
33
|
-
/// <remarks>
|
|
34
|
-
/// A cache e semeada pelo <see cref="FakeChatClient"/>, NUNCA por chamada paga: o caminho
|
|
35
|
-
/// de escrita exercitado e o da biblioteca, e a leitura offline le exatamente o que aquela
|
|
36
|
-
/// escrita produziu.
|
|
37
|
-
/// </remarks>
|
|
38
9
|
private static async Task<EvalRunSummary> RunOfflineAsync(EvalWorkspace ws, params string[] responses)
|
|
39
10
|
{
|
|
40
11
|
using var fake = new FakeChatClient
|
|
@@ -60,9 +31,6 @@ public sealed class EvalFailuresTests
|
|
|
60
31
|
[Fact]
|
|
61
32
|
public async Task AFailedScenarioIsRegisteredWithScenarioAssertionKindAndReason()
|
|
62
33
|
{
|
|
63
|
-
// O caso central: uma assercao reprova, e a falha chega ao sumario COM NOME. Sem isso
|
|
64
|
-
// o veredito viaja como contador — "Expected 0, Actual 1" — e quem le tem de abrir o
|
|
65
|
-
// dataset para descobrir o que quebrou.
|
|
66
34
|
using var ws = new EvalWorkspace();
|
|
67
35
|
ws.WriteScenario(
|
|
68
36
|
"triagem", "acme", "saudacao", "bom dia",
|
|
@@ -79,7 +47,6 @@ public sealed class EvalFailuresTests
|
|
|
79
47
|
Assert.Contains("FALHOU", failure.Reason, StringComparison.Ordinal);
|
|
80
48
|
Assert.Contains("bom dia", failure.Reason, StringComparison.Ordinal);
|
|
81
49
|
|
|
82
|
-
// A linha legivel, no formato que a rubrica eval-de-modelo.md cita como exemplo.
|
|
83
50
|
Assert.Contains(
|
|
84
51
|
"eval-fail: triagem.acme.saudacao",
|
|
85
52
|
summary.DescribeFailures(),
|
|
@@ -89,9 +56,6 @@ public sealed class EvalFailuresTests
|
|
|
89
56
|
[Fact]
|
|
90
57
|
public async Task TwoReprovedAssertionsInOneScenarioProduceTWOLinesEachNamingItsOwnRule()
|
|
91
58
|
{
|
|
92
|
-
// `Failures` e LISTA, e nao contador, porque as duas linhas dizem coisas diferentes:
|
|
93
|
-
// uma resposta pode quebrar em mais de uma regua, e isso separa "o prompt mudou" de
|
|
94
|
-
// "o modelo parou de responder". Se o tipo fosse um booleano, esta distincao morreria.
|
|
95
59
|
using var ws = new EvalWorkspace();
|
|
96
60
|
ws.WriteScenario(
|
|
97
61
|
"triagem", "acme", "saudacao", "bom dia",
|
|
@@ -110,18 +74,12 @@ public sealed class EvalFailuresTests
|
|
|
110
74
|
Assert.Contains(summary.Failures, f => f.Assertion == "cumprimenta" && f.Kind == "Contains");
|
|
111
75
|
Assert.Contains(summary.Failures, f => f.Assertion == "sem-dado-pessoal" && f.Kind == "NoPii");
|
|
112
76
|
|
|
113
|
-
// Toda linha nomeia o cenario: uma falha sem cenario obriga a mesma busca que ela
|
|
114
|
-
// deveria dispensar.
|
|
115
77
|
Assert.All(summary.Failures, f => Assert.Equal("triagem.acme.saudacao", f.Scenario));
|
|
116
78
|
}
|
|
117
79
|
|
|
118
80
|
[Fact]
|
|
119
81
|
public async Task AScenarioWithNOAssertionsIsCountedANDNamed()
|
|
120
82
|
{
|
|
121
|
-
// O cenario sem assercao nenhuma reprova por VACUIDADE: ele nao cobra nada. Nesse caso
|
|
122
|
-
// nao existe outcome reprovado para nomear, e sem um ramo proprio ele seria contado em
|
|
123
|
-
// `Failed` e invisivel em `Failures` — um numero sem frase, que e metade exata do
|
|
124
|
-
// defeito que este tipo existe para fechar.
|
|
125
83
|
using var ws = new EvalWorkspace();
|
|
126
84
|
ws.WriteScenario("triagem", "acme", "vazio", "bom dia", "[]");
|
|
127
85
|
|
|
@@ -133,16 +91,12 @@ public sealed class EvalFailuresTests
|
|
|
133
91
|
Assert.Equal("triagem.acme.vazio", failure.Scenario);
|
|
134
92
|
Assert.Contains("sem assercao deterministica", failure.Reason, StringComparison.Ordinal);
|
|
135
93
|
|
|
136
|
-
// Contado E nomeado: o numero e a frase tem de concordar sempre.
|
|
137
94
|
Assert.Equal(summary.Failed, summary.Failures.Count);
|
|
138
95
|
}
|
|
139
96
|
|
|
140
97
|
[Fact]
|
|
141
98
|
public async Task AGreenRunRegistersNoFailureAndSaysSoInWords()
|
|
142
99
|
{
|
|
143
|
-
// A reciproca vive AO LADO dos casos que falham de verdade, nunca sozinha: uma suite
|
|
144
|
-
// cuja unica testemunha e `Assert.Empty` num run verde e verdadeira precisamente
|
|
145
|
-
// quando o mecanismo esta morto.
|
|
146
100
|
using var ws = new EvalWorkspace();
|
|
147
101
|
ws.WriteScenario(
|
|
148
102
|
"triagem", "acme", "saudacao", "bom dia",
|
|
@@ -158,8 +112,6 @@ public sealed class EvalFailuresTests
|
|
|
158
112
|
[Fact]
|
|
159
113
|
public async Task TheNamedFailuresSurviveTheROUNDTRIPThroughTheSummaryFileTheCLIReads()
|
|
160
114
|
{
|
|
161
|
-
// O sumario e um ARQUIVO: quem consome as falhas e o CLI, do outro lado do disco. Uma
|
|
162
|
-
// lista que existisse so em memoria seria invisivel exatamente onde precisa ser lida.
|
|
163
115
|
using var ws = new EvalWorkspace();
|
|
164
116
|
ws.WriteScenario(
|
|
165
117
|
"triagem", "acme", "saudacao", "bom dia",
|