@dotsetlabs/bellwether 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +291 -0
- package/LICENSE +21 -0
- package/README.md +739 -0
- package/dist/auth/credentials.d.ts +64 -0
- package/dist/auth/credentials.js +218 -0
- package/dist/auth/index.d.ts +6 -0
- package/dist/auth/index.js +6 -0
- package/dist/auth/keychain.d.ts +64 -0
- package/dist/auth/keychain.js +268 -0
- package/dist/baseline/ab-testing.d.ts +80 -0
- package/dist/baseline/ab-testing.js +236 -0
- package/dist/baseline/ai-compatibility-scorer.d.ts +95 -0
- package/dist/baseline/ai-compatibility-scorer.js +606 -0
- package/dist/baseline/calibration.d.ts +77 -0
- package/dist/baseline/calibration.js +136 -0
- package/dist/baseline/category-matching.d.ts +85 -0
- package/dist/baseline/category-matching.js +289 -0
- package/dist/baseline/change-impact-analyzer.d.ts +98 -0
- package/dist/baseline/change-impact-analyzer.js +592 -0
- package/dist/baseline/comparator.d.ts +64 -0
- package/dist/baseline/comparator.js +916 -0
- package/dist/baseline/confidence.d.ts +55 -0
- package/dist/baseline/confidence.js +122 -0
- package/dist/baseline/converter.d.ts +61 -0
- package/dist/baseline/converter.js +585 -0
- package/dist/baseline/dependency-analyzer.d.ts +89 -0
- package/dist/baseline/dependency-analyzer.js +567 -0
- package/dist/baseline/deprecation-tracker.d.ts +133 -0
- package/dist/baseline/deprecation-tracker.js +322 -0
- package/dist/baseline/diff.d.ts +55 -0
- package/dist/baseline/diff.js +1584 -0
- package/dist/baseline/documentation-scorer.d.ts +205 -0
- package/dist/baseline/documentation-scorer.js +466 -0
- package/dist/baseline/embeddings.d.ts +118 -0
- package/dist/baseline/embeddings.js +251 -0
- package/dist/baseline/error-analyzer.d.ts +198 -0
- package/dist/baseline/error-analyzer.js +721 -0
- package/dist/baseline/evaluation/evaluator.d.ts +42 -0
- package/dist/baseline/evaluation/evaluator.js +323 -0
- package/dist/baseline/evaluation/expanded-dataset.d.ts +45 -0
- package/dist/baseline/evaluation/expanded-dataset.js +1164 -0
- package/dist/baseline/evaluation/golden-dataset.d.ts +58 -0
- package/dist/baseline/evaluation/golden-dataset.js +717 -0
- package/dist/baseline/evaluation/index.d.ts +15 -0
- package/dist/baseline/evaluation/index.js +15 -0
- package/dist/baseline/evaluation/types.d.ts +186 -0
- package/dist/baseline/evaluation/types.js +8 -0
- package/dist/baseline/external-dependency-detector.d.ts +181 -0
- package/dist/baseline/external-dependency-detector.js +524 -0
- package/dist/baseline/golden-output.d.ts +162 -0
- package/dist/baseline/golden-output.js +636 -0
- package/dist/baseline/health-scorer.d.ts +174 -0
- package/dist/baseline/health-scorer.js +451 -0
- package/dist/baseline/incremental-checker.d.ts +97 -0
- package/dist/baseline/incremental-checker.js +174 -0
- package/dist/baseline/index.d.ts +31 -0
- package/dist/baseline/index.js +42 -0
- package/dist/baseline/migration-generator.d.ts +137 -0
- package/dist/baseline/migration-generator.js +554 -0
- package/dist/baseline/migrations.d.ts +60 -0
- package/dist/baseline/migrations.js +197 -0
- package/dist/baseline/performance-tracker.d.ts +214 -0
- package/dist/baseline/performance-tracker.js +577 -0
- package/dist/baseline/pr-comment-generator.d.ts +117 -0
- package/dist/baseline/pr-comment-generator.js +546 -0
- package/dist/baseline/response-fingerprint.d.ts +127 -0
- package/dist/baseline/response-fingerprint.js +728 -0
- package/dist/baseline/response-schema-tracker.d.ts +129 -0
- package/dist/baseline/response-schema-tracker.js +420 -0
- package/dist/baseline/risk-scorer.d.ts +54 -0
- package/dist/baseline/risk-scorer.js +434 -0
- package/dist/baseline/saver.d.ts +89 -0
- package/dist/baseline/saver.js +554 -0
- package/dist/baseline/scenario-generator.d.ts +151 -0
- package/dist/baseline/scenario-generator.js +905 -0
- package/dist/baseline/schema-compare.d.ts +86 -0
- package/dist/baseline/schema-compare.js +557 -0
- package/dist/baseline/schema-evolution.d.ts +189 -0
- package/dist/baseline/schema-evolution.js +467 -0
- package/dist/baseline/semantic.d.ts +203 -0
- package/dist/baseline/semantic.js +908 -0
- package/dist/baseline/synonyms.d.ts +60 -0
- package/dist/baseline/synonyms.js +386 -0
- package/dist/baseline/telemetry.d.ts +165 -0
- package/dist/baseline/telemetry.js +294 -0
- package/dist/baseline/test-pruner.d.ts +120 -0
- package/dist/baseline/test-pruner.js +387 -0
- package/dist/baseline/types.d.ts +449 -0
- package/dist/baseline/types.js +5 -0
- package/dist/baseline/version.d.ts +138 -0
- package/dist/baseline/version.js +206 -0
- package/dist/cache/index.d.ts +5 -0
- package/dist/cache/index.js +5 -0
- package/dist/cache/response-cache.d.ts +151 -0
- package/dist/cache/response-cache.js +287 -0
- package/dist/ci/index.d.ts +60 -0
- package/dist/ci/index.js +342 -0
- package/dist/cli/commands/auth.d.ts +12 -0
- package/dist/cli/commands/auth.js +352 -0
- package/dist/cli/commands/badge.d.ts +3 -0
- package/dist/cli/commands/badge.js +74 -0
- package/dist/cli/commands/baseline-accept.d.ts +15 -0
- package/dist/cli/commands/baseline-accept.js +178 -0
- package/dist/cli/commands/baseline-migrate.d.ts +12 -0
- package/dist/cli/commands/baseline-migrate.js +164 -0
- package/dist/cli/commands/baseline.d.ts +14 -0
- package/dist/cli/commands/baseline.js +449 -0
- package/dist/cli/commands/beta.d.ts +10 -0
- package/dist/cli/commands/beta.js +231 -0
- package/dist/cli/commands/check.d.ts +11 -0
- package/dist/cli/commands/check.js +820 -0
- package/dist/cli/commands/cloud/badge.d.ts +3 -0
- package/dist/cli/commands/cloud/badge.js +74 -0
- package/dist/cli/commands/cloud/diff.d.ts +6 -0
- package/dist/cli/commands/cloud/diff.js +79 -0
- package/dist/cli/commands/cloud/history.d.ts +6 -0
- package/dist/cli/commands/cloud/history.js +102 -0
- package/dist/cli/commands/cloud/link.d.ts +9 -0
- package/dist/cli/commands/cloud/link.js +119 -0
- package/dist/cli/commands/cloud/login.d.ts +7 -0
- package/dist/cli/commands/cloud/login.js +499 -0
- package/dist/cli/commands/cloud/projects.d.ts +6 -0
- package/dist/cli/commands/cloud/projects.js +44 -0
- package/dist/cli/commands/cloud/shared.d.ts +7 -0
- package/dist/cli/commands/cloud/shared.js +42 -0
- package/dist/cli/commands/cloud/teams.d.ts +8 -0
- package/dist/cli/commands/cloud/teams.js +169 -0
- package/dist/cli/commands/cloud/upload.d.ts +8 -0
- package/dist/cli/commands/cloud/upload.js +181 -0
- package/dist/cli/commands/contract.d.ts +11 -0
- package/dist/cli/commands/contract.js +280 -0
- package/dist/cli/commands/discover.d.ts +3 -0
- package/dist/cli/commands/discover.js +82 -0
- package/dist/cli/commands/eval.d.ts +9 -0
- package/dist/cli/commands/eval.js +187 -0
- package/dist/cli/commands/explore.d.ts +11 -0
- package/dist/cli/commands/explore.js +437 -0
- package/dist/cli/commands/feedback.d.ts +9 -0
- package/dist/cli/commands/feedback.js +174 -0
- package/dist/cli/commands/golden.d.ts +12 -0
- package/dist/cli/commands/golden.js +407 -0
- package/dist/cli/commands/history.d.ts +10 -0
- package/dist/cli/commands/history.js +202 -0
- package/dist/cli/commands/init.d.ts +9 -0
- package/dist/cli/commands/init.js +219 -0
- package/dist/cli/commands/interview.d.ts +3 -0
- package/dist/cli/commands/interview.js +903 -0
- package/dist/cli/commands/link.d.ts +10 -0
- package/dist/cli/commands/link.js +169 -0
- package/dist/cli/commands/login.d.ts +7 -0
- package/dist/cli/commands/login.js +499 -0
- package/dist/cli/commands/preset.d.ts +33 -0
- package/dist/cli/commands/preset.js +297 -0
- package/dist/cli/commands/profile.d.ts +33 -0
- package/dist/cli/commands/profile.js +286 -0
- package/dist/cli/commands/registry.d.ts +11 -0
- package/dist/cli/commands/registry.js +146 -0
- package/dist/cli/commands/shared.d.ts +79 -0
- package/dist/cli/commands/shared.js +196 -0
- package/dist/cli/commands/teams.d.ts +8 -0
- package/dist/cli/commands/teams.js +169 -0
- package/dist/cli/commands/test.d.ts +9 -0
- package/dist/cli/commands/test.js +500 -0
- package/dist/cli/commands/upload.d.ts +8 -0
- package/dist/cli/commands/upload.js +223 -0
- package/dist/cli/commands/validate-config.d.ts +6 -0
- package/dist/cli/commands/validate-config.js +35 -0
- package/dist/cli/commands/verify.d.ts +11 -0
- package/dist/cli/commands/verify.js +283 -0
- package/dist/cli/commands/watch.d.ts +12 -0
- package/dist/cli/commands/watch.js +253 -0
- package/dist/cli/index.d.ts +3 -0
- package/dist/cli/index.js +178 -0
- package/dist/cli/interactive.d.ts +47 -0
- package/dist/cli/interactive.js +216 -0
- package/dist/cli/output/terminal-reporter.d.ts +19 -0
- package/dist/cli/output/terminal-reporter.js +104 -0
- package/dist/cli/output.d.ts +226 -0
- package/dist/cli/output.js +438 -0
- package/dist/cli/utils/env.d.ts +5 -0
- package/dist/cli/utils/env.js +14 -0
- package/dist/cli/utils/progress.d.ts +59 -0
- package/dist/cli/utils/progress.js +206 -0
- package/dist/cli/utils/server-context.d.ts +10 -0
- package/dist/cli/utils/server-context.js +36 -0
- package/dist/cloud/auth.d.ts +144 -0
- package/dist/cloud/auth.js +374 -0
- package/dist/cloud/client.d.ts +24 -0
- package/dist/cloud/client.js +65 -0
- package/dist/cloud/http-client.d.ts +38 -0
- package/dist/cloud/http-client.js +215 -0
- package/dist/cloud/index.d.ts +23 -0
- package/dist/cloud/index.js +25 -0
- package/dist/cloud/mock-client.d.ts +107 -0
- package/dist/cloud/mock-client.js +545 -0
- package/dist/cloud/types.d.ts +515 -0
- package/dist/cloud/types.js +15 -0
- package/dist/config/defaults.d.ts +160 -0
- package/dist/config/defaults.js +169 -0
- package/dist/config/loader.d.ts +24 -0
- package/dist/config/loader.js +122 -0
- package/dist/config/template.d.ts +42 -0
- package/dist/config/template.js +647 -0
- package/dist/config/validator.d.ts +2112 -0
- package/dist/config/validator.js +658 -0
- package/dist/constants/cloud.d.ts +107 -0
- package/dist/constants/cloud.js +110 -0
- package/dist/constants/core.d.ts +521 -0
- package/dist/constants/core.js +556 -0
- package/dist/constants/testing.d.ts +1283 -0
- package/dist/constants/testing.js +1568 -0
- package/dist/constants.d.ts +10 -0
- package/dist/constants.js +10 -0
- package/dist/contract/index.d.ts +6 -0
- package/dist/contract/index.js +5 -0
- package/dist/contract/validator.d.ts +177 -0
- package/dist/contract/validator.js +574 -0
- package/dist/cost/index.d.ts +6 -0
- package/dist/cost/index.js +5 -0
- package/dist/cost/tracker.d.ts +134 -0
- package/dist/cost/tracker.js +313 -0
- package/dist/discovery/discovery.d.ts +16 -0
- package/dist/discovery/discovery.js +173 -0
- package/dist/discovery/types.d.ts +51 -0
- package/dist/discovery/types.js +2 -0
- package/dist/docs/agents.d.ts +3 -0
- package/dist/docs/agents.js +995 -0
- package/dist/docs/contract.d.ts +51 -0
- package/dist/docs/contract.js +1681 -0
- package/dist/docs/generator.d.ts +4 -0
- package/dist/docs/generator.js +4 -0
- package/dist/docs/html-reporter.d.ts +9 -0
- package/dist/docs/html-reporter.js +757 -0
- package/dist/docs/index.d.ts +10 -0
- package/dist/docs/index.js +11 -0
- package/dist/docs/junit-reporter.d.ts +18 -0
- package/dist/docs/junit-reporter.js +210 -0
- package/dist/docs/report.d.ts +14 -0
- package/dist/docs/report.js +44 -0
- package/dist/docs/sarif-reporter.d.ts +19 -0
- package/dist/docs/sarif-reporter.js +335 -0
- package/dist/docs/shared.d.ts +35 -0
- package/dist/docs/shared.js +162 -0
- package/dist/docs/templates.d.ts +12 -0
- package/dist/docs/templates.js +76 -0
- package/dist/errors/index.d.ts +6 -0
- package/dist/errors/index.js +6 -0
- package/dist/errors/retry.d.ts +92 -0
- package/dist/errors/retry.js +323 -0
- package/dist/errors/types.d.ts +321 -0
- package/dist/errors/types.js +584 -0
- package/dist/index.d.ts +32 -0
- package/dist/index.js +32 -0
- package/dist/interview/dependency-resolver.d.ts +11 -0
- package/dist/interview/dependency-resolver.js +32 -0
- package/dist/interview/interviewer.d.ts +232 -0
- package/dist/interview/interviewer.js +1939 -0
- package/dist/interview/mock-response-generator.d.ts +7 -0
- package/dist/interview/mock-response-generator.js +102 -0
- package/dist/interview/orchestrator.d.ts +237 -0
- package/dist/interview/orchestrator.js +1296 -0
- package/dist/interview/rate-limiter.d.ts +15 -0
- package/dist/interview/rate-limiter.js +55 -0
- package/dist/interview/response-validator.d.ts +10 -0
- package/dist/interview/response-validator.js +132 -0
- package/dist/interview/schema-inferrer.d.ts +8 -0
- package/dist/interview/schema-inferrer.js +71 -0
- package/dist/interview/schema-test-generator.d.ts +71 -0
- package/dist/interview/schema-test-generator.js +834 -0
- package/dist/interview/smart-value-generator.d.ts +155 -0
- package/dist/interview/smart-value-generator.js +554 -0
- package/dist/interview/stateful-test-runner.d.ts +19 -0
- package/dist/interview/stateful-test-runner.js +106 -0
- package/dist/interview/types.d.ts +561 -0
- package/dist/interview/types.js +2 -0
- package/dist/llm/anthropic.d.ts +41 -0
- package/dist/llm/anthropic.js +355 -0
- package/dist/llm/client.d.ts +123 -0
- package/dist/llm/client.js +42 -0
- package/dist/llm/factory.d.ts +38 -0
- package/dist/llm/factory.js +145 -0
- package/dist/llm/fallback.d.ts +140 -0
- package/dist/llm/fallback.js +379 -0
- package/dist/llm/index.d.ts +18 -0
- package/dist/llm/index.js +15 -0
- package/dist/llm/ollama.d.ts +37 -0
- package/dist/llm/ollama.js +330 -0
- package/dist/llm/openai.d.ts +25 -0
- package/dist/llm/openai.js +320 -0
- package/dist/llm/token-budget.d.ts +161 -0
- package/dist/llm/token-budget.js +395 -0
- package/dist/logging/logger.d.ts +70 -0
- package/dist/logging/logger.js +130 -0
- package/dist/metrics/collector.d.ts +106 -0
- package/dist/metrics/collector.js +547 -0
- package/dist/metrics/index.d.ts +7 -0
- package/dist/metrics/index.js +7 -0
- package/dist/metrics/prometheus.d.ts +20 -0
- package/dist/metrics/prometheus.js +241 -0
- package/dist/metrics/types.d.ts +209 -0
- package/dist/metrics/types.js +5 -0
- package/dist/persona/builtins.d.ts +54 -0
- package/dist/persona/builtins.js +219 -0
- package/dist/persona/index.d.ts +8 -0
- package/dist/persona/index.js +8 -0
- package/dist/persona/loader.d.ts +30 -0
- package/dist/persona/loader.js +190 -0
- package/dist/persona/types.d.ts +144 -0
- package/dist/persona/types.js +5 -0
- package/dist/persona/validation.d.ts +94 -0
- package/dist/persona/validation.js +332 -0
- package/dist/prompts/index.d.ts +5 -0
- package/dist/prompts/index.js +5 -0
- package/dist/prompts/templates.d.ts +180 -0
- package/dist/prompts/templates.js +431 -0
- package/dist/registry/client.d.ts +49 -0
- package/dist/registry/client.js +191 -0
- package/dist/registry/index.d.ts +7 -0
- package/dist/registry/index.js +6 -0
- package/dist/registry/types.d.ts +140 -0
- package/dist/registry/types.js +6 -0
- package/dist/scenarios/evaluator.d.ts +43 -0
- package/dist/scenarios/evaluator.js +206 -0
- package/dist/scenarios/index.d.ts +10 -0
- package/dist/scenarios/index.js +9 -0
- package/dist/scenarios/loader.d.ts +20 -0
- package/dist/scenarios/loader.js +285 -0
- package/dist/scenarios/types.d.ts +153 -0
- package/dist/scenarios/types.js +8 -0
- package/dist/security/index.d.ts +17 -0
- package/dist/security/index.js +18 -0
- package/dist/security/payloads.d.ts +61 -0
- package/dist/security/payloads.js +268 -0
- package/dist/security/security-tester.d.ts +42 -0
- package/dist/security/security-tester.js +582 -0
- package/dist/security/types.d.ts +166 -0
- package/dist/security/types.js +8 -0
- package/dist/transport/base-transport.d.ts +59 -0
- package/dist/transport/base-transport.js +38 -0
- package/dist/transport/http-transport.d.ts +67 -0
- package/dist/transport/http-transport.js +238 -0
- package/dist/transport/mcp-client.d.ts +141 -0
- package/dist/transport/mcp-client.js +496 -0
- package/dist/transport/sse-transport.d.ts +88 -0
- package/dist/transport/sse-transport.js +316 -0
- package/dist/transport/stdio-transport.d.ts +43 -0
- package/dist/transport/stdio-transport.js +238 -0
- package/dist/transport/types.d.ts +125 -0
- package/dist/transport/types.js +16 -0
- package/dist/utils/concurrency.d.ts +123 -0
- package/dist/utils/concurrency.js +213 -0
- package/dist/utils/formatters.d.ts +16 -0
- package/dist/utils/formatters.js +37 -0
- package/dist/utils/index.d.ts +8 -0
- package/dist/utils/index.js +8 -0
- package/dist/utils/jsonpath.d.ts +87 -0
- package/dist/utils/jsonpath.js +326 -0
- package/dist/utils/markdown.d.ts +113 -0
- package/dist/utils/markdown.js +265 -0
- package/dist/utils/network.d.ts +14 -0
- package/dist/utils/network.js +17 -0
- package/dist/utils/sanitize.d.ts +92 -0
- package/dist/utils/sanitize.js +191 -0
- package/dist/utils/semantic.d.ts +194 -0
- package/dist/utils/semantic.js +1051 -0
- package/dist/utils/smart-truncate.d.ts +94 -0
- package/dist/utils/smart-truncate.js +361 -0
- package/dist/utils/timeout.d.ts +153 -0
- package/dist/utils/timeout.js +205 -0
- package/dist/utils/yaml-parser.d.ts +58 -0
- package/dist/utils/yaml-parser.js +86 -0
- package/dist/validation/index.d.ts +32 -0
- package/dist/validation/index.js +32 -0
- package/dist/validation/semantic-test-generator.d.ts +50 -0
- package/dist/validation/semantic-test-generator.js +176 -0
- package/dist/validation/semantic-types.d.ts +66 -0
- package/dist/validation/semantic-types.js +94 -0
- package/dist/validation/semantic-validator.d.ts +38 -0
- package/dist/validation/semantic-validator.js +340 -0
- package/dist/verification/index.d.ts +6 -0
- package/dist/verification/index.js +5 -0
- package/dist/verification/types.d.ts +133 -0
- package/dist/verification/types.js +5 -0
- package/dist/verification/verifier.d.ts +30 -0
- package/dist/verification/verifier.js +309 -0
- package/dist/version.d.ts +19 -0
- package/dist/version.js +48 -0
- package/dist/workflow/auto-generator.d.ts +27 -0
- package/dist/workflow/auto-generator.js +513 -0
- package/dist/workflow/discovery.d.ts +40 -0
- package/dist/workflow/discovery.js +195 -0
- package/dist/workflow/executor.d.ts +82 -0
- package/dist/workflow/executor.js +611 -0
- package/dist/workflow/index.d.ts +10 -0
- package/dist/workflow/index.js +10 -0
- package/dist/workflow/loader.d.ts +24 -0
- package/dist/workflow/loader.js +194 -0
- package/dist/workflow/state-tracker.d.ts +98 -0
- package/dist/workflow/state-tracker.js +424 -0
- package/dist/workflow/types.d.ts +337 -0
- package/dist/workflow/types.js +5 -0
- package/package.json +94 -0
- package/schemas/bellwether-check.schema.json +651 -0
|
@@ -0,0 +1,1584 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Diff output formatting for human and machine consumption.
|
|
3
|
+
*/
|
|
4
|
+
import { formatSchemaEvolutionDiff } from './response-schema-tracker.js';
|
|
5
|
+
import { formatErrorTrendReport } from './error-analyzer.js';
|
|
6
|
+
import { getGradeIndicator } from './documentation-scorer.js';
|
|
7
|
+
/**
|
|
8
|
+
* Format diff for human-readable console output.
|
|
9
|
+
*/
|
|
10
|
+
export function formatDiffText(diff, useColors = true) {
|
|
11
|
+
const lines = [];
|
|
12
|
+
const { red, green, yellow, cyan, bold } = useColors ? colors : noColors;
|
|
13
|
+
lines.push(bold('Drift Report'));
|
|
14
|
+
lines.push('═'.repeat(50));
|
|
15
|
+
lines.push('');
|
|
16
|
+
const severityBadge = getSeverityBadge(diff.severity, useColors);
|
|
17
|
+
lines.push(`Severity: ${severityBadge}`);
|
|
18
|
+
lines.push('');
|
|
19
|
+
lines.push(diff.summary);
|
|
20
|
+
lines.push('');
|
|
21
|
+
if (diff.toolsRemoved.length > 0) {
|
|
22
|
+
lines.push(red('─── Tools Removed ───'));
|
|
23
|
+
for (const tool of diff.toolsRemoved) {
|
|
24
|
+
lines.push(` ${red('✗')} ${tool}`);
|
|
25
|
+
}
|
|
26
|
+
lines.push('');
|
|
27
|
+
}
|
|
28
|
+
if (diff.toolsAdded.length > 0) {
|
|
29
|
+
lines.push(green('─── Tools Added ───'));
|
|
30
|
+
for (const tool of diff.toolsAdded) {
|
|
31
|
+
lines.push(` ${green('+')} ${tool}`);
|
|
32
|
+
}
|
|
33
|
+
lines.push('');
|
|
34
|
+
}
|
|
35
|
+
if (diff.toolsModified.length > 0) {
|
|
36
|
+
lines.push(yellow('─── Tools Modified ───'));
|
|
37
|
+
for (const toolDiff of diff.toolsModified) {
|
|
38
|
+
lines.push(` ${yellow('~')} ${bold(toolDiff.tool)}`);
|
|
39
|
+
if (toolDiff.schemaChanged) {
|
|
40
|
+
lines.push(` ${red('• Schema changed')}`);
|
|
41
|
+
}
|
|
42
|
+
if (toolDiff.descriptionChanged) {
|
|
43
|
+
lines.push(` ${yellow('• Description changed')}`);
|
|
44
|
+
}
|
|
45
|
+
for (const change of toolDiff.changes) {
|
|
46
|
+
const icon = getChangeIcon(change, useColors);
|
|
47
|
+
lines.push(` ${icon} ${change.description}`);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
lines.push('');
|
|
51
|
+
}
|
|
52
|
+
if (diff.behaviorChanges.length > 0) {
|
|
53
|
+
lines.push(cyan('─── Change Details ───'));
|
|
54
|
+
lines.push('');
|
|
55
|
+
const changesByTool = groupChangesByTool(diff.behaviorChanges);
|
|
56
|
+
for (const [tool, changes] of changesByTool) {
|
|
57
|
+
lines.push(` ${bold(tool)}:`);
|
|
58
|
+
for (const change of changes) {
|
|
59
|
+
const sevColor = getSeverityColor(change.severity, useColors);
|
|
60
|
+
lines.push(` ${sevColor(`[${change.severity.toUpperCase()}]`)} ${change.aspect}`);
|
|
61
|
+
if (change.before) {
|
|
62
|
+
lines.push(` ${red('- ' + change.before)}`);
|
|
63
|
+
}
|
|
64
|
+
if (change.after) {
|
|
65
|
+
lines.push(` ${green('+ ' + change.after)}`);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
lines.push('');
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
// Performance regressions
|
|
72
|
+
if (diff.performanceReport?.hasRegressions) {
|
|
73
|
+
lines.push(red('─── Performance Regressions ───'));
|
|
74
|
+
for (const regression of diff.performanceReport.regressions) {
|
|
75
|
+
const percentStr = (regression.regressionPercent * 100).toFixed(1);
|
|
76
|
+
const confidenceNote = regression.isReliable
|
|
77
|
+
? ''
|
|
78
|
+
: ` ${yellow('(low confidence)')}`;
|
|
79
|
+
lines.push(` ${red('!')} ${regression.toolName}: ` +
|
|
80
|
+
`${regression.previousP50Ms.toFixed(0)}ms → ` +
|
|
81
|
+
`${regression.currentP50Ms.toFixed(0)}ms (+${percentStr}%)${confidenceNote}`);
|
|
82
|
+
}
|
|
83
|
+
lines.push('');
|
|
84
|
+
// Show low confidence tools warning
|
|
85
|
+
if (diff.performanceReport.lowConfidenceTools && diff.performanceReport.lowConfidenceTools.length > 0) {
|
|
86
|
+
lines.push(yellow(' Note: Some tools have low confidence metrics.'));
|
|
87
|
+
lines.push(yellow(` Run with more samples for reliable baselines: ${diff.performanceReport.lowConfidenceTools.join(', ')}`));
|
|
88
|
+
lines.push('');
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
else if (diff.performanceReport?.improvementCount ?? 0 > 0) {
|
|
92
|
+
lines.push(green('─── Performance ───'));
|
|
93
|
+
lines.push(` ${green('✓')} ${diff.performanceReport?.improvementCount} tool(s) improved`);
|
|
94
|
+
lines.push('');
|
|
95
|
+
}
|
|
96
|
+
// Performance confidence changes
|
|
97
|
+
if (diff.performanceReport?.confidenceChanges && diff.performanceReport.confidenceChanges.length > 0) {
|
|
98
|
+
lines.push(cyan('─── Confidence Changes ───'));
|
|
99
|
+
for (const change of diff.performanceReport.confidenceChanges) {
|
|
100
|
+
const icon = change.improved ? green('↑') : change.degraded ? yellow('↓') : '→';
|
|
101
|
+
lines.push(` ${icon} ${change.toolName}: ${change.summary}`);
|
|
102
|
+
}
|
|
103
|
+
lines.push('');
|
|
104
|
+
}
|
|
105
|
+
// Security findings
|
|
106
|
+
if (diff.securityReport) {
|
|
107
|
+
const secReport = diff.securityReport;
|
|
108
|
+
if (secReport.degraded || secReport.newFindings.length > 0) {
|
|
109
|
+
lines.push(red('─── Security Findings ───'));
|
|
110
|
+
lines.push(` ${secReport.summary}`);
|
|
111
|
+
lines.push('');
|
|
112
|
+
if (secReport.newFindings.length > 0) {
|
|
113
|
+
lines.push(red(' New Findings:'));
|
|
114
|
+
for (const finding of secReport.newFindings) {
|
|
115
|
+
const riskColor = getRiskLevelColor(finding.riskLevel, useColors);
|
|
116
|
+
lines.push(` ${riskColor('●')} [${finding.riskLevel.toUpperCase()}] ${finding.title}`);
|
|
117
|
+
lines.push(` Tool: ${finding.tool}, Parameter: ${finding.parameter}`);
|
|
118
|
+
lines.push(` ${finding.cweId}: ${finding.description}`);
|
|
119
|
+
}
|
|
120
|
+
lines.push('');
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
else if (secReport.resolvedFindings.length > 0) {
|
|
124
|
+
lines.push(green('─── Security Improvements ───'));
|
|
125
|
+
lines.push(` ${green('✓')} ${secReport.resolvedFindings.length} finding(s) resolved`);
|
|
126
|
+
lines.push('');
|
|
127
|
+
}
|
|
128
|
+
// Show risk score change
|
|
129
|
+
if (secReport.riskScoreChange !== 0) {
|
|
130
|
+
const changeIcon = secReport.riskScoreChange > 0 ? red('↑') : green('↓');
|
|
131
|
+
lines.push(` Risk score: ${secReport.previousRiskScore} → ${secReport.currentRiskScore} (${changeIcon} ${Math.abs(secReport.riskScoreChange)})`);
|
|
132
|
+
lines.push('');
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
// Schema evolution issues
|
|
136
|
+
if (diff.schemaEvolutionReport) {
|
|
137
|
+
const schemaReport = diff.schemaEvolutionReport;
|
|
138
|
+
if (schemaReport.hasBreakingChanges || schemaReport.unstableCount > 0) {
|
|
139
|
+
lines.push(red('─── Schema Evolution Issues ───'));
|
|
140
|
+
lines.push(` ${formatSchemaEvolutionSummary(schemaReport)}`);
|
|
141
|
+
lines.push('');
|
|
142
|
+
for (const issue of schemaReport.toolsWithIssues) {
|
|
143
|
+
const issueIcon = issue.isBreaking ? red('✗') : yellow('⚠');
|
|
144
|
+
lines.push(` ${issueIcon} ${bold(issue.toolName)}`);
|
|
145
|
+
lines.push(` ${issue.summary}`);
|
|
146
|
+
if (issue.fieldsRemoved.length > 0) {
|
|
147
|
+
lines.push(` ${red('- Removed: ' + issue.fieldsRemoved.join(', '))}`);
|
|
148
|
+
}
|
|
149
|
+
if (issue.fieldsAdded.length > 0) {
|
|
150
|
+
lines.push(` ${green('+ Added: ' + issue.fieldsAdded.join(', '))}`);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
lines.push('');
|
|
154
|
+
}
|
|
155
|
+
else if (schemaReport.stableCount > 0) {
|
|
156
|
+
lines.push(green('─── Schema Stability ───'));
|
|
157
|
+
lines.push(` ${green('✓')} ${schemaReport.stableCount} tool(s) with stable response schemas`);
|
|
158
|
+
lines.push('');
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
// Add schema evolution diff lines for modified tools
|
|
162
|
+
for (const toolDiff of diff.toolsModified) {
|
|
163
|
+
if (toolDiff.schemaEvolutionDiff?.structureChanged) {
|
|
164
|
+
const diffLines = formatSchemaEvolutionDiff(toolDiff.schemaEvolutionDiff, useColors);
|
|
165
|
+
if (diffLines.length > 0) {
|
|
166
|
+
lines.push(yellow(`─── ${toolDiff.tool} Schema Evolution ───`));
|
|
167
|
+
lines.push(...diffLines);
|
|
168
|
+
lines.push('');
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
// Error trend report
|
|
173
|
+
if (diff.errorTrendReport) {
|
|
174
|
+
const errorReport = diff.errorTrendReport;
|
|
175
|
+
if (errorReport.significantChange) {
|
|
176
|
+
lines.push(yellow('─── Error Trend Analysis ───'));
|
|
177
|
+
lines.push(formatErrorTrendReport(errorReport, useColors));
|
|
178
|
+
lines.push('');
|
|
179
|
+
}
|
|
180
|
+
else if (errorReport.trends.length > 0) {
|
|
181
|
+
lines.push(green('─── Error Patterns ───'));
|
|
182
|
+
lines.push(` ${green('✓')} Error patterns stable`);
|
|
183
|
+
lines.push('');
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
// Documentation score report
|
|
187
|
+
if (diff.documentationScoreReport) {
|
|
188
|
+
const docReport = diff.documentationScoreReport;
|
|
189
|
+
const indicator = getGradeIndicator(docReport.currentGrade);
|
|
190
|
+
if (docReport.degraded) {
|
|
191
|
+
lines.push(yellow('─── Documentation Quality ───'));
|
|
192
|
+
lines.push(` ${yellow(indicator)} Score: ${docReport.previousScore} → ${docReport.currentScore} (${docReport.change})`);
|
|
193
|
+
lines.push(` ${yellow('Grade:')} ${docReport.previousGrade} → ${docReport.currentGrade}`);
|
|
194
|
+
if (docReport.newIssues > 0) {
|
|
195
|
+
lines.push(` ${red('!')} New issues: ${docReport.newIssues}`);
|
|
196
|
+
}
|
|
197
|
+
lines.push('');
|
|
198
|
+
}
|
|
199
|
+
else if (docReport.improved) {
|
|
200
|
+
lines.push(green('─── Documentation Quality ───'));
|
|
201
|
+
lines.push(` ${green(indicator)} Score: ${docReport.previousScore} → ${docReport.currentScore} (+${docReport.change})`);
|
|
202
|
+
lines.push(` ${green('Grade:')} ${docReport.previousGrade} → ${docReport.currentGrade}`);
|
|
203
|
+
if (docReport.issuesFixed > 0) {
|
|
204
|
+
lines.push(` ${green('✓')} Issues fixed: ${docReport.issuesFixed}`);
|
|
205
|
+
}
|
|
206
|
+
lines.push('');
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
lines.push('─── Statistics ───');
|
|
210
|
+
lines.push(` Breaking changes: ${diff.breakingCount}`);
|
|
211
|
+
lines.push(` Warnings: ${diff.warningCount}`);
|
|
212
|
+
lines.push(` Info: ${diff.infoCount}`);
|
|
213
|
+
if (diff.performanceReport) {
|
|
214
|
+
lines.push(` Performance regressions: ${diff.performanceReport.regressionCount}`);
|
|
215
|
+
if (diff.performanceReport.lowConfidenceTools && diff.performanceReport.lowConfidenceTools.length > 0) {
|
|
216
|
+
lines.push(` Low confidence tools: ${diff.performanceReport.lowConfidenceTools.length}`);
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
if (diff.securityReport) {
|
|
220
|
+
lines.push(` New security findings: ${diff.securityReport.newFindings.length}`);
|
|
221
|
+
lines.push(` Resolved findings: ${diff.securityReport.resolvedFindings.length}`);
|
|
222
|
+
}
|
|
223
|
+
if (diff.schemaEvolutionReport) {
|
|
224
|
+
lines.push(` Schema stability: ${diff.schemaEvolutionReport.stableCount} stable, ${diff.schemaEvolutionReport.unstableCount} unstable`);
|
|
225
|
+
if (diff.schemaEvolutionReport.structureChangedCount > 0) {
|
|
226
|
+
lines.push(` Schema structure changes: ${diff.schemaEvolutionReport.structureChangedCount}`);
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
if (diff.errorTrendReport) {
|
|
230
|
+
const et = diff.errorTrendReport;
|
|
231
|
+
if (et.newCategories.length > 0) {
|
|
232
|
+
lines.push(` New error types: ${et.newCategories.length}`);
|
|
233
|
+
}
|
|
234
|
+
if (et.resolvedCategories.length > 0) {
|
|
235
|
+
lines.push(` Resolved error types: ${et.resolvedCategories.length}`);
|
|
236
|
+
}
|
|
237
|
+
if (et.increasingCategories.length > 0) {
|
|
238
|
+
lines.push(` Increasing errors: ${et.increasingCategories.length}`);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
if (diff.documentationScoreReport) {
|
|
242
|
+
const doc = diff.documentationScoreReport;
|
|
243
|
+
lines.push(` Documentation score: ${doc.currentScore}/100 (${doc.currentGrade})`);
|
|
244
|
+
if (doc.change !== 0) {
|
|
245
|
+
const sign = doc.change > 0 ? '+' : '';
|
|
246
|
+
lines.push(` Documentation change: ${sign}${doc.change}`);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
lines.push('');
|
|
250
|
+
return lines.join('\n');
|
|
251
|
+
}
|
|
252
|
+
/**
|
|
253
|
+
* Format diff as JSON.
|
|
254
|
+
*/
|
|
255
|
+
export function formatDiffJson(diff) {
|
|
256
|
+
return JSON.stringify(diff, null, 2);
|
|
257
|
+
}
|
|
258
|
+
/**
|
|
259
|
+
* Format diff in a compact single-line format for CI logs.
|
|
260
|
+
*/
|
|
261
|
+
export function formatDiffCompact(diff) {
|
|
262
|
+
const parts = [];
|
|
263
|
+
parts.push(`severity=${diff.severity}`);
|
|
264
|
+
parts.push(`breaking=${diff.breakingCount}`);
|
|
265
|
+
parts.push(`warnings=${diff.warningCount}`);
|
|
266
|
+
parts.push(`info=${diff.infoCount}`);
|
|
267
|
+
if (diff.toolsRemoved.length > 0) {
|
|
268
|
+
parts.push(`removed=[${diff.toolsRemoved.join(',')}]`);
|
|
269
|
+
}
|
|
270
|
+
if (diff.toolsAdded.length > 0) {
|
|
271
|
+
parts.push(`added=[${diff.toolsAdded.join(',')}]`);
|
|
272
|
+
}
|
|
273
|
+
if (diff.toolsModified.length > 0) {
|
|
274
|
+
parts.push(`modified=[${diff.toolsModified.map((t) => t.tool).join(',')}]`);
|
|
275
|
+
}
|
|
276
|
+
if (diff.performanceReport?.regressionCount ?? 0 > 0) {
|
|
277
|
+
parts.push(`perf_regressions=${diff.performanceReport?.regressionCount}`);
|
|
278
|
+
}
|
|
279
|
+
if (diff.performanceReport?.lowConfidenceTools && diff.performanceReport.lowConfidenceTools.length > 0) {
|
|
280
|
+
parts.push(`low_confidence_tools=${diff.performanceReport.lowConfidenceTools.length}`);
|
|
281
|
+
}
|
|
282
|
+
if (diff.securityReport) {
|
|
283
|
+
if (diff.securityReport.newFindings.length > 0) {
|
|
284
|
+
parts.push(`new_security_findings=${diff.securityReport.newFindings.length}`);
|
|
285
|
+
}
|
|
286
|
+
if (diff.securityReport.resolvedFindings.length > 0) {
|
|
287
|
+
parts.push(`resolved_findings=${diff.securityReport.resolvedFindings.length}`);
|
|
288
|
+
}
|
|
289
|
+
if (diff.securityReport.degraded) {
|
|
290
|
+
parts.push(`security_degraded=true`);
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
if (diff.schemaEvolutionReport) {
|
|
294
|
+
if (diff.schemaEvolutionReport.unstableCount > 0) {
|
|
295
|
+
parts.push(`schema_unstable=${diff.schemaEvolutionReport.unstableCount}`);
|
|
296
|
+
}
|
|
297
|
+
if (diff.schemaEvolutionReport.structureChangedCount > 0) {
|
|
298
|
+
parts.push(`schema_changed=${diff.schemaEvolutionReport.structureChangedCount}`);
|
|
299
|
+
}
|
|
300
|
+
if (diff.schemaEvolutionReport.hasBreakingChanges) {
|
|
301
|
+
parts.push(`schema_breaking=true`);
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
if (diff.errorTrendReport) {
|
|
305
|
+
if (diff.errorTrendReport.newCategories.length > 0) {
|
|
306
|
+
parts.push(`new_error_types=${diff.errorTrendReport.newCategories.length}`);
|
|
307
|
+
}
|
|
308
|
+
if (diff.errorTrendReport.resolvedCategories.length > 0) {
|
|
309
|
+
parts.push(`resolved_error_types=${diff.errorTrendReport.resolvedCategories.length}`);
|
|
310
|
+
}
|
|
311
|
+
if (diff.errorTrendReport.significantChange) {
|
|
312
|
+
parts.push(`error_trend_change=true`);
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
if (diff.documentationScoreReport) {
|
|
316
|
+
parts.push(`doc_score=${diff.documentationScoreReport.currentScore}`);
|
|
317
|
+
parts.push(`doc_grade=${diff.documentationScoreReport.currentGrade}`);
|
|
318
|
+
if (diff.documentationScoreReport.degraded) {
|
|
319
|
+
parts.push(`doc_degraded=true`);
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
return parts.join(' ');
|
|
323
|
+
}
|
|
324
|
+
/**
|
|
325
|
+
* Format diff for GitHub Actions annotations.
|
|
326
|
+
*/
|
|
327
|
+
export function formatDiffGitHubActions(diff) {
|
|
328
|
+
const lines = [];
|
|
329
|
+
if (diff.severity === 'breaking') {
|
|
330
|
+
lines.push(`::error::Drift detected: ${diff.summary}`);
|
|
331
|
+
}
|
|
332
|
+
else if (diff.severity === 'warning') {
|
|
333
|
+
lines.push(`::warning::Drift detected: ${diff.summary}`);
|
|
334
|
+
}
|
|
335
|
+
else if (diff.severity === 'info') {
|
|
336
|
+
lines.push(`::notice::Minor changes: ${diff.summary}`);
|
|
337
|
+
}
|
|
338
|
+
for (const change of diff.behaviorChanges) {
|
|
339
|
+
const level = change.severity === 'breaking' ? 'error' :
|
|
340
|
+
change.severity === 'warning' ? 'warning' : 'notice';
|
|
341
|
+
lines.push(`::${level}::${change.tool} - ${change.description}`);
|
|
342
|
+
}
|
|
343
|
+
for (const tool of diff.toolsRemoved) {
|
|
344
|
+
lines.push(`::error::Tool removed: ${tool}`);
|
|
345
|
+
}
|
|
346
|
+
for (const tool of diff.toolsAdded) {
|
|
347
|
+
lines.push(`::notice::Tool added: ${tool}`);
|
|
348
|
+
}
|
|
349
|
+
// Performance regressions with confidence
|
|
350
|
+
if (diff.performanceReport?.hasRegressions) {
|
|
351
|
+
for (const regression of diff.performanceReport.regressions) {
|
|
352
|
+
const percentStr = (regression.regressionPercent * 100).toFixed(1);
|
|
353
|
+
const confidenceNote = regression.isReliable ? '' : ' (low confidence)';
|
|
354
|
+
lines.push(`::warning::Performance regression: ${regression.toolName} +${percentStr}%${confidenceNote}`);
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
// Low confidence warning
|
|
358
|
+
if (diff.performanceReport?.lowConfidenceTools && diff.performanceReport.lowConfidenceTools.length > 0) {
|
|
359
|
+
lines.push(`::notice::Low confidence metrics for ${diff.performanceReport.lowConfidenceTools.length} tool(s): ${diff.performanceReport.lowConfidenceTools.join(', ')}`);
|
|
360
|
+
}
|
|
361
|
+
// Security findings
|
|
362
|
+
if (diff.securityReport) {
|
|
363
|
+
for (const finding of diff.securityReport.newFindings) {
|
|
364
|
+
const level = finding.riskLevel === 'critical' || finding.riskLevel === 'high'
|
|
365
|
+
? 'error'
|
|
366
|
+
: finding.riskLevel === 'medium'
|
|
367
|
+
? 'warning'
|
|
368
|
+
: 'notice';
|
|
369
|
+
lines.push(`::${level}::Security [${finding.riskLevel.toUpperCase()}] ${finding.tool}: ${finding.title} (${finding.cweId})`);
|
|
370
|
+
}
|
|
371
|
+
if (diff.securityReport.degraded) {
|
|
372
|
+
lines.push(`::warning::Security posture degraded - ${diff.securityReport.summary}`);
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
// Schema evolution issues
|
|
376
|
+
if (diff.schemaEvolutionReport) {
|
|
377
|
+
for (const issue of diff.schemaEvolutionReport.toolsWithIssues) {
|
|
378
|
+
const level = issue.isBreaking ? 'error' : 'warning';
|
|
379
|
+
lines.push(`::${level}::Schema evolution [${issue.toolName}]: ${issue.summary}`);
|
|
380
|
+
}
|
|
381
|
+
if (diff.schemaEvolutionReport.hasBreakingChanges) {
|
|
382
|
+
lines.push(`::error::Breaking schema changes detected in ${diff.schemaEvolutionReport.structureChangedCount} tool(s)`);
|
|
383
|
+
}
|
|
384
|
+
else if (diff.schemaEvolutionReport.unstableCount > 0) {
|
|
385
|
+
lines.push(`::warning::${diff.schemaEvolutionReport.unstableCount} tool(s) have unstable response schemas`);
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
// Error trend analysis
|
|
389
|
+
if (diff.errorTrendReport) {
|
|
390
|
+
const et = diff.errorTrendReport;
|
|
391
|
+
if (et.newCategories.length > 0) {
|
|
392
|
+
lines.push(`::warning::New error types detected: ${et.newCategories.join(', ')}`);
|
|
393
|
+
}
|
|
394
|
+
if (et.increasingCategories.length > 0) {
|
|
395
|
+
lines.push(`::warning::Increasing errors: ${et.increasingCategories.join(', ')}`);
|
|
396
|
+
}
|
|
397
|
+
if (et.resolvedCategories.length > 0) {
|
|
398
|
+
lines.push(`::notice::Resolved error types: ${et.resolvedCategories.join(', ')}`);
|
|
399
|
+
}
|
|
400
|
+
if (et.significantChange) {
|
|
401
|
+
lines.push(`::warning::Error behavior significantly changed - ${et.summary}`);
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
// Documentation score changes
|
|
405
|
+
if (diff.documentationScoreReport) {
|
|
406
|
+
const doc = diff.documentationScoreReport;
|
|
407
|
+
if (doc.degraded) {
|
|
408
|
+
lines.push(`::warning::Documentation quality degraded: ${doc.previousScore} -> ${doc.currentScore} (${doc.currentGrade})`);
|
|
409
|
+
}
|
|
410
|
+
else if (doc.improved) {
|
|
411
|
+
lines.push(`::notice::Documentation quality improved: ${doc.previousScore} -> ${doc.currentScore} (${doc.currentGrade})`);
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
return lines.join('\n');
|
|
415
|
+
}
|
|
416
|
+
/**
|
|
417
|
+
* Format diff as markdown.
|
|
418
|
+
*/
|
|
419
|
+
export function formatDiffMarkdown(diff) {
|
|
420
|
+
const lines = [];
|
|
421
|
+
lines.push('## Drift Report');
|
|
422
|
+
lines.push('');
|
|
423
|
+
lines.push(`**Severity:** ${getSeverityEmoji(diff.severity)} ${diff.severity.toUpperCase()}`);
|
|
424
|
+
lines.push('');
|
|
425
|
+
lines.push(diff.summary);
|
|
426
|
+
lines.push('');
|
|
427
|
+
if (diff.toolsRemoved.length > 0 || diff.toolsAdded.length > 0 || diff.toolsModified.length > 0) {
|
|
428
|
+
lines.push('### Tool Changes');
|
|
429
|
+
lines.push('');
|
|
430
|
+
lines.push('| Tool | Status | Details |');
|
|
431
|
+
lines.push('|------|--------|---------|');
|
|
432
|
+
for (const tool of diff.toolsRemoved) {
|
|
433
|
+
lines.push(`| ${tool} | ❌ Removed | Breaking change |`);
|
|
434
|
+
}
|
|
435
|
+
for (const tool of diff.toolsAdded) {
|
|
436
|
+
lines.push(`| ${tool} | ✅ Added | New tool |`);
|
|
437
|
+
}
|
|
438
|
+
for (const toolDiff of diff.toolsModified) {
|
|
439
|
+
const details = [
|
|
440
|
+
toolDiff.schemaChanged ? 'Schema changed' : '',
|
|
441
|
+
toolDiff.descriptionChanged ? 'Description changed' : '',
|
|
442
|
+
`${toolDiff.changes.length} change(s)`,
|
|
443
|
+
].filter(Boolean).join(', ');
|
|
444
|
+
lines.push(`| ${toolDiff.tool} | ⚠️ Modified | ${details} |`);
|
|
445
|
+
}
|
|
446
|
+
lines.push('');
|
|
447
|
+
}
|
|
448
|
+
if (diff.behaviorChanges.length > 0) {
|
|
449
|
+
lines.push('### Changes');
|
|
450
|
+
lines.push('');
|
|
451
|
+
lines.push('| Tool | Aspect | Severity | Description |');
|
|
452
|
+
lines.push('|------|--------|----------|-------------|');
|
|
453
|
+
for (const change of diff.behaviorChanges) {
|
|
454
|
+
const sevEmoji = change.severity === 'breaking' ? '🔴' :
|
|
455
|
+
change.severity === 'warning' ? '🟡' : '🟢';
|
|
456
|
+
lines.push(`| ${change.tool} | ${change.aspect} | ${sevEmoji} ${change.severity} | ${change.description} |`);
|
|
457
|
+
}
|
|
458
|
+
lines.push('');
|
|
459
|
+
}
|
|
460
|
+
// Security findings section
|
|
461
|
+
if (diff.securityReport) {
|
|
462
|
+
const secReport = diff.securityReport;
|
|
463
|
+
if (secReport.newFindings.length > 0 || secReport.resolvedFindings.length > 0) {
|
|
464
|
+
lines.push('### Security');
|
|
465
|
+
lines.push('');
|
|
466
|
+
if (secReport.degraded) {
|
|
467
|
+
lines.push(`⚠️ **Security posture degraded**: ${secReport.summary}`);
|
|
468
|
+
lines.push('');
|
|
469
|
+
}
|
|
470
|
+
if (secReport.newFindings.length > 0) {
|
|
471
|
+
lines.push('#### New Findings');
|
|
472
|
+
lines.push('');
|
|
473
|
+
lines.push('| Risk | Tool | Finding | CWE |');
|
|
474
|
+
lines.push('|------|------|---------|-----|');
|
|
475
|
+
for (const finding of secReport.newFindings) {
|
|
476
|
+
const riskEmoji = getRiskLevelEmoji(finding.riskLevel);
|
|
477
|
+
lines.push(`| ${riskEmoji} ${finding.riskLevel} | ${finding.tool} | ${finding.title} | ${finding.cweId} |`);
|
|
478
|
+
}
|
|
479
|
+
lines.push('');
|
|
480
|
+
}
|
|
481
|
+
if (secReport.resolvedFindings.length > 0) {
|
|
482
|
+
lines.push('#### Resolved Findings');
|
|
483
|
+
lines.push('');
|
|
484
|
+
lines.push(`✅ ${secReport.resolvedFindings.length} security finding(s) resolved`);
|
|
485
|
+
lines.push('');
|
|
486
|
+
}
|
|
487
|
+
lines.push(`**Risk Score:** ${secReport.previousRiskScore} → ${secReport.currentRiskScore} (${secReport.riskScoreChange >= 0 ? '+' : ''}${secReport.riskScoreChange})`);
|
|
488
|
+
lines.push('');
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
// Schema evolution section
|
|
492
|
+
if (diff.schemaEvolutionReport) {
|
|
493
|
+
const schemaReport = diff.schemaEvolutionReport;
|
|
494
|
+
if (schemaReport.toolsWithIssues.length > 0 || schemaReport.structureChangedCount > 0) {
|
|
495
|
+
lines.push('### Schema Evolution');
|
|
496
|
+
lines.push('');
|
|
497
|
+
if (schemaReport.hasBreakingChanges) {
|
|
498
|
+
lines.push('⚠️ **Breaking schema changes detected**');
|
|
499
|
+
lines.push('');
|
|
500
|
+
}
|
|
501
|
+
if (schemaReport.toolsWithIssues.length > 0) {
|
|
502
|
+
lines.push('| Tool | Status | Changes |');
|
|
503
|
+
lines.push('|------|--------|---------|');
|
|
504
|
+
for (const issue of schemaReport.toolsWithIssues) {
|
|
505
|
+
const statusIcon = issue.isBreaking ? '🔴' : issue.becameUnstable ? '🟡' : '🔵';
|
|
506
|
+
const status = issue.isBreaking ? 'Breaking' : issue.becameUnstable ? 'Unstable' : 'Changed';
|
|
507
|
+
lines.push(`| ${issue.toolName} | ${statusIcon} ${status} | ${issue.summary} |`);
|
|
508
|
+
}
|
|
509
|
+
lines.push('');
|
|
510
|
+
}
|
|
511
|
+
lines.push(`**Stability:** ${schemaReport.stableCount} stable, ${schemaReport.unstableCount} unstable`);
|
|
512
|
+
if (schemaReport.structureChangedCount > 0) {
|
|
513
|
+
lines.push(`**Structure changes:** ${schemaReport.structureChangedCount} tool(s)`);
|
|
514
|
+
}
|
|
515
|
+
lines.push('');
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
// Error trend section
|
|
519
|
+
if (diff.errorTrendReport) {
|
|
520
|
+
const et = diff.errorTrendReport;
|
|
521
|
+
if (et.significantChange || et.trends.length > 0) {
|
|
522
|
+
lines.push('### Error Trends');
|
|
523
|
+
lines.push('');
|
|
524
|
+
if (et.significantChange) {
|
|
525
|
+
lines.push(`⚠️ **Error behavior changed**: ${et.summary}`);
|
|
526
|
+
lines.push('');
|
|
527
|
+
}
|
|
528
|
+
if (et.newCategories.length > 0 || et.resolvedCategories.length > 0 ||
|
|
529
|
+
et.increasingCategories.length > 0 || et.decreasingCategories.length > 0) {
|
|
530
|
+
lines.push('| Category | Trend | Previous | Current | Change |');
|
|
531
|
+
lines.push('|----------|-------|----------|---------|--------|');
|
|
532
|
+
for (const trend of et.trends.filter(t => t.trend !== 'stable')) {
|
|
533
|
+
const trendEmoji = getTrendEmoji(trend.trend);
|
|
534
|
+
const changeStr = trend.changePercent !== 0
|
|
535
|
+
? `${trend.changePercent > 0 ? '+' : ''}${trend.changePercent}%`
|
|
536
|
+
: '-';
|
|
537
|
+
lines.push(`| ${trend.category} | ${trendEmoji} ${trend.trend} | ${trend.previousCount} | ${trend.currentCount} | ${changeStr} |`);
|
|
538
|
+
}
|
|
539
|
+
lines.push('');
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
// Performance section
|
|
544
|
+
if (diff.performanceReport) {
|
|
545
|
+
const perfReport = diff.performanceReport;
|
|
546
|
+
if (perfReport.hasRegressions || perfReport.improvementCount > 0 ||
|
|
547
|
+
(perfReport.lowConfidenceTools && perfReport.lowConfidenceTools.length > 0)) {
|
|
548
|
+
lines.push('### Performance');
|
|
549
|
+
lines.push('');
|
|
550
|
+
if (perfReport.hasRegressions) {
|
|
551
|
+
lines.push('#### Regressions');
|
|
552
|
+
lines.push('');
|
|
553
|
+
lines.push('| Tool | Previous | Current | Change | Confidence |');
|
|
554
|
+
lines.push('|------|----------|---------|--------|------------|');
|
|
555
|
+
for (const regression of perfReport.regressions) {
|
|
556
|
+
const percentStr = (regression.regressionPercent * 100).toFixed(1);
|
|
557
|
+
const confidenceEmoji = regression.isReliable ? '✓' : '⚠️';
|
|
558
|
+
const confidenceLabel = regression.currentConfidence ?? 'unknown';
|
|
559
|
+
lines.push(`| ${regression.toolName} | ${regression.previousP50Ms.toFixed(0)}ms | ${regression.currentP50Ms.toFixed(0)}ms | +${percentStr}% | ${confidenceEmoji} ${confidenceLabel} |`);
|
|
560
|
+
}
|
|
561
|
+
lines.push('');
|
|
562
|
+
}
|
|
563
|
+
if (perfReport.lowConfidenceTools && perfReport.lowConfidenceTools.length > 0) {
|
|
564
|
+
lines.push(`> **⚠️ Low confidence metrics**: ${perfReport.lowConfidenceTools.join(', ')}`);
|
|
565
|
+
lines.push('> Consider running with more samples for reliable baselines.');
|
|
566
|
+
lines.push('');
|
|
567
|
+
}
|
|
568
|
+
if (perfReport.confidenceChanges && perfReport.confidenceChanges.length > 0) {
|
|
569
|
+
lines.push('#### Confidence Changes');
|
|
570
|
+
lines.push('');
|
|
571
|
+
lines.push('| Tool | Previous | Current | Status |');
|
|
572
|
+
lines.push('|------|----------|---------|--------|');
|
|
573
|
+
for (const change of perfReport.confidenceChanges) {
|
|
574
|
+
const statusEmoji = change.improved ? '📈' : change.degraded ? '📉' : '➡️';
|
|
575
|
+
lines.push(`| ${change.toolName} | ${change.previousLevel ?? 'N/A'} | ${change.currentLevel} | ${statusEmoji} ${change.improved ? 'Improved' : change.degraded ? 'Degraded' : 'Changed'} |`);
|
|
576
|
+
}
|
|
577
|
+
lines.push('');
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
// Documentation quality section
|
|
582
|
+
if (diff.documentationScoreReport) {
|
|
583
|
+
const doc = diff.documentationScoreReport;
|
|
584
|
+
lines.push('### Documentation Quality');
|
|
585
|
+
lines.push('');
|
|
586
|
+
const changeIcon = doc.improved ? '📈' : doc.degraded ? '📉' : '➡️';
|
|
587
|
+
const sign = doc.change > 0 ? '+' : '';
|
|
588
|
+
lines.push(`**Score:** ${doc.currentScore}/100 (${doc.currentGrade}) ${changeIcon}`);
|
|
589
|
+
if (doc.change !== 0) {
|
|
590
|
+
lines.push(`**Change:** ${doc.previousScore} → ${doc.currentScore} (${sign}${doc.change})`);
|
|
591
|
+
lines.push(`**Grade:** ${doc.previousGrade} → ${doc.currentGrade}`);
|
|
592
|
+
}
|
|
593
|
+
if (doc.issuesFixed > 0) {
|
|
594
|
+
lines.push(`**Issues fixed:** ${doc.issuesFixed}`);
|
|
595
|
+
}
|
|
596
|
+
if (doc.newIssues > 0) {
|
|
597
|
+
lines.push(`**New issues:** ${doc.newIssues}`);
|
|
598
|
+
}
|
|
599
|
+
lines.push('');
|
|
600
|
+
}
|
|
601
|
+
lines.push('### Statistics');
|
|
602
|
+
lines.push('');
|
|
603
|
+
lines.push(`- Breaking changes: **${diff.breakingCount}**`);
|
|
604
|
+
lines.push(`- Warnings: **${diff.warningCount}**`);
|
|
605
|
+
lines.push(`- Info: **${diff.infoCount}**`);
|
|
606
|
+
if (diff.performanceReport) {
|
|
607
|
+
lines.push(`- Performance regressions: **${diff.performanceReport.regressionCount}**`);
|
|
608
|
+
if (diff.performanceReport.lowConfidenceTools && diff.performanceReport.lowConfidenceTools.length > 0) {
|
|
609
|
+
lines.push(`- Low confidence tools: **${diff.performanceReport.lowConfidenceTools.length}**`);
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
if (diff.securityReport) {
|
|
613
|
+
lines.push(`- New security findings: **${diff.securityReport.newFindings.length}**`);
|
|
614
|
+
lines.push(`- Resolved findings: **${diff.securityReport.resolvedFindings.length}**`);
|
|
615
|
+
}
|
|
616
|
+
if (diff.schemaEvolutionReport) {
|
|
617
|
+
lines.push(`- Stable schemas: **${diff.schemaEvolutionReport.stableCount}**`);
|
|
618
|
+
lines.push(`- Unstable schemas: **${diff.schemaEvolutionReport.unstableCount}**`);
|
|
619
|
+
if (diff.schemaEvolutionReport.structureChangedCount > 0) {
|
|
620
|
+
lines.push(`- Schema structure changes: **${diff.schemaEvolutionReport.structureChangedCount}**`);
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
if (diff.errorTrendReport) {
|
|
624
|
+
const et = diff.errorTrendReport;
|
|
625
|
+
if (et.newCategories.length > 0) {
|
|
626
|
+
lines.push(`- New error types: **${et.newCategories.length}**`);
|
|
627
|
+
}
|
|
628
|
+
if (et.resolvedCategories.length > 0) {
|
|
629
|
+
lines.push(`- Resolved error types: **${et.resolvedCategories.length}**`);
|
|
630
|
+
}
|
|
631
|
+
if (et.increasingCategories.length > 0) {
|
|
632
|
+
lines.push(`- Increasing errors: **${et.increasingCategories.length}**`);
|
|
633
|
+
}
|
|
634
|
+
if (et.decreasingCategories.length > 0) {
|
|
635
|
+
lines.push(`- Decreasing errors: **${et.decreasingCategories.length}**`);
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
if (diff.documentationScoreReport) {
|
|
639
|
+
const doc = diff.documentationScoreReport;
|
|
640
|
+
lines.push(`- Documentation score: **${doc.currentScore}/100** (${doc.currentGrade})`);
|
|
641
|
+
if (doc.change !== 0) {
|
|
642
|
+
const sign = doc.change > 0 ? '+' : '';
|
|
643
|
+
lines.push(`- Documentation change: **${sign}${doc.change}**`);
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
return lines.join('\n');
|
|
647
|
+
}
|
|
648
|
+
/**
|
|
649
|
+
* Get emoji for error trend direction.
|
|
650
|
+
*/
|
|
651
|
+
function getTrendEmoji(trend) {
|
|
652
|
+
switch (trend) {
|
|
653
|
+
case 'new':
|
|
654
|
+
return '🆕';
|
|
655
|
+
case 'resolved':
|
|
656
|
+
return '✅';
|
|
657
|
+
case 'increasing':
|
|
658
|
+
return '📈';
|
|
659
|
+
case 'decreasing':
|
|
660
|
+
return '📉';
|
|
661
|
+
case 'stable':
|
|
662
|
+
return '➡️';
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
/**
|
|
666
|
+
* Format diff as JUnit XML for CI dashboard integration.
|
|
667
|
+
*
|
|
668
|
+
* JUnit XML is widely supported by CI/CD systems (Jenkins, GitLab CI,
|
|
669
|
+
* CircleCI, Azure DevOps, etc.) for test result visualization.
|
|
670
|
+
*
|
|
671
|
+
* @param diff - The behavioral diff to format
|
|
672
|
+
* @param suiteName - Name for the test suite (default: 'bellwether')
|
|
673
|
+
* @returns JUnit XML string
|
|
674
|
+
*/
|
|
675
|
+
export function formatDiffJUnit(diff, suiteName = 'bellwether') {
|
|
676
|
+
const timestamp = new Date().toISOString();
|
|
677
|
+
const totalTests = diff.toolsAdded.length +
|
|
678
|
+
diff.toolsRemoved.length +
|
|
679
|
+
diff.behaviorChanges.length;
|
|
680
|
+
const failures = diff.breakingCount;
|
|
681
|
+
const errors = 0;
|
|
682
|
+
const skipped = 0;
|
|
683
|
+
const lines = [
|
|
684
|
+
'<?xml version="1.0" encoding="UTF-8"?>',
|
|
685
|
+
`<testsuites name="${escapeXml(suiteName)}" tests="${totalTests}" failures="${failures}" errors="${errors}" skipped="${skipped}" timestamp="${timestamp}">`,
|
|
686
|
+
` <testsuite name="drift-detection" tests="${totalTests}" failures="${failures}" errors="${errors}" skipped="${skipped}">`,
|
|
687
|
+
];
|
|
688
|
+
// Test case for each removed tool (failure - breaking)
|
|
689
|
+
for (const tool of diff.toolsRemoved) {
|
|
690
|
+
const name = escapeXml(`tool-present-${tool}`);
|
|
691
|
+
lines.push(` <testcase name="${name}" classname="drift.tools">`);
|
|
692
|
+
lines.push(` <failure message="Tool removed: ${escapeXml(tool)}" type="breaking">`);
|
|
693
|
+
lines.push(`Tool "${escapeXml(tool)}" was present in baseline but is now missing.`);
|
|
694
|
+
lines.push('This is a breaking change that may affect consumers.');
|
|
695
|
+
lines.push(' </failure>');
|
|
696
|
+
lines.push(' </testcase>');
|
|
697
|
+
}
|
|
698
|
+
// Test case for each added tool (passes - info)
|
|
699
|
+
for (const tool of diff.toolsAdded) {
|
|
700
|
+
const name = escapeXml(`tool-new-${tool}`);
|
|
701
|
+
lines.push(` <testcase name="${name}" classname="drift.tools">`);
|
|
702
|
+
lines.push(` <system-out>New tool added: ${escapeXml(tool)}</system-out>`);
|
|
703
|
+
lines.push(' </testcase>');
|
|
704
|
+
}
|
|
705
|
+
// Test case for each behavior change
|
|
706
|
+
for (const change of diff.behaviorChanges) {
|
|
707
|
+
const name = escapeXml(`${change.tool}-${change.aspect}`);
|
|
708
|
+
lines.push(` <testcase name="${name}" classname="drift.behavior">`);
|
|
709
|
+
if (change.severity === 'breaking') {
|
|
710
|
+
lines.push(` <failure message="${escapeXml(change.description)}" type="breaking">`);
|
|
711
|
+
lines.push(`Tool: ${escapeXml(change.tool)}`);
|
|
712
|
+
lines.push(`Aspect: ${escapeXml(change.aspect)}`);
|
|
713
|
+
if (change.before) {
|
|
714
|
+
lines.push(`Before: ${escapeXml(change.before)}`);
|
|
715
|
+
}
|
|
716
|
+
if (change.after) {
|
|
717
|
+
lines.push(`After: ${escapeXml(change.after)}`);
|
|
718
|
+
}
|
|
719
|
+
lines.push(' </failure>');
|
|
720
|
+
}
|
|
721
|
+
else if (change.severity === 'warning') {
|
|
722
|
+
lines.push(` <system-err>[WARNING] ${escapeXml(change.description)}</system-err>`);
|
|
723
|
+
}
|
|
724
|
+
else {
|
|
725
|
+
lines.push(` <system-out>[INFO] ${escapeXml(change.description)}</system-out>`);
|
|
726
|
+
}
|
|
727
|
+
lines.push(' </testcase>');
|
|
728
|
+
}
|
|
729
|
+
// Test cases for performance regressions
|
|
730
|
+
if (diff.performanceReport) {
|
|
731
|
+
for (const regression of diff.performanceReport.regressions) {
|
|
732
|
+
const name = escapeXml(`performance-${regression.toolName}`);
|
|
733
|
+
const percentStr = (regression.regressionPercent * 100).toFixed(1);
|
|
734
|
+
const confidenceNote = regression.isReliable ? '' : ' (low confidence)';
|
|
735
|
+
lines.push(` <testcase name="${name}" classname="drift.performance">`);
|
|
736
|
+
lines.push(` <failure message="Performance regression: +${percentStr}%${confidenceNote}" type="regression">`);
|
|
737
|
+
lines.push(`Tool: ${escapeXml(regression.toolName)}`);
|
|
738
|
+
lines.push(`Previous p50: ${regression.previousP50Ms.toFixed(0)}ms`);
|
|
739
|
+
lines.push(`Current p50: ${regression.currentP50Ms.toFixed(0)}ms`);
|
|
740
|
+
lines.push(`Regression: +${percentStr}%`);
|
|
741
|
+
lines.push(`Confidence: ${regression.currentConfidence ?? 'unknown'}${regression.isReliable ? '' : ' (unreliable)'}`);
|
|
742
|
+
lines.push(' </failure>');
|
|
743
|
+
lines.push(' </testcase>');
|
|
744
|
+
}
|
|
745
|
+
// Low confidence tools
|
|
746
|
+
if (diff.performanceReport.lowConfidenceTools && diff.performanceReport.lowConfidenceTools.length > 0) {
|
|
747
|
+
for (const tool of diff.performanceReport.lowConfidenceTools) {
|
|
748
|
+
const name = escapeXml(`confidence-${tool}`);
|
|
749
|
+
lines.push(` <testcase name="${name}" classname="drift.confidence">`);
|
|
750
|
+
lines.push(` <system-err>[NOTICE] Low confidence metrics for ${escapeXml(tool)}. Run with more samples for reliable baselines.</system-err>`);
|
|
751
|
+
lines.push(' </testcase>');
|
|
752
|
+
}
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
// Test cases for security findings
|
|
756
|
+
if (diff.securityReport) {
|
|
757
|
+
for (const finding of diff.securityReport.newFindings) {
|
|
758
|
+
const name = escapeXml(`security-${finding.tool}-${finding.category}`);
|
|
759
|
+
const isFailure = finding.riskLevel === 'critical' || finding.riskLevel === 'high';
|
|
760
|
+
lines.push(` <testcase name="${name}" classname="drift.security">`);
|
|
761
|
+
if (isFailure) {
|
|
762
|
+
lines.push(` <failure message="${escapeXml(finding.title)}" type="${finding.riskLevel}">`);
|
|
763
|
+
lines.push(`Tool: ${escapeXml(finding.tool)}`);
|
|
764
|
+
lines.push(`Parameter: ${escapeXml(finding.parameter)}`);
|
|
765
|
+
lines.push(`CWE: ${escapeXml(finding.cweId)}`);
|
|
766
|
+
lines.push(`Description: ${escapeXml(finding.description)}`);
|
|
767
|
+
lines.push(`Remediation: ${escapeXml(finding.remediation)}`);
|
|
768
|
+
lines.push(' </failure>');
|
|
769
|
+
}
|
|
770
|
+
else {
|
|
771
|
+
lines.push(` <system-err>[${finding.riskLevel.toUpperCase()}] ${escapeXml(finding.title)}</system-err>`);
|
|
772
|
+
}
|
|
773
|
+
lines.push(' </testcase>');
|
|
774
|
+
}
|
|
775
|
+
// Show resolved findings as passing tests
|
|
776
|
+
for (const finding of diff.securityReport.resolvedFindings) {
|
|
777
|
+
const name = escapeXml(`security-resolved-${finding.tool}-${finding.category}`);
|
|
778
|
+
lines.push(` <testcase name="${name}" classname="drift.security">`);
|
|
779
|
+
lines.push(` <system-out>Resolved: ${escapeXml(finding.title)} (${escapeXml(finding.cweId)})</system-out>`);
|
|
780
|
+
lines.push(' </testcase>');
|
|
781
|
+
}
|
|
782
|
+
}
|
|
783
|
+
// Test cases for schema evolution issues
|
|
784
|
+
if (diff.schemaEvolutionReport) {
|
|
785
|
+
for (const issue of diff.schemaEvolutionReport.toolsWithIssues) {
|
|
786
|
+
const name = escapeXml(`schema-evolution-${issue.toolName}`);
|
|
787
|
+
lines.push(` <testcase name="${name}" classname="drift.schema">`);
|
|
788
|
+
if (issue.isBreaking) {
|
|
789
|
+
lines.push(` <failure message="${escapeXml(issue.summary)}" type="breaking">`);
|
|
790
|
+
lines.push(`Tool: ${escapeXml(issue.toolName)}`);
|
|
791
|
+
if (issue.fieldsRemoved.length > 0) {
|
|
792
|
+
lines.push(`Fields removed: ${escapeXml(issue.fieldsRemoved.join(', '))}`);
|
|
793
|
+
}
|
|
794
|
+
if (issue.fieldsAdded.length > 0) {
|
|
795
|
+
lines.push(`Fields added: ${escapeXml(issue.fieldsAdded.join(', '))}`);
|
|
796
|
+
}
|
|
797
|
+
lines.push(' </failure>');
|
|
798
|
+
}
|
|
799
|
+
else if (issue.becameUnstable) {
|
|
800
|
+
lines.push(` <system-err>[WARNING] Schema became unstable: ${escapeXml(issue.summary)}</system-err>`);
|
|
801
|
+
}
|
|
802
|
+
else {
|
|
803
|
+
lines.push(` <system-out>[INFO] Schema changed: ${escapeXml(issue.summary)}</system-out>`);
|
|
804
|
+
}
|
|
805
|
+
lines.push(' </testcase>');
|
|
806
|
+
}
|
|
807
|
+
// Show stable schemas as passing tests
|
|
808
|
+
if (diff.schemaEvolutionReport.stableCount > 0 && diff.schemaEvolutionReport.toolsWithIssues.length === 0) {
|
|
809
|
+
lines.push(` <testcase name="schema-stability-check" classname="drift.schema">`);
|
|
810
|
+
lines.push(` <system-out>${diff.schemaEvolutionReport.stableCount} tool(s) have stable response schemas</system-out>`);
|
|
811
|
+
lines.push(' </testcase>');
|
|
812
|
+
}
|
|
813
|
+
}
|
|
814
|
+
// Test cases for error trend changes
|
|
815
|
+
if (diff.errorTrendReport) {
|
|
816
|
+
const et = diff.errorTrendReport;
|
|
817
|
+
// New error types (warnings)
|
|
818
|
+
for (const category of et.newCategories) {
|
|
819
|
+
const name = escapeXml(`error-trend-new-${category}`);
|
|
820
|
+
lines.push(` <testcase name="${name}" classname="drift.errors">`);
|
|
821
|
+
lines.push(` <system-err>[WARNING] New error type detected: ${escapeXml(category)}</system-err>`);
|
|
822
|
+
lines.push(' </testcase>');
|
|
823
|
+
}
|
|
824
|
+
// Resolved error types (info)
|
|
825
|
+
for (const category of et.resolvedCategories) {
|
|
826
|
+
const name = escapeXml(`error-trend-resolved-${category}`);
|
|
827
|
+
lines.push(` <testcase name="${name}" classname="drift.errors">`);
|
|
828
|
+
lines.push(` <system-out>Resolved: ${escapeXml(category)} error type no longer occurs</system-out>`);
|
|
829
|
+
lines.push(' </testcase>');
|
|
830
|
+
}
|
|
831
|
+
// Increasing error types (warnings)
|
|
832
|
+
for (const category of et.increasingCategories) {
|
|
833
|
+
const trend = et.trends.find(t => t.category === category);
|
|
834
|
+
const name = escapeXml(`error-trend-increasing-${category}`);
|
|
835
|
+
lines.push(` <testcase name="${name}" classname="drift.errors">`);
|
|
836
|
+
lines.push(` <system-err>[WARNING] Error frequency increasing: ${escapeXml(category)}${trend ? ` (+${trend.changePercent}%)` : ''}</system-err>`);
|
|
837
|
+
lines.push(' </testcase>');
|
|
838
|
+
}
|
|
839
|
+
// Overall error trend summary
|
|
840
|
+
if (et.significantChange) {
|
|
841
|
+
lines.push(` <testcase name="error-trend-summary" classname="drift.errors">`);
|
|
842
|
+
lines.push(` <system-err>[WARNING] ${escapeXml(et.summary)}</system-err>`);
|
|
843
|
+
lines.push(' </testcase>');
|
|
844
|
+
}
|
|
845
|
+
else if (et.trends.length > 0) {
|
|
846
|
+
lines.push(` <testcase name="error-trend-summary" classname="drift.errors">`);
|
|
847
|
+
lines.push(` <system-out>Error patterns stable</system-out>`);
|
|
848
|
+
lines.push(' </testcase>');
|
|
849
|
+
}
|
|
850
|
+
}
|
|
851
|
+
// Test case for documentation quality
|
|
852
|
+
if (diff.documentationScoreReport) {
|
|
853
|
+
const doc = diff.documentationScoreReport;
|
|
854
|
+
const name = 'documentation-quality-score';
|
|
855
|
+
lines.push(` <testcase name="${name}" classname="drift.documentation">`);
|
|
856
|
+
if (doc.degraded) {
|
|
857
|
+
lines.push(` <system-err>[WARNING] Documentation quality degraded: ${doc.previousScore} -> ${doc.currentScore} (${doc.currentGrade})</system-err>`);
|
|
858
|
+
if (doc.newIssues > 0) {
|
|
859
|
+
lines.push(` <system-err>New documentation issues: ${doc.newIssues}</system-err>`);
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
else if (doc.improved) {
|
|
863
|
+
lines.push(` <system-out>Documentation quality improved: ${doc.previousScore} -> ${doc.currentScore} (${doc.currentGrade})</system-out>`);
|
|
864
|
+
if (doc.issuesFixed > 0) {
|
|
865
|
+
lines.push(` <system-out>Issues fixed: ${doc.issuesFixed}</system-out>`);
|
|
866
|
+
}
|
|
867
|
+
}
|
|
868
|
+
else {
|
|
869
|
+
lines.push(` <system-out>Documentation quality: ${doc.currentScore}/100 (${doc.currentGrade})</system-out>`);
|
|
870
|
+
}
|
|
871
|
+
lines.push(' </testcase>');
|
|
872
|
+
}
|
|
873
|
+
lines.push(' </testsuite>');
|
|
874
|
+
lines.push('</testsuites>');
|
|
875
|
+
return lines.join('\n');
|
|
876
|
+
}
|
|
877
|
+
/**
|
|
878
|
+
* Format diff as SARIF (Static Analysis Results Interchange Format) for GitHub Code Scanning.
|
|
879
|
+
*
|
|
880
|
+
* SARIF is the standard format for GitHub's code scanning feature and can be
|
|
881
|
+
* uploaded to show drift detection results in pull request reviews.
|
|
882
|
+
*
|
|
883
|
+
* @see https://docs.oasis-open.org/sarif/sarif/v2.1.0/sarif-v2.1.0.html
|
|
884
|
+
*
|
|
885
|
+
* @param diff - The behavioral diff to format
|
|
886
|
+
* @param baselinePath - Path to the baseline file (for location references)
|
|
887
|
+
* @returns SARIF JSON string
|
|
888
|
+
*/
|
|
889
|
+
export function formatDiffSarif(diff, baselinePath = 'bellwether-baseline.json') {
|
|
890
|
+
const sarif = {
|
|
891
|
+
$schema: 'https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json',
|
|
892
|
+
version: '2.1.0',
|
|
893
|
+
runs: [
|
|
894
|
+
{
|
|
895
|
+
tool: {
|
|
896
|
+
driver: {
|
|
897
|
+
name: 'bellwether',
|
|
898
|
+
version: '1.0.0',
|
|
899
|
+
informationUri: 'https://github.com/dotsetlabs/bellwether',
|
|
900
|
+
rules: [
|
|
901
|
+
{
|
|
902
|
+
id: 'BWH001',
|
|
903
|
+
name: 'ToolRemoved',
|
|
904
|
+
shortDescription: { text: 'Tool was removed from server' },
|
|
905
|
+
fullDescription: {
|
|
906
|
+
text: 'A tool that existed in the baseline is no longer present. This is a breaking change that may affect consumers relying on this tool.',
|
|
907
|
+
},
|
|
908
|
+
defaultConfiguration: { level: 'error' },
|
|
909
|
+
help: {
|
|
910
|
+
text: 'Ensure the tool removal was intentional and update consumers. Consider deprecation warnings before removal.',
|
|
911
|
+
},
|
|
912
|
+
},
|
|
913
|
+
{
|
|
914
|
+
id: 'BWH002',
|
|
915
|
+
name: 'SchemaBreakingChange',
|
|
916
|
+
shortDescription: { text: 'Breaking schema change detected' },
|
|
917
|
+
fullDescription: {
|
|
918
|
+
text: 'A breaking change was detected in a tool schema, such as a removed parameter, type change, or new required field.',
|
|
919
|
+
},
|
|
920
|
+
defaultConfiguration: { level: 'error' },
|
|
921
|
+
help: {
|
|
922
|
+
text: 'Review schema changes and update consumers accordingly. Breaking changes require version bumps.',
|
|
923
|
+
},
|
|
924
|
+
},
|
|
925
|
+
{
|
|
926
|
+
id: 'BWH003',
|
|
927
|
+
name: 'SchemaWarningChange',
|
|
928
|
+
shortDescription: { text: 'Schema warning change detected' },
|
|
929
|
+
fullDescription: {
|
|
930
|
+
text: 'A warning-level change was detected in a tool schema that may affect some consumers.',
|
|
931
|
+
},
|
|
932
|
+
defaultConfiguration: { level: 'warning' },
|
|
933
|
+
help: {
|
|
934
|
+
text: 'Review schema changes for potential impact on consumers.',
|
|
935
|
+
},
|
|
936
|
+
},
|
|
937
|
+
{
|
|
938
|
+
id: 'BWH004',
|
|
939
|
+
name: 'ToolAdded',
|
|
940
|
+
shortDescription: { text: 'New tool added to server' },
|
|
941
|
+
fullDescription: {
|
|
942
|
+
text: 'A new tool was added that did not exist in the baseline. This is typically safe but should be documented.',
|
|
943
|
+
},
|
|
944
|
+
defaultConfiguration: { level: 'note' },
|
|
945
|
+
help: {
|
|
946
|
+
text: 'Document the new tool and notify consumers of new functionality.',
|
|
947
|
+
},
|
|
948
|
+
},
|
|
949
|
+
{
|
|
950
|
+
id: 'BWH005',
|
|
951
|
+
name: 'ResponseStructureChanged',
|
|
952
|
+
shortDescription: { text: 'Response structure changed' },
|
|
953
|
+
fullDescription: {
|
|
954
|
+
text: 'The structure of a tool response has changed, which may affect consumers parsing the response.',
|
|
955
|
+
},
|
|
956
|
+
defaultConfiguration: { level: 'warning' },
|
|
957
|
+
help: {
|
|
958
|
+
text: 'Review response structure changes and update consumers that depend on specific fields.',
|
|
959
|
+
},
|
|
960
|
+
},
|
|
961
|
+
{
|
|
962
|
+
id: 'BWH006',
|
|
963
|
+
name: 'ErrorPatternChanged',
|
|
964
|
+
shortDescription: { text: 'Error pattern changed' },
|
|
965
|
+
fullDescription: {
|
|
966
|
+
text: 'The error behavior of a tool has changed, with new or modified error patterns.',
|
|
967
|
+
},
|
|
968
|
+
defaultConfiguration: { level: 'warning' },
|
|
969
|
+
help: {
|
|
970
|
+
text: 'Review error handling changes and ensure consumers handle new error cases.',
|
|
971
|
+
},
|
|
972
|
+
},
|
|
973
|
+
{
|
|
974
|
+
id: 'BWH007',
|
|
975
|
+
name: 'SecurityFinding',
|
|
976
|
+
shortDescription: { text: 'Security vulnerability detected' },
|
|
977
|
+
fullDescription: {
|
|
978
|
+
text: 'A security vulnerability was detected during testing. The tool may be susceptible to injection attacks or other security issues.',
|
|
979
|
+
},
|
|
980
|
+
defaultConfiguration: { level: 'error' },
|
|
981
|
+
help: {
|
|
982
|
+
text: 'Review the security finding and implement appropriate input validation, sanitization, or other mitigations.',
|
|
983
|
+
},
|
|
984
|
+
},
|
|
985
|
+
{
|
|
986
|
+
id: 'BWH008',
|
|
987
|
+
name: 'SchemaEvolutionBreaking',
|
|
988
|
+
shortDescription: { text: 'Breaking response schema change' },
|
|
989
|
+
fullDescription: {
|
|
990
|
+
text: 'A breaking change was detected in a tool response schema, such as removed fields or type changes that may affect consumers.',
|
|
991
|
+
},
|
|
992
|
+
defaultConfiguration: { level: 'error' },
|
|
993
|
+
help: {
|
|
994
|
+
text: 'Review the schema changes and update consumers accordingly. Consider versioning or migration paths for breaking changes.',
|
|
995
|
+
},
|
|
996
|
+
},
|
|
997
|
+
{
|
|
998
|
+
id: 'BWH009',
|
|
999
|
+
name: 'SchemaEvolutionUnstable',
|
|
1000
|
+
shortDescription: { text: 'Unstable response schema detected' },
|
|
1001
|
+
fullDescription: {
|
|
1002
|
+
text: 'The tool response schema is inconsistent across samples, indicating potential reliability issues.',
|
|
1003
|
+
},
|
|
1004
|
+
defaultConfiguration: { level: 'warning' },
|
|
1005
|
+
help: {
|
|
1006
|
+
text: 'Investigate why the response schema varies. Consider normalizing responses or documenting optional fields.',
|
|
1007
|
+
},
|
|
1008
|
+
},
|
|
1009
|
+
{
|
|
1010
|
+
id: 'BWH010',
|
|
1011
|
+
name: 'ErrorTrendNew',
|
|
1012
|
+
shortDescription: { text: 'New error type detected' },
|
|
1013
|
+
fullDescription: {
|
|
1014
|
+
text: 'A new type of error was detected that did not occur in the previous baseline.',
|
|
1015
|
+
},
|
|
1016
|
+
defaultConfiguration: { level: 'warning' },
|
|
1017
|
+
help: {
|
|
1018
|
+
text: 'Review the new error type and ensure consumers handle it appropriately.',
|
|
1019
|
+
},
|
|
1020
|
+
},
|
|
1021
|
+
{
|
|
1022
|
+
id: 'BWH011',
|
|
1023
|
+
name: 'ErrorTrendIncreasing',
|
|
1024
|
+
shortDescription: { text: 'Error frequency increasing' },
|
|
1025
|
+
fullDescription: {
|
|
1026
|
+
text: 'The frequency of a specific error type has significantly increased compared to the baseline.',
|
|
1027
|
+
},
|
|
1028
|
+
defaultConfiguration: { level: 'warning' },
|
|
1029
|
+
help: {
|
|
1030
|
+
text: 'Investigate why errors are increasing. This may indicate a regression or environmental issue.',
|
|
1031
|
+
},
|
|
1032
|
+
},
|
|
1033
|
+
{
|
|
1034
|
+
id: 'BWH012',
|
|
1035
|
+
name: 'PerformanceRegression',
|
|
1036
|
+
shortDescription: { text: 'Performance regression detected' },
|
|
1037
|
+
fullDescription: {
|
|
1038
|
+
text: 'A tool has significantly slower response times compared to the baseline.',
|
|
1039
|
+
},
|
|
1040
|
+
defaultConfiguration: { level: 'warning' },
|
|
1041
|
+
help: {
|
|
1042
|
+
text: 'Investigate the cause of the performance regression. Review recent changes that may have impacted response times.',
|
|
1043
|
+
},
|
|
1044
|
+
},
|
|
1045
|
+
{
|
|
1046
|
+
id: 'BWH013',
|
|
1047
|
+
name: 'LowConfidenceMetrics',
|
|
1048
|
+
shortDescription: { text: 'Low confidence performance metrics' },
|
|
1049
|
+
fullDescription: {
|
|
1050
|
+
text: 'Performance metrics have low statistical confidence due to insufficient samples or high variability.',
|
|
1051
|
+
},
|
|
1052
|
+
defaultConfiguration: { level: 'note' },
|
|
1053
|
+
help: {
|
|
1054
|
+
text: 'Run with more samples (--samples flag) to establish reliable performance baselines.',
|
|
1055
|
+
},
|
|
1056
|
+
},
|
|
1057
|
+
{
|
|
1058
|
+
id: 'BWH014',
|
|
1059
|
+
name: 'DocumentationQualityDegraded',
|
|
1060
|
+
shortDescription: { text: 'Documentation quality degraded' },
|
|
1061
|
+
fullDescription: {
|
|
1062
|
+
text: 'The documentation quality score has decreased compared to the baseline, indicating potential documentation issues.',
|
|
1063
|
+
},
|
|
1064
|
+
defaultConfiguration: { level: 'warning' },
|
|
1065
|
+
help: {
|
|
1066
|
+
text: 'Review tool and parameter descriptions. Ensure all tools have meaningful descriptions and parameters are documented.',
|
|
1067
|
+
},
|
|
1068
|
+
},
|
|
1069
|
+
{
|
|
1070
|
+
id: 'BWH015',
|
|
1071
|
+
name: 'DocumentationQualityLow',
|
|
1072
|
+
shortDescription: { text: 'Low documentation quality score' },
|
|
1073
|
+
fullDescription: {
|
|
1074
|
+
text: 'The documentation quality score is below acceptable thresholds (grade D or F).',
|
|
1075
|
+
},
|
|
1076
|
+
defaultConfiguration: { level: 'warning' },
|
|
1077
|
+
help: {
|
|
1078
|
+
text: 'Add descriptions to tools and parameters. Consider adding examples to improve documentation.',
|
|
1079
|
+
},
|
|
1080
|
+
},
|
|
1081
|
+
],
|
|
1082
|
+
},
|
|
1083
|
+
},
|
|
1084
|
+
results: [],
|
|
1085
|
+
},
|
|
1086
|
+
],
|
|
1087
|
+
};
|
|
1088
|
+
const results = sarif.runs[0].results;
|
|
1089
|
+
// Add results for removed tools (breaking)
|
|
1090
|
+
for (const tool of diff.toolsRemoved) {
|
|
1091
|
+
results.push({
|
|
1092
|
+
ruleId: 'BWH001',
|
|
1093
|
+
level: 'error',
|
|
1094
|
+
message: { text: `Tool "${tool}" was removed from the server` },
|
|
1095
|
+
locations: [
|
|
1096
|
+
{
|
|
1097
|
+
physicalLocation: {
|
|
1098
|
+
artifactLocation: { uri: baselinePath },
|
|
1099
|
+
region: { startLine: 1 },
|
|
1100
|
+
},
|
|
1101
|
+
},
|
|
1102
|
+
],
|
|
1103
|
+
properties: { tool, changeType: 'removed' },
|
|
1104
|
+
});
|
|
1105
|
+
}
|
|
1106
|
+
// Add results for added tools (info)
|
|
1107
|
+
for (const tool of diff.toolsAdded) {
|
|
1108
|
+
results.push({
|
|
1109
|
+
ruleId: 'BWH004',
|
|
1110
|
+
level: 'note',
|
|
1111
|
+
message: { text: `New tool "${tool}" was added to the server` },
|
|
1112
|
+
locations: [
|
|
1113
|
+
{
|
|
1114
|
+
physicalLocation: {
|
|
1115
|
+
artifactLocation: { uri: baselinePath },
|
|
1116
|
+
region: { startLine: 1 },
|
|
1117
|
+
},
|
|
1118
|
+
},
|
|
1119
|
+
],
|
|
1120
|
+
properties: { tool, changeType: 'added' },
|
|
1121
|
+
});
|
|
1122
|
+
}
|
|
1123
|
+
// Add results for behavior changes
|
|
1124
|
+
for (const change of diff.behaviorChanges) {
|
|
1125
|
+
let ruleId;
|
|
1126
|
+
let level;
|
|
1127
|
+
// Map aspect and severity to appropriate rule
|
|
1128
|
+
if (change.aspect === 'schema') {
|
|
1129
|
+
ruleId = change.severity === 'breaking' ? 'BWH002' : 'BWH003';
|
|
1130
|
+
level = change.severity === 'breaking' ? 'error' : 'warning';
|
|
1131
|
+
}
|
|
1132
|
+
else if (change.aspect === 'response_structure') {
|
|
1133
|
+
ruleId = 'BWH005';
|
|
1134
|
+
level = change.severity === 'breaking' ? 'error' : 'warning';
|
|
1135
|
+
}
|
|
1136
|
+
else if (change.aspect === 'error_pattern' || change.aspect === 'error_handling') {
|
|
1137
|
+
ruleId = 'BWH006';
|
|
1138
|
+
level = change.severity === 'breaking' ? 'error' : 'warning';
|
|
1139
|
+
}
|
|
1140
|
+
else {
|
|
1141
|
+
// Default to schema rules for other aspects
|
|
1142
|
+
ruleId = change.severity === 'breaking' ? 'BWH002' : 'BWH003';
|
|
1143
|
+
level =
|
|
1144
|
+
change.severity === 'breaking'
|
|
1145
|
+
? 'error'
|
|
1146
|
+
: change.severity === 'warning'
|
|
1147
|
+
? 'warning'
|
|
1148
|
+
: 'note';
|
|
1149
|
+
}
|
|
1150
|
+
results.push({
|
|
1151
|
+
ruleId,
|
|
1152
|
+
level,
|
|
1153
|
+
message: { text: `${change.tool}: ${change.description}` },
|
|
1154
|
+
locations: [
|
|
1155
|
+
{
|
|
1156
|
+
physicalLocation: {
|
|
1157
|
+
artifactLocation: { uri: baselinePath },
|
|
1158
|
+
region: { startLine: 1 },
|
|
1159
|
+
},
|
|
1160
|
+
},
|
|
1161
|
+
],
|
|
1162
|
+
properties: {
|
|
1163
|
+
tool: change.tool,
|
|
1164
|
+
aspect: change.aspect,
|
|
1165
|
+
before: change.before,
|
|
1166
|
+
after: change.after,
|
|
1167
|
+
severity: change.severity,
|
|
1168
|
+
},
|
|
1169
|
+
});
|
|
1170
|
+
}
|
|
1171
|
+
// Add results for security findings
|
|
1172
|
+
if (diff.securityReport) {
|
|
1173
|
+
for (const finding of diff.securityReport.newFindings) {
|
|
1174
|
+
const level = finding.riskLevel === 'critical' || finding.riskLevel === 'high'
|
|
1175
|
+
? 'error'
|
|
1176
|
+
: finding.riskLevel === 'medium'
|
|
1177
|
+
? 'warning'
|
|
1178
|
+
: 'note';
|
|
1179
|
+
results.push({
|
|
1180
|
+
ruleId: 'BWH007',
|
|
1181
|
+
level,
|
|
1182
|
+
message: { text: `[${finding.riskLevel.toUpperCase()}] ${finding.tool}: ${finding.title}` },
|
|
1183
|
+
locations: [
|
|
1184
|
+
{
|
|
1185
|
+
physicalLocation: {
|
|
1186
|
+
artifactLocation: { uri: baselinePath },
|
|
1187
|
+
region: { startLine: 1 },
|
|
1188
|
+
},
|
|
1189
|
+
},
|
|
1190
|
+
],
|
|
1191
|
+
properties: {
|
|
1192
|
+
tool: finding.tool,
|
|
1193
|
+
parameter: finding.parameter,
|
|
1194
|
+
category: finding.category,
|
|
1195
|
+
riskLevel: finding.riskLevel,
|
|
1196
|
+
cweId: finding.cweId,
|
|
1197
|
+
description: finding.description,
|
|
1198
|
+
remediation: finding.remediation,
|
|
1199
|
+
evidence: finding.evidence,
|
|
1200
|
+
},
|
|
1201
|
+
});
|
|
1202
|
+
}
|
|
1203
|
+
}
|
|
1204
|
+
// Add results for schema evolution issues
|
|
1205
|
+
if (diff.schemaEvolutionReport) {
|
|
1206
|
+
for (const issue of diff.schemaEvolutionReport.toolsWithIssues) {
|
|
1207
|
+
const ruleId = issue.isBreaking ? 'BWH008' : 'BWH009';
|
|
1208
|
+
const level = issue.isBreaking
|
|
1209
|
+
? 'error'
|
|
1210
|
+
: issue.becameUnstable
|
|
1211
|
+
? 'warning'
|
|
1212
|
+
: 'note';
|
|
1213
|
+
results.push({
|
|
1214
|
+
ruleId,
|
|
1215
|
+
level,
|
|
1216
|
+
message: { text: `${issue.toolName}: ${issue.summary}` },
|
|
1217
|
+
locations: [
|
|
1218
|
+
{
|
|
1219
|
+
physicalLocation: {
|
|
1220
|
+
artifactLocation: { uri: baselinePath },
|
|
1221
|
+
region: { startLine: 1 },
|
|
1222
|
+
},
|
|
1223
|
+
},
|
|
1224
|
+
],
|
|
1225
|
+
properties: {
|
|
1226
|
+
tool: issue.toolName,
|
|
1227
|
+
isBreaking: issue.isBreaking,
|
|
1228
|
+
becameUnstable: issue.becameUnstable,
|
|
1229
|
+
fieldsAdded: issue.fieldsAdded,
|
|
1230
|
+
fieldsRemoved: issue.fieldsRemoved,
|
|
1231
|
+
},
|
|
1232
|
+
});
|
|
1233
|
+
}
|
|
1234
|
+
}
|
|
1235
|
+
// Add results for performance regressions and confidence
|
|
1236
|
+
if (diff.performanceReport) {
|
|
1237
|
+
// Performance regressions
|
|
1238
|
+
for (const regression of diff.performanceReport.regressions) {
|
|
1239
|
+
const percentStr = (regression.regressionPercent * 100).toFixed(1);
|
|
1240
|
+
const confidenceNote = regression.isReliable ? '' : ' (low confidence)';
|
|
1241
|
+
results.push({
|
|
1242
|
+
ruleId: 'BWH012',
|
|
1243
|
+
level: 'warning',
|
|
1244
|
+
message: {
|
|
1245
|
+
text: `Performance regression: ${regression.toolName} +${percentStr}%${confidenceNote}`,
|
|
1246
|
+
},
|
|
1247
|
+
locations: [
|
|
1248
|
+
{
|
|
1249
|
+
physicalLocation: {
|
|
1250
|
+
artifactLocation: { uri: baselinePath },
|
|
1251
|
+
region: { startLine: 1 },
|
|
1252
|
+
},
|
|
1253
|
+
},
|
|
1254
|
+
],
|
|
1255
|
+
properties: {
|
|
1256
|
+
tool: regression.toolName,
|
|
1257
|
+
previousP50Ms: regression.previousP50Ms,
|
|
1258
|
+
currentP50Ms: regression.currentP50Ms,
|
|
1259
|
+
regressionPercent: regression.regressionPercent,
|
|
1260
|
+
isReliable: regression.isReliable,
|
|
1261
|
+
currentConfidence: regression.currentConfidence,
|
|
1262
|
+
},
|
|
1263
|
+
});
|
|
1264
|
+
}
|
|
1265
|
+
// Low confidence tools
|
|
1266
|
+
if (diff.performanceReport.lowConfidenceTools) {
|
|
1267
|
+
for (const tool of diff.performanceReport.lowConfidenceTools) {
|
|
1268
|
+
results.push({
|
|
1269
|
+
ruleId: 'BWH013',
|
|
1270
|
+
level: 'note',
|
|
1271
|
+
message: {
|
|
1272
|
+
text: `Low confidence metrics for "${tool}". Consider running with more samples.`,
|
|
1273
|
+
},
|
|
1274
|
+
locations: [
|
|
1275
|
+
{
|
|
1276
|
+
physicalLocation: {
|
|
1277
|
+
artifactLocation: { uri: baselinePath },
|
|
1278
|
+
region: { startLine: 1 },
|
|
1279
|
+
},
|
|
1280
|
+
},
|
|
1281
|
+
],
|
|
1282
|
+
properties: {
|
|
1283
|
+
tool,
|
|
1284
|
+
recommendation: 'Run with --samples flag for reliable baselines',
|
|
1285
|
+
},
|
|
1286
|
+
});
|
|
1287
|
+
}
|
|
1288
|
+
}
|
|
1289
|
+
}
|
|
1290
|
+
// Add results for error trend changes
|
|
1291
|
+
if (diff.errorTrendReport) {
|
|
1292
|
+
const et = diff.errorTrendReport;
|
|
1293
|
+
// New error types
|
|
1294
|
+
for (const category of et.newCategories) {
|
|
1295
|
+
results.push({
|
|
1296
|
+
ruleId: 'BWH010',
|
|
1297
|
+
level: 'warning',
|
|
1298
|
+
message: { text: `New error type detected: ${category}` },
|
|
1299
|
+
locations: [
|
|
1300
|
+
{
|
|
1301
|
+
physicalLocation: {
|
|
1302
|
+
artifactLocation: { uri: baselinePath },
|
|
1303
|
+
region: { startLine: 1 },
|
|
1304
|
+
},
|
|
1305
|
+
},
|
|
1306
|
+
],
|
|
1307
|
+
properties: {
|
|
1308
|
+
category,
|
|
1309
|
+
trend: 'new',
|
|
1310
|
+
significance: 'high',
|
|
1311
|
+
},
|
|
1312
|
+
});
|
|
1313
|
+
}
|
|
1314
|
+
// Increasing error types
|
|
1315
|
+
for (const category of et.increasingCategories) {
|
|
1316
|
+
const trend = et.trends.find(t => t.category === category);
|
|
1317
|
+
results.push({
|
|
1318
|
+
ruleId: 'BWH011',
|
|
1319
|
+
level: 'warning',
|
|
1320
|
+
message: { text: `Error frequency increasing: ${category}${trend ? ` (+${trend.changePercent}%)` : ''}` },
|
|
1321
|
+
locations: [
|
|
1322
|
+
{
|
|
1323
|
+
physicalLocation: {
|
|
1324
|
+
artifactLocation: { uri: baselinePath },
|
|
1325
|
+
region: { startLine: 1 },
|
|
1326
|
+
},
|
|
1327
|
+
},
|
|
1328
|
+
],
|
|
1329
|
+
properties: {
|
|
1330
|
+
category,
|
|
1331
|
+
trend: 'increasing',
|
|
1332
|
+
previousCount: trend?.previousCount,
|
|
1333
|
+
currentCount: trend?.currentCount,
|
|
1334
|
+
changePercent: trend?.changePercent,
|
|
1335
|
+
},
|
|
1336
|
+
});
|
|
1337
|
+
}
|
|
1338
|
+
}
|
|
1339
|
+
// Add results for documentation quality changes
|
|
1340
|
+
if (diff.documentationScoreReport) {
|
|
1341
|
+
const doc = diff.documentationScoreReport;
|
|
1342
|
+
if (doc.degraded) {
|
|
1343
|
+
results.push({
|
|
1344
|
+
ruleId: 'BWH014',
|
|
1345
|
+
level: 'warning',
|
|
1346
|
+
message: {
|
|
1347
|
+
text: `Documentation quality degraded: ${doc.previousScore} -> ${doc.currentScore} (${doc.currentGrade})`,
|
|
1348
|
+
},
|
|
1349
|
+
locations: [
|
|
1350
|
+
{
|
|
1351
|
+
physicalLocation: {
|
|
1352
|
+
artifactLocation: { uri: baselinePath },
|
|
1353
|
+
region: { startLine: 1 },
|
|
1354
|
+
},
|
|
1355
|
+
},
|
|
1356
|
+
],
|
|
1357
|
+
properties: {
|
|
1358
|
+
previousScore: doc.previousScore,
|
|
1359
|
+
currentScore: doc.currentScore,
|
|
1360
|
+
change: doc.change,
|
|
1361
|
+
previousGrade: doc.previousGrade,
|
|
1362
|
+
currentGrade: doc.currentGrade,
|
|
1363
|
+
newIssues: doc.newIssues,
|
|
1364
|
+
},
|
|
1365
|
+
});
|
|
1366
|
+
}
|
|
1367
|
+
// Add warning for low documentation quality regardless of change
|
|
1368
|
+
if (doc.currentGrade === 'D' || doc.currentGrade === 'F') {
|
|
1369
|
+
results.push({
|
|
1370
|
+
ruleId: 'BWH015',
|
|
1371
|
+
level: 'warning',
|
|
1372
|
+
message: {
|
|
1373
|
+
text: `Low documentation quality: ${doc.currentScore}/100 (${doc.currentGrade})`,
|
|
1374
|
+
},
|
|
1375
|
+
locations: [
|
|
1376
|
+
{
|
|
1377
|
+
physicalLocation: {
|
|
1378
|
+
artifactLocation: { uri: baselinePath },
|
|
1379
|
+
region: { startLine: 1 },
|
|
1380
|
+
},
|
|
1381
|
+
},
|
|
1382
|
+
],
|
|
1383
|
+
properties: {
|
|
1384
|
+
score: doc.currentScore,
|
|
1385
|
+
grade: doc.currentGrade,
|
|
1386
|
+
},
|
|
1387
|
+
});
|
|
1388
|
+
}
|
|
1389
|
+
}
|
|
1390
|
+
return JSON.stringify(sarif, null, 2);
|
|
1391
|
+
}
|
|
1392
|
+
/**
|
|
1393
|
+
* Escape XML special characters to prevent injection.
|
|
1394
|
+
*/
|
|
1395
|
+
function escapeXml(str) {
|
|
1396
|
+
return str
|
|
1397
|
+
.replace(/&/g, '&')
|
|
1398
|
+
.replace(/</g, '<')
|
|
1399
|
+
.replace(/>/g, '>')
|
|
1400
|
+
.replace(/"/g, '"')
|
|
1401
|
+
.replace(/'/g, ''');
|
|
1402
|
+
}
|
|
1403
|
+
const colors = {
|
|
1404
|
+
red: (s) => `\x1b[31m${s}\x1b[0m`,
|
|
1405
|
+
green: (s) => `\x1b[32m${s}\x1b[0m`,
|
|
1406
|
+
yellow: (s) => `\x1b[33m${s}\x1b[0m`,
|
|
1407
|
+
cyan: (s) => `\x1b[36m${s}\x1b[0m`,
|
|
1408
|
+
bold: (s) => `\x1b[1m${s}\x1b[0m`,
|
|
1409
|
+
dim: (s) => `\x1b[2m${s}\x1b[0m`,
|
|
1410
|
+
};
|
|
1411
|
+
const noColors = {
|
|
1412
|
+
red: (s) => s,
|
|
1413
|
+
green: (s) => s,
|
|
1414
|
+
yellow: (s) => s,
|
|
1415
|
+
cyan: (s) => s,
|
|
1416
|
+
bold: (s) => s,
|
|
1417
|
+
dim: (s) => s,
|
|
1418
|
+
};
|
|
1419
|
+
function getSeverityBadge(severity, useColors) {
|
|
1420
|
+
const c = useColors ? colors : noColors;
|
|
1421
|
+
switch (severity) {
|
|
1422
|
+
case 'none':
|
|
1423
|
+
return c.green('✓ NONE');
|
|
1424
|
+
case 'info':
|
|
1425
|
+
return c.cyan('ℹ INFO');
|
|
1426
|
+
case 'warning':
|
|
1427
|
+
return c.yellow('⚠ WARNING');
|
|
1428
|
+
case 'breaking':
|
|
1429
|
+
return c.red('✗ BREAKING');
|
|
1430
|
+
}
|
|
1431
|
+
}
|
|
1432
|
+
function getSeverityEmoji(severity) {
|
|
1433
|
+
switch (severity) {
|
|
1434
|
+
case 'none':
|
|
1435
|
+
return '✅';
|
|
1436
|
+
case 'info':
|
|
1437
|
+
return 'ℹ️';
|
|
1438
|
+
case 'warning':
|
|
1439
|
+
return '⚠️';
|
|
1440
|
+
case 'breaking':
|
|
1441
|
+
return '❌';
|
|
1442
|
+
}
|
|
1443
|
+
}
|
|
1444
|
+
function getChangeIcon(change, useColors) {
|
|
1445
|
+
const c = useColors ? colors : noColors;
|
|
1446
|
+
switch (change.severity) {
|
|
1447
|
+
case 'breaking':
|
|
1448
|
+
return c.red('●');
|
|
1449
|
+
case 'warning':
|
|
1450
|
+
return c.yellow('●');
|
|
1451
|
+
case 'info':
|
|
1452
|
+
return c.cyan('●');
|
|
1453
|
+
default:
|
|
1454
|
+
return '○';
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
function getSeverityColor(severity, useColors) {
|
|
1458
|
+
const c = useColors ? colors : noColors;
|
|
1459
|
+
switch (severity) {
|
|
1460
|
+
case 'breaking':
|
|
1461
|
+
return c.red;
|
|
1462
|
+
case 'warning':
|
|
1463
|
+
return c.yellow;
|
|
1464
|
+
case 'info':
|
|
1465
|
+
return c.cyan;
|
|
1466
|
+
default:
|
|
1467
|
+
return (s) => s;
|
|
1468
|
+
}
|
|
1469
|
+
}
|
|
1470
|
+
function groupChangesByTool(changes) {
|
|
1471
|
+
const map = new Map();
|
|
1472
|
+
for (const change of changes) {
|
|
1473
|
+
const existing = map.get(change.tool) || [];
|
|
1474
|
+
existing.push(change);
|
|
1475
|
+
map.set(change.tool, existing);
|
|
1476
|
+
}
|
|
1477
|
+
return map;
|
|
1478
|
+
}
|
|
1479
|
+
/**
|
|
1480
|
+
* Get color function for risk level.
|
|
1481
|
+
*/
|
|
1482
|
+
function getRiskLevelColor(riskLevel, useColors) {
|
|
1483
|
+
const c = useColors ? colors : noColors;
|
|
1484
|
+
switch (riskLevel) {
|
|
1485
|
+
case 'critical':
|
|
1486
|
+
case 'high':
|
|
1487
|
+
return c.red;
|
|
1488
|
+
case 'medium':
|
|
1489
|
+
return c.yellow;
|
|
1490
|
+
case 'low':
|
|
1491
|
+
return c.cyan;
|
|
1492
|
+
case 'info':
|
|
1493
|
+
default:
|
|
1494
|
+
return c.dim;
|
|
1495
|
+
}
|
|
1496
|
+
}
|
|
1497
|
+
/**
|
|
1498
|
+
* Get emoji for risk level (used in markdown output).
|
|
1499
|
+
*/
|
|
1500
|
+
function getRiskLevelEmoji(riskLevel) {
|
|
1501
|
+
switch (riskLevel) {
|
|
1502
|
+
case 'critical':
|
|
1503
|
+
return '🔴';
|
|
1504
|
+
case 'high':
|
|
1505
|
+
return '🟠';
|
|
1506
|
+
case 'medium':
|
|
1507
|
+
return '🟡';
|
|
1508
|
+
case 'low':
|
|
1509
|
+
return '🔵';
|
|
1510
|
+
case 'info':
|
|
1511
|
+
default:
|
|
1512
|
+
return '⚪';
|
|
1513
|
+
}
|
|
1514
|
+
}
|
|
1515
|
+
/**
|
|
1516
|
+
* Format a summary for schema evolution report.
|
|
1517
|
+
*/
|
|
1518
|
+
function formatSchemaEvolutionSummary(report) {
|
|
1519
|
+
const parts = [];
|
|
1520
|
+
if (report.hasBreakingChanges) {
|
|
1521
|
+
parts.push('Breaking schema changes detected');
|
|
1522
|
+
}
|
|
1523
|
+
if (report.unstableCount > 0) {
|
|
1524
|
+
parts.push(`${report.unstableCount} unstable schema(s)`);
|
|
1525
|
+
}
|
|
1526
|
+
if (report.structureChangedCount > 0) {
|
|
1527
|
+
parts.push(`${report.structureChangedCount} structure change(s)`);
|
|
1528
|
+
}
|
|
1529
|
+
if (parts.length === 0) {
|
|
1530
|
+
return `${report.stableCount} stable schema(s)`;
|
|
1531
|
+
}
|
|
1532
|
+
return parts.join(', ');
|
|
1533
|
+
}
|
|
1534
|
+
/**
|
|
1535
|
+
* Format a standalone security report for display.
|
|
1536
|
+
* Used when only security data is available (not a full diff).
|
|
1537
|
+
*/
|
|
1538
|
+
export function formatSecurityReport(report, useColors = true) {
|
|
1539
|
+
const lines = [];
|
|
1540
|
+
const { red, green, bold, dim } = useColors ? colors : noColors;
|
|
1541
|
+
lines.push(bold('Security Report'));
|
|
1542
|
+
lines.push('═'.repeat(50));
|
|
1543
|
+
lines.push('');
|
|
1544
|
+
// Summary
|
|
1545
|
+
lines.push(report.summary);
|
|
1546
|
+
lines.push('');
|
|
1547
|
+
// Risk score
|
|
1548
|
+
const scoreColor = report.degraded ? red : green;
|
|
1549
|
+
const changeSymbol = report.riskScoreChange > 0 ? '↑' : report.riskScoreChange < 0 ? '↓' : '→';
|
|
1550
|
+
lines.push(`Risk Score: ${report.previousRiskScore} ${changeSymbol} ${scoreColor(String(report.currentRiskScore))}`);
|
|
1551
|
+
lines.push('');
|
|
1552
|
+
// New findings
|
|
1553
|
+
if (report.newFindings.length > 0) {
|
|
1554
|
+
lines.push(red('─── New Findings ───'));
|
|
1555
|
+
for (const finding of report.newFindings) {
|
|
1556
|
+
const riskColor = getRiskLevelColor(finding.riskLevel, useColors);
|
|
1557
|
+
lines.push(` ${riskColor('●')} [${finding.riskLevel.toUpperCase()}] ${finding.title}`);
|
|
1558
|
+
lines.push(` Tool: ${finding.tool}`);
|
|
1559
|
+
lines.push(` Parameter: ${finding.parameter}`);
|
|
1560
|
+
lines.push(` ${finding.cweId}: ${finding.description}`);
|
|
1561
|
+
lines.push(` ${dim('Remediation:')} ${finding.remediation}`);
|
|
1562
|
+
lines.push('');
|
|
1563
|
+
}
|
|
1564
|
+
}
|
|
1565
|
+
// Resolved findings
|
|
1566
|
+
if (report.resolvedFindings.length > 0) {
|
|
1567
|
+
lines.push(green('─── Resolved Findings ───'));
|
|
1568
|
+
for (const finding of report.resolvedFindings) {
|
|
1569
|
+
lines.push(` ${green('✓')} ${finding.title} (${finding.tool})`);
|
|
1570
|
+
}
|
|
1571
|
+
lines.push('');
|
|
1572
|
+
}
|
|
1573
|
+
// Statistics
|
|
1574
|
+
lines.push('─── Statistics ───');
|
|
1575
|
+
lines.push(` New findings: ${report.newFindings.length}`);
|
|
1576
|
+
lines.push(` Resolved findings: ${report.resolvedFindings.length}`);
|
|
1577
|
+
const criticalHigh = report.newFindings.filter((f) => f.riskLevel === 'critical' || f.riskLevel === 'high').length;
|
|
1578
|
+
if (criticalHigh > 0) {
|
|
1579
|
+
lines.push(` ${red('Critical/High severity:')} ${criticalHigh}`);
|
|
1580
|
+
}
|
|
1581
|
+
lines.push('');
|
|
1582
|
+
return lines.join('\n');
|
|
1583
|
+
}
|
|
1584
|
+
//# sourceMappingURL=diff.js.map
|