@sanity/ailf 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +89 -0
- package/bin/ailf.js +64 -0
- package/canonical/grader-references/README.md +88 -0
- package/canonical/grader-references/groq.yaml +234 -0
- package/canonical/grader-references/studio-setup.yaml +275 -0
- package/canonical/reference-solutions/.gitkeep +1 -0
- package/canonical/reference-solutions/frameworks/nuxt.ts +119 -0
- package/canonical/reference-solutions/frameworks/remix.tsx +100 -0
- package/canonical/reference-solutions/functions/publish-webhook.ts +60 -0
- package/canonical/reference-solutions/groq/advanced-filtering.ts +379 -0
- package/canonical/reference-solutions/groq/blog-queries.ts +137 -0
- package/canonical/reference-solutions/groq/joins-references.ts +300 -0
- package/canonical/reference-solutions/nextjs/app-router-integration.tsx +128 -0
- package/canonical/reference-solutions/studio-setup/blog-schema.ts +143 -0
- package/canonical/reference-solutions/studio-setup/custom-tool.tsx +78 -0
- package/canonical/reference-solutions/visual-editing/live-preview.tsx +137 -0
- package/canonical/reference-solutions/visual-editing/presentation-nextjs.tsx +130 -0
- package/config/airbyte/ai_literacy_framework.connector.yaml +639 -0
- package/config/bigquery/README.md +74 -0
- package/config/bigquery/views/area_scores.sql +87 -0
- package/config/bigquery/views/reports.sql +49 -0
- package/config/features.yaml +116 -0
- package/config/models.yaml +115 -0
- package/config/prompts.yaml +75 -0
- package/config/rubrics.yaml +62 -0
- package/config/schedules.yaml +43 -0
- package/config/sinks.yaml +54 -0
- package/config/sources.yaml +51 -0
- package/config/thresholds.yaml +49 -0
- package/dist/_vendor/ailf-core/examples/index.d.ts +190 -0
- package/dist/_vendor/ailf-core/examples/index.js +285 -0
- package/dist/_vendor/ailf-core/index.d.ts +17 -0
- package/dist/_vendor/ailf-core/index.js +17 -0
- package/dist/_vendor/ailf-core/ports/cache-store.d.ts +72 -0
- package/dist/_vendor/ailf-core/ports/cache-store.js +17 -0
- package/dist/_vendor/ailf-core/ports/config-source.d.ts +33 -0
- package/dist/_vendor/ailf-core/ports/config-source.js +15 -0
- package/dist/_vendor/ailf-core/ports/context.d.ts +172 -0
- package/dist/_vendor/ailf-core/ports/context.js +14 -0
- package/dist/_vendor/ailf-core/ports/doc-fetcher.d.ts +131 -0
- package/dist/_vendor/ailf-core/ports/doc-fetcher.js +12 -0
- package/dist/_vendor/ailf-core/ports/eval-runner.d.ts +24 -0
- package/dist/_vendor/ailf-core/ports/eval-runner.js +8 -0
- package/dist/_vendor/ailf-core/ports/index.d.ts +15 -0
- package/dist/_vendor/ailf-core/ports/index.js +7 -0
- package/dist/_vendor/ailf-core/ports/logger.d.ts +36 -0
- package/dist/_vendor/ailf-core/ports/logger.js +11 -0
- package/dist/_vendor/ailf-core/ports/pipeline-step.d.ts +46 -0
- package/dist/_vendor/ailf-core/ports/pipeline-step.js +8 -0
- package/dist/_vendor/ailf-core/ports/task-source.d.ts +159 -0
- package/dist/_vendor/ailf-core/ports/task-source.js +72 -0
- package/dist/_vendor/ailf-core/schemas/callback-payload.d.ts +24 -0
- package/dist/_vendor/ailf-core/schemas/callback-payload.js +29 -0
- package/dist/_vendor/ailf-core/schemas/eval-config.d.ts +55 -0
- package/dist/_vendor/ailf-core/schemas/eval-config.js +78 -0
- package/dist/_vendor/ailf-core/schemas/index.d.ts +16 -0
- package/dist/_vendor/ailf-core/schemas/index.js +16 -0
- package/dist/_vendor/ailf-core/schemas/pipeline-request.d.ts +125 -0
- package/dist/_vendor/ailf-core/schemas/pipeline-request.js +67 -0
- package/dist/_vendor/ailf-core/schemas/pipeline.d.ts +531 -0
- package/dist/_vendor/ailf-core/schemas/pipeline.js +318 -0
- package/dist/_vendor/ailf-core/schemas/schedules.d.ts +68 -0
- package/dist/_vendor/ailf-core/schemas/schedules.js +74 -0
- package/dist/_vendor/ailf-core/schemas/sinks.d.ts +207 -0
- package/dist/_vendor/ailf-core/schemas/sinks.js +108 -0
- package/dist/_vendor/ailf-core/services/comparison-formatters.d.ts +18 -0
- package/dist/_vendor/ailf-core/services/comparison-formatters.js +189 -0
- package/dist/_vendor/ailf-core/services/config-helpers.d.ts +41 -0
- package/dist/_vendor/ailf-core/services/config-helpers.js +86 -0
- package/dist/_vendor/ailf-core/services/index.d.ts +12 -0
- package/dist/_vendor/ailf-core/services/index.js +12 -0
- package/dist/_vendor/ailf-core/services/scoring.d.ts +49 -0
- package/dist/_vendor/ailf-core/services/scoring.js +222 -0
- package/dist/_vendor/ailf-core/types/index.d.ts +1082 -0
- package/dist/_vendor/ailf-core/types/index.js +21 -0
- package/dist/_vendor/ailf-core/types/scoring-input.d.ts +54 -0
- package/dist/_vendor/ailf-core/types/scoring-input.js +9 -0
- package/dist/_vendor/ailf-shared/dimension-names.d.ts +21 -0
- package/dist/_vendor/ailf-shared/dimension-names.js +27 -0
- package/dist/_vendor/ailf-shared/document-ref.d.ts +29 -0
- package/dist/_vendor/ailf-shared/document-ref.js +1 -0
- package/dist/_vendor/ailf-shared/eval-modes.d.ts +12 -0
- package/dist/_vendor/ailf-shared/eval-modes.js +8 -0
- package/dist/_vendor/ailf-shared/index.d.ts +16 -0
- package/dist/_vendor/ailf-shared/index.js +16 -0
- package/dist/_vendor/ailf-shared/noise-threshold.d.ts +9 -0
- package/dist/_vendor/ailf-shared/noise-threshold.js +9 -0
- package/dist/_vendor/ailf-shared/score-grades.d.ts +17 -0
- package/dist/_vendor/ailf-shared/score-grades.js +23 -0
- package/dist/adapters/cache/content-lake-cache.d.ts +24 -0
- package/dist/adapters/cache/content-lake-cache.js +59 -0
- package/dist/adapters/cache/filesystem-cache.d.ts +18 -0
- package/dist/adapters/cache/filesystem-cache.js +54 -0
- package/dist/adapters/cache/index.d.ts +2 -0
- package/dist/adapters/cache/index.js +2 -0
- package/dist/adapters/config-sources/cli-config-adapter.d.ts +17 -0
- package/dist/adapters/config-sources/cli-config-adapter.js +23 -0
- package/dist/adapters/config-sources/file-config-adapter.d.ts +26 -0
- package/dist/adapters/config-sources/file-config-adapter.js +96 -0
- package/dist/adapters/config-sources/index.d.ts +2 -0
- package/dist/adapters/config-sources/index.js +2 -0
- package/dist/adapters/doc-fetchers/index.d.ts +1 -0
- package/dist/adapters/doc-fetchers/index.js +1 -0
- package/dist/adapters/doc-fetchers/sanity-doc-fetcher.d.ts +76 -0
- package/dist/adapters/doc-fetchers/sanity-doc-fetcher.js +620 -0
- package/dist/adapters/eval-runners/index.d.ts +1 -0
- package/dist/adapters/eval-runners/index.js +1 -0
- package/dist/adapters/eval-runners/promptfoo-eval-adapter.d.ts +14 -0
- package/dist/adapters/eval-runners/promptfoo-eval-adapter.js +63 -0
- package/dist/adapters/index.d.ts +12 -0
- package/dist/adapters/index.js +12 -0
- package/dist/adapters/loggers/console-logger.d.ts +22 -0
- package/dist/adapters/loggers/console-logger.js +54 -0
- package/dist/adapters/loggers/index.d.ts +9 -0
- package/dist/adapters/loggers/index.js +9 -0
- package/dist/adapters/loggers/json-logger.d.ts +18 -0
- package/dist/adapters/loggers/json-logger.js +33 -0
- package/dist/adapters/loggers/quiet-logger.d.ts +16 -0
- package/dist/adapters/loggers/quiet-logger.js +30 -0
- package/dist/adapters/task-sources/composite-task-source.d.ts +20 -0
- package/dist/adapters/task-sources/composite-task-source.js +59 -0
- package/dist/adapters/task-sources/content-lake-task-source.d.ts +20 -0
- package/dist/adapters/task-sources/content-lake-task-source.js +219 -0
- package/dist/adapters/task-sources/index.d.ts +7 -0
- package/dist/adapters/task-sources/index.js +7 -0
- package/dist/adapters/task-sources/repo-schemas.d.ts +245 -0
- package/dist/adapters/task-sources/repo-schemas.js +234 -0
- package/dist/adapters/task-sources/repo-task-source.d.ts +22 -0
- package/dist/adapters/task-sources/repo-task-source.js +104 -0
- package/dist/adapters/task-sources/repo-trigger.d.ts +52 -0
- package/dist/adapters/task-sources/repo-trigger.js +153 -0
- package/dist/adapters/task-sources/repo-validation.d.ts +49 -0
- package/dist/adapters/task-sources/repo-validation.js +164 -0
- package/dist/adapters/task-sources/yaml-task-source.d.ts +18 -0
- package/dist/adapters/task-sources/yaml-task-source.js +136 -0
- package/dist/agent-observer/agentic-provider.d.ts +132 -0
- package/dist/agent-observer/agentic-provider.js +983 -0
- package/dist/agent-observer/classifier.d.ts +62 -0
- package/dist/agent-observer/classifier.js +269 -0
- package/dist/agent-observer/index.d.ts +7 -0
- package/dist/agent-observer/index.js +4 -0
- package/dist/agent-observer/pricing.d.ts +35 -0
- package/dist/agent-observer/pricing.js +82 -0
- package/dist/agent-observer/provider.d.ts +77 -0
- package/dist/agent-observer/provider.js +151 -0
- package/dist/agent-observer/proxy.d.ts +91 -0
- package/dist/agent-observer/proxy.js +321 -0
- package/dist/agent-observer/test-imports.d.ts +7 -0
- package/dist/agent-observer/test-imports.js +185 -0
- package/dist/agent-observer/types.d.ts +137 -0
- package/dist/agent-observer/types.js +16 -0
- package/dist/assertions/source-isolation.d.ts +72 -0
- package/dist/assertions/source-isolation.js +117 -0
- package/dist/cli.d.ts +24 -0
- package/dist/cli.js +199 -0
- package/dist/commands/agent-report.d.ts +5 -0
- package/dist/commands/agent-report.js +69 -0
- package/dist/commands/baseline.d.ts +9 -0
- package/dist/commands/baseline.js +141 -0
- package/dist/commands/cache.d.ts +13 -0
- package/dist/commands/cache.js +135 -0
- package/dist/commands/calculate-scores.d.ts +8 -0
- package/dist/commands/calculate-scores.js +48 -0
- package/dist/commands/compare.d.ts +8 -0
- package/dist/commands/compare.js +120 -0
- package/dist/commands/completion.d.ts +18 -0
- package/dist/commands/completion.js +260 -0
- package/dist/commands/coverage-audit.d.ts +7 -0
- package/dist/commands/coverage-audit.js +40 -0
- package/dist/commands/discovery-report.d.ts +10 -0
- package/dist/commands/discovery-report.js +44 -0
- package/dist/commands/eval.d.ts +9 -0
- package/dist/commands/eval.js +35 -0
- package/dist/commands/explain-handler.d.ts +34 -0
- package/dist/commands/explain-handler.js +719 -0
- package/dist/commands/fetch-docs.d.ts +8 -0
- package/dist/commands/fetch-docs.js +128 -0
- package/dist/commands/generate-configs.d.ts +8 -0
- package/dist/commands/generate-configs.js +46 -0
- package/dist/commands/grader/index.d.ts +11 -0
- package/dist/commands/grader/index.js +118 -0
- package/dist/commands/init.d.ts +19 -0
- package/dist/commands/init.js +150 -0
- package/dist/commands/interactive.d.ts +12 -0
- package/dist/commands/interactive.js +238 -0
- package/dist/commands/lookup-doc.d.ts +15 -0
- package/dist/commands/lookup-doc.js +84 -0
- package/dist/commands/measure-retrieval.d.ts +5 -0
- package/dist/commands/measure-retrieval.js +65 -0
- package/dist/commands/pipeline-action.d.ts +71 -0
- package/dist/commands/pipeline-action.js +305 -0
- package/dist/commands/pipeline.d.ts +62 -0
- package/dist/commands/pipeline.js +53 -0
- package/dist/commands/pr-comment.d.ts +8 -0
- package/dist/commands/pr-comment.js +47 -0
- package/dist/commands/publish.d.ts +26 -0
- package/dist/commands/publish.js +253 -0
- package/dist/commands/readiness-report.d.ts +10 -0
- package/dist/commands/readiness-report.js +104 -0
- package/dist/commands/shared/options.d.ts +29 -0
- package/dist/commands/shared/options.js +57 -0
- package/dist/commands/update-quality-scores.d.ts +5 -0
- package/dist/commands/update-quality-scores.js +20 -0
- package/dist/commands/validate-tasks.d.ts +16 -0
- package/dist/commands/validate-tasks.js +93 -0
- package/dist/commands/validate.d.ts +9 -0
- package/dist/commands/validate.js +73 -0
- package/dist/commands/webhook-server.d.ts +5 -0
- package/dist/commands/webhook-server.js +30 -0
- package/dist/commands/weekly-digest.d.ts +10 -0
- package/dist/commands/weekly-digest.js +104 -0
- package/dist/composition-root.d.ts +26 -0
- package/dist/composition-root.js +107 -0
- package/dist/interpolate.d.ts +26 -0
- package/dist/interpolate.js +70 -0
- package/dist/job-store.d.ts +104 -0
- package/dist/job-store.js +188 -0
- package/dist/lib/agent-behavior-report.d.ts +8 -0
- package/dist/lib/agent-behavior-report.js +185 -0
- package/dist/lib/baseline.d.ts +19 -0
- package/dist/lib/baseline.js +153 -0
- package/dist/lib/calculate-scores.d.ts +23 -0
- package/dist/lib/calculate-scores.js +42 -0
- package/dist/lib/compare.d.ts +18 -0
- package/dist/lib/compare.js +170 -0
- package/dist/lib/coverage-audit.d.ts +4 -0
- package/dist/lib/coverage-audit.js +42 -0
- package/dist/lib/discovery-report.d.ts +13 -0
- package/dist/lib/discovery-report.js +57 -0
- package/dist/lib/fetch-docs.d.ts +30 -0
- package/dist/lib/fetch-docs.js +171 -0
- package/dist/lib/generate-configs.d.ts +25 -0
- package/dist/lib/generate-configs.js +42 -0
- package/dist/lib/grader-api.d.ts +21 -0
- package/dist/lib/grader-api.js +34 -0
- package/dist/lib/grader-compare.d.ts +19 -0
- package/dist/lib/grader-compare.js +91 -0
- package/dist/lib/grader-consistency.d.ts +27 -0
- package/dist/lib/grader-consistency.js +79 -0
- package/dist/lib/grader-sensitivity.d.ts +19 -0
- package/dist/lib/grader-sensitivity.js +75 -0
- package/dist/lib/grader-validate.d.ts +19 -0
- package/dist/lib/grader-validate.js +78 -0
- package/dist/lib/measure-retrieval.d.ts +14 -0
- package/dist/lib/measure-retrieval.js +71 -0
- package/dist/lib/pr-comment.d.ts +16 -0
- package/dist/lib/pr-comment.js +28 -0
- package/dist/lib/readiness-report.d.ts +13 -0
- package/dist/lib/readiness-report.js +108 -0
- package/dist/lib/webhook-server.d.ts +11 -0
- package/dist/lib/webhook-server.js +24 -0
- package/dist/lib/weekly-digest.d.ts +24 -0
- package/dist/lib/weekly-digest.js +148 -0
- package/dist/orchestration/build-app-context.d.ts +27 -0
- package/dist/orchestration/build-app-context.js +81 -0
- package/dist/orchestration/build-step-sequence.d.ts +15 -0
- package/dist/orchestration/build-step-sequence.js +84 -0
- package/dist/orchestration/config-to-source-overrides.d.ts +9 -0
- package/dist/orchestration/config-to-source-overrides.js +28 -0
- package/dist/orchestration/env-bridge.d.ts +21 -0
- package/dist/orchestration/env-bridge.js +66 -0
- package/dist/orchestration/index.d.ts +11 -0
- package/dist/orchestration/index.js +11 -0
- package/dist/orchestration/pipeline-orchestrator.d.ts +24 -0
- package/dist/orchestration/pipeline-orchestrator.js +153 -0
- package/dist/orchestration/step-runner.d.ts +20 -0
- package/dist/orchestration/step-runner.js +88 -0
- package/dist/orchestration/steps/calculate-scores-step.d.ts +13 -0
- package/dist/orchestration/steps/calculate-scores-step.js +95 -0
- package/dist/orchestration/steps/callback-step.d.ts +24 -0
- package/dist/orchestration/steps/callback-step.js +76 -0
- package/dist/orchestration/steps/compare-step.d.ts +14 -0
- package/dist/orchestration/steps/compare-step.js +92 -0
- package/dist/orchestration/steps/discovery-report-step.d.ts +13 -0
- package/dist/orchestration/steps/discovery-report-step.js +55 -0
- package/dist/orchestration/steps/fetch-docs-shell.d.ts +17 -0
- package/dist/orchestration/steps/fetch-docs-shell.js +30 -0
- package/dist/orchestration/steps/fetch-docs-step.d.ts +14 -0
- package/dist/orchestration/steps/fetch-docs-step.js +135 -0
- package/dist/orchestration/steps/gap-analysis-step.d.ts +16 -0
- package/dist/orchestration/steps/gap-analysis-step.js +136 -0
- package/dist/orchestration/steps/generate-configs-step.d.ts +14 -0
- package/dist/orchestration/steps/generate-configs-step.js +85 -0
- package/dist/orchestration/steps/grader-consistency-step.d.ts +13 -0
- package/dist/orchestration/steps/grader-consistency-step.js +64 -0
- package/dist/orchestration/steps/index.d.ts +19 -0
- package/dist/orchestration/steps/index.js +19 -0
- package/dist/orchestration/steps/mirror-repo-tasks-step.d.ts +21 -0
- package/dist/orchestration/steps/mirror-repo-tasks-step.js +94 -0
- package/dist/orchestration/steps/publish-report-step.d.ts +26 -0
- package/dist/orchestration/steps/publish-report-step.js +216 -0
- package/dist/orchestration/steps/readiness-step.d.ts +13 -0
- package/dist/orchestration/steps/readiness-step.js +91 -0
- package/dist/orchestration/steps/report-step.d.ts +12 -0
- package/dist/orchestration/steps/report-step.js +49 -0
- package/dist/orchestration/steps/run-eval-step.d.ts +17 -0
- package/dist/orchestration/steps/run-eval-step.js +195 -0
- package/dist/orchestration/steps/validate-step.d.ts +12 -0
- package/dist/orchestration/steps/validate-step.js +41 -0
- package/dist/pipeline/agent-behavior-report.d.ts +53 -0
- package/dist/pipeline/agent-behavior-report.js +132 -0
- package/dist/pipeline/attribution.d.ts +47 -0
- package/dist/pipeline/attribution.js +226 -0
- package/dist/pipeline/baseline.d.ts +37 -0
- package/dist/pipeline/baseline.js +141 -0
- package/dist/pipeline/cache.d.ts +101 -0
- package/dist/pipeline/cache.js +283 -0
- package/dist/pipeline/calculate-scores.d.ts +102 -0
- package/dist/pipeline/calculate-scores.js +1128 -0
- package/dist/pipeline/callback-delivery.d.ts +50 -0
- package/dist/pipeline/callback-delivery.js +89 -0
- package/dist/pipeline/checks.d.ts +39 -0
- package/dist/pipeline/checks.js +280 -0
- package/dist/pipeline/classify-url.d.ts +61 -0
- package/dist/pipeline/classify-url.js +93 -0
- package/dist/pipeline/compare.d.ts +31 -0
- package/dist/pipeline/compare.js +208 -0
- package/dist/pipeline/coverage-audit.d.ts +39 -0
- package/dist/pipeline/coverage-audit.js +165 -0
- package/dist/pipeline/degradations.d.ts +85 -0
- package/dist/pipeline/degradations.js +242 -0
- package/dist/pipeline/discovery-report.d.ts +55 -0
- package/dist/pipeline/discovery-report.js +178 -0
- package/dist/pipeline/eval-constants.d.ts +68 -0
- package/dist/pipeline/eval-constants.js +111 -0
- package/dist/pipeline/eval-fingerprint.d.ts +66 -0
- package/dist/pipeline/eval-fingerprint.js +175 -0
- package/dist/pipeline/expand-tasks.d.ts +220 -0
- package/dist/pipeline/expand-tasks.js +421 -0
- package/dist/pipeline/failure-modes.d.ts +46 -0
- package/dist/pipeline/failure-modes.js +348 -0
- package/dist/pipeline/fetch-url-content.d.ts +44 -0
- package/dist/pipeline/fetch-url-content.js +93 -0
- package/dist/pipeline/gap-analysis.d.ts +48 -0
- package/dist/pipeline/gap-analysis.js +231 -0
- package/dist/pipeline/generate-configs.d.ts +72 -0
- package/dist/pipeline/generate-configs.js +395 -0
- package/dist/pipeline/grader-api.d.ts +49 -0
- package/dist/pipeline/grader-api.js +200 -0
- package/dist/pipeline/grader-compare-runner.d.ts +44 -0
- package/dist/pipeline/grader-compare-runner.js +301 -0
- package/dist/pipeline/grader-comparison.d.ts +111 -0
- package/dist/pipeline/grader-comparison.js +161 -0
- package/dist/pipeline/grader-consistency-runner.d.ts +60 -0
- package/dist/pipeline/grader-consistency-runner.js +270 -0
- package/dist/pipeline/grader-consistency.d.ts +103 -0
- package/dist/pipeline/grader-consistency.js +146 -0
- package/dist/pipeline/grader-sensitivity-runner.d.ts +40 -0
- package/dist/pipeline/grader-sensitivity-runner.js +282 -0
- package/dist/pipeline/grader-sensitivity.d.ts +94 -0
- package/dist/pipeline/grader-sensitivity.js +144 -0
- package/dist/pipeline/grader-validate-runner.d.ts +38 -0
- package/dist/pipeline/grader-validate-runner.js +229 -0
- package/dist/pipeline/grader-validation.d.ts +107 -0
- package/dist/pipeline/grader-validation.js +169 -0
- package/dist/pipeline/map-request-to-config.d.ts +19 -0
- package/dist/pipeline/map-request-to-config.js +80 -0
- package/dist/pipeline/measure-retrieval.d.ts +59 -0
- package/dist/pipeline/measure-retrieval.js +111 -0
- package/dist/pipeline/mirror-repo-tasks.d.ts +86 -0
- package/dist/pipeline/mirror-repo-tasks.js +350 -0
- package/dist/pipeline/plan-format.d.ts +33 -0
- package/dist/pipeline/plan-format.js +202 -0
- package/dist/pipeline/plan.d.ts +169 -0
- package/dist/pipeline/plan.js +708 -0
- package/dist/pipeline/pr-comment.d.ts +19 -0
- package/dist/pipeline/pr-comment.js +502 -0
- package/dist/pipeline/probe.d.ts +52 -0
- package/dist/pipeline/probe.js +390 -0
- package/dist/pipeline/provenance.d.ts +47 -0
- package/dist/pipeline/provenance.js +146 -0
- package/dist/pipeline/readiness-report.d.ts +87 -0
- package/dist/pipeline/readiness-report.js +205 -0
- package/dist/pipeline/release-classification.d.ts +54 -0
- package/dist/pipeline/release-classification.js +238 -0
- package/dist/pipeline/release-report.d.ts +37 -0
- package/dist/pipeline/release-report.js +222 -0
- package/dist/pipeline/repo-eval-comment.d.ts +37 -0
- package/dist/pipeline/repo-eval-comment.js +165 -0
- package/dist/pipeline/repo-threshold-evaluator.d.ts +89 -0
- package/dist/pipeline/repo-threshold-evaluator.js +162 -0
- package/dist/pipeline/resolve-mappings.d.ts +35 -0
- package/dist/pipeline/resolve-mappings.js +72 -0
- package/dist/pipeline/retrieval-metrics.d.ts +39 -0
- package/dist/pipeline/retrieval-metrics.js +136 -0
- package/dist/pipeline/reverse-mapping.d.ts +67 -0
- package/dist/pipeline/reverse-mapping.js +88 -0
- package/dist/pipeline/schemas.d.ts +9 -0
- package/dist/pipeline/schemas.js +9 -0
- package/dist/pipeline/steps/calculate-scores-step.d.ts +11 -0
- package/dist/pipeline/steps/calculate-scores-step.js +89 -0
- package/dist/pipeline/steps/compare-step.d.ts +18 -0
- package/dist/pipeline/steps/compare-step.js +90 -0
- package/dist/pipeline/steps/eval-step.d.ts +53 -0
- package/dist/pipeline/steps/eval-step.js +347 -0
- package/dist/pipeline/steps/fetch-docs-step.d.ts +11 -0
- package/dist/pipeline/steps/fetch-docs-step.js +84 -0
- package/dist/pipeline/steps/generate-configs-step.d.ts +11 -0
- package/dist/pipeline/steps/generate-configs-step.js +98 -0
- package/dist/pipeline/steps/grader-consistency-step.d.ts +21 -0
- package/dist/pipeline/steps/grader-consistency-step.js +74 -0
- package/dist/pipeline/steps/publish-report-step.d.ts +57 -0
- package/dist/pipeline/steps/publish-report-step.js +243 -0
- package/dist/pipeline/steps/report-step.d.ts +13 -0
- package/dist/pipeline/steps/report-step.js +56 -0
- package/dist/pipeline/steps/update-scores-step.d.ts +11 -0
- package/dist/pipeline/steps/update-scores-step.js +42 -0
- package/dist/pipeline/targeted-loo.d.ts +88 -0
- package/dist/pipeline/targeted-loo.js +203 -0
- package/dist/pipeline/thresholds.d.ts +27 -0
- package/dist/pipeline/thresholds.js +245 -0
- package/dist/pipeline/types.d.ts +10 -0
- package/dist/pipeline/types.js +10 -0
- package/dist/pipeline/validate.d.ts +67 -0
- package/dist/pipeline/validate.js +406 -0
- package/dist/pipeline/webhook-server.d.ts +37 -0
- package/dist/pipeline/webhook-server.js +133 -0
- package/dist/report-store.d.ts +84 -0
- package/dist/report-store.js +208 -0
- package/dist/sanity/client.d.ts +38 -0
- package/dist/sanity/client.js +86 -0
- package/dist/sanity/portable-text.d.ts +11 -0
- package/dist/sanity/portable-text.js +211 -0
- package/dist/sanity/queries.d.ts +133 -0
- package/dist/sanity/queries.js +300 -0
- package/dist/schedules/digest.d.ts +116 -0
- package/dist/schedules/digest.js +156 -0
- package/dist/schedules/index.d.ts +12 -0
- package/dist/schedules/index.js +10 -0
- package/dist/schedules/loader.d.ts +31 -0
- package/dist/schedules/loader.js +73 -0
- package/dist/schedules/schema.d.ts +9 -0
- package/dist/schedules/schema.js +9 -0
- package/dist/scripts/agent-behavior-report.d.ts +19 -0
- package/dist/scripts/agent-behavior-report.js +315 -0
- package/dist/scripts/baseline.d.ts +43 -0
- package/dist/scripts/baseline.js +267 -0
- package/dist/scripts/calculate-scores.d.ts +166 -0
- package/dist/scripts/calculate-scores.js +1296 -0
- package/dist/scripts/compare.d.ts +22 -0
- package/dist/scripts/compare.js +334 -0
- package/dist/scripts/coverage-audit.d.ts +44 -0
- package/dist/scripts/coverage-audit.js +209 -0
- package/dist/scripts/debug-eval.d.ts +19 -0
- package/dist/scripts/debug-eval.js +73 -0
- package/dist/scripts/discovery-report.d.ts +58 -0
- package/dist/scripts/discovery-report.js +250 -0
- package/dist/scripts/fetch-docs.d.ts +35 -0
- package/dist/scripts/fetch-docs.js +472 -0
- package/dist/scripts/generate-configs.d.ts +66 -0
- package/dist/scripts/generate-configs.js +459 -0
- package/dist/scripts/grader-api.d.ts +27 -0
- package/dist/scripts/grader-api.js +206 -0
- package/dist/scripts/grader-compare.d.ts +22 -0
- package/dist/scripts/grader-compare.js +368 -0
- package/dist/scripts/grader-consistency.d.ts +20 -0
- package/dist/scripts/grader-consistency.js +313 -0
- package/dist/scripts/grader-sensitivity.d.ts +22 -0
- package/dist/scripts/grader-sensitivity.js +354 -0
- package/dist/scripts/grader-validate.d.ts +19 -0
- package/dist/scripts/grader-validate.js +267 -0
- package/dist/scripts/measure-retrieval.d.ts +10 -0
- package/dist/scripts/measure-retrieval.js +145 -0
- package/dist/scripts/migrate-tasks-to-content-lake.d.ts +24 -0
- package/dist/scripts/migrate-tasks-to-content-lake.js +327 -0
- package/dist/scripts/pipeline.d.ts +76 -0
- package/dist/scripts/pipeline.js +1031 -0
- package/dist/scripts/pr-comment.d.ts +10 -0
- package/dist/scripts/pr-comment.js +510 -0
- package/dist/scripts/readiness-report.d.ts +88 -0
- package/dist/scripts/readiness-report.js +342 -0
- package/dist/scripts/update-quality-scores.d.ts +15 -0
- package/dist/scripts/update-quality-scores.js +184 -0
- package/dist/scripts/validate-task-sources.d.ts +21 -0
- package/dist/scripts/validate-task-sources.js +210 -0
- package/dist/scripts/validate.d.ts +13 -0
- package/dist/scripts/validate.js +79 -0
- package/dist/scripts/webhook-server.d.ts +26 -0
- package/dist/scripts/webhook-server.js +147 -0
- package/dist/scripts/weekly-digest.d.ts +24 -0
- package/dist/scripts/weekly-digest.js +144 -0
- package/dist/sinks/bigquery/index.d.ts +131 -0
- package/dist/sinks/bigquery/index.js +222 -0
- package/dist/sinks/format-slack.d.ts +64 -0
- package/dist/sinks/format-slack.js +306 -0
- package/dist/sinks/index.d.ts +23 -0
- package/dist/sinks/index.js +18 -0
- package/dist/sinks/loader.d.ts +18 -0
- package/dist/sinks/loader.js +82 -0
- package/dist/sinks/retry.d.ts +24 -0
- package/dist/sinks/retry.js +52 -0
- package/dist/sinks/schema.d.ts +9 -0
- package/dist/sinks/schema.js +9 -0
- package/dist/sinks/slack/format.d.ts +65 -0
- package/dist/sinks/slack/format.js +327 -0
- package/dist/sinks/slack/index.d.ts +27 -0
- package/dist/sinks/slack/index.js +78 -0
- package/dist/sinks/slack-sink.d.ts +27 -0
- package/dist/sinks/slack-sink.js +78 -0
- package/dist/sinks/types.d.ts +59 -0
- package/dist/sinks/types.js +44 -0
- package/dist/sinks/webhook/index.d.ts +19 -0
- package/dist/sinks/webhook/index.js +50 -0
- package/dist/sinks/webhook-sink.d.ts +19 -0
- package/dist/sinks/webhook-sink.js +50 -0
- package/dist/sources.d.ts +104 -0
- package/dist/sources.js +292 -0
- package/dist/webhook/budget.d.ts +42 -0
- package/dist/webhook/budget.js +60 -0
- package/dist/webhook/debounce.d.ts +67 -0
- package/dist/webhook/debounce.js +76 -0
- package/dist/webhook/dispatch.d.ts +45 -0
- package/dist/webhook/dispatch.js +84 -0
- package/dist/webhook/eval-request-handler.d.ts +87 -0
- package/dist/webhook/eval-request-handler.js +181 -0
- package/dist/webhook/handler.d.ts +88 -0
- package/dist/webhook/handler.js +203 -0
- package/dist/webhook/index.d.ts +17 -0
- package/dist/webhook/index.js +12 -0
- package/dist/webhook/types.d.ts +109 -0
- package/dist/webhook/types.js +10 -0
- package/package.json +72 -0
- package/tasks/.expanded.agentic.yaml +51 -0
- package/tasks/.expanded.yaml +66 -0
- package/tasks/frameworks.yaml +98 -0
- package/tasks/functions.yaml +51 -0
- package/tasks/groq.yaml +216 -0
- package/tasks/nextjs-live.yaml +62 -0
- package/tasks/studio-setup.yaml +111 -0
- package/tasks/visual-editing.yaml +120 -0
|
@@ -0,0 +1,531 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @sanity/ailf-core — Pipeline schemas
|
|
3
|
+
*
|
|
4
|
+
* Zod schemas for runtime validation of task YAML files and related configs.
|
|
5
|
+
* These schemas are the single source of truth for what constitutes a valid
|
|
6
|
+
* task definition — the TypeScript interfaces in types.ts are derived from
|
|
7
|
+
* (or should match) these schemas.
|
|
8
|
+
*
|
|
9
|
+
* Extracted from packages/eval/src/pipeline/schemas.ts (Phase 0d).
|
|
10
|
+
*/
|
|
11
|
+
import { z } from "zod";
|
|
12
|
+
/**
|
|
13
|
+
* Schema for a single rubric template (e.g., task-completion, code-correctness).
|
|
14
|
+
*
|
|
15
|
+
* The optional `dimension` field tags which scoring dimension this template
|
|
16
|
+
* belongs to. When present, it propagates into assertion metadata so the
|
|
17
|
+
* scoring engine can classify rubrics structurally instead of heuristically.
|
|
18
|
+
* See docs/design-docs/structured-dimensions.md.
|
|
19
|
+
*/
|
|
20
|
+
export declare const RubricTemplateSchema: z.ZodObject<{
|
|
21
|
+
criteria_label: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
22
|
+
dimension: z.ZodOptional<z.ZodString>;
|
|
23
|
+
header: z.ZodString;
|
|
24
|
+
scale: z.ZodArray<z.ZodString>;
|
|
25
|
+
}, z.core.$strip>;
|
|
26
|
+
/** Inferred TypeScript type for a rubric template. */
|
|
27
|
+
export type RubricTemplate = z.infer<typeof RubricTemplateSchema>;
|
|
28
|
+
/**
|
|
29
|
+
* Schema for the full config/rubrics.yaml config file.
|
|
30
|
+
*
|
|
31
|
+
* Each dimension is scored on a uniform 0–100 scale. The `weights` section
|
|
32
|
+
* defines how dimensions are combined into a composite score (must sum to 1.0).
|
|
33
|
+
*/
|
|
34
|
+
export declare const RubricConfigSchema: z.ZodObject<{
|
|
35
|
+
footer: z.ZodString;
|
|
36
|
+
templates: z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
37
|
+
criteria_label: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
38
|
+
dimension: z.ZodOptional<z.ZodString>;
|
|
39
|
+
header: z.ZodString;
|
|
40
|
+
scale: z.ZodArray<z.ZodString>;
|
|
41
|
+
}, z.core.$strip>>;
|
|
42
|
+
weights: z.ZodRecord<z.ZodString, z.ZodNumber>;
|
|
43
|
+
}, z.core.$strip>;
|
|
44
|
+
/** Inferred TypeScript type for the rubrics config. */
|
|
45
|
+
export type RubricConfig = z.infer<typeof RubricConfigSchema>;
|
|
46
|
+
/**
|
|
47
|
+
* Schema for a single product feature in the feature registry.
|
|
48
|
+
*/
|
|
49
|
+
export declare const FeatureSchema: z.ZodObject<{
|
|
50
|
+
area: z.ZodOptional<z.ZodString>;
|
|
51
|
+
id: z.ZodString;
|
|
52
|
+
name: z.ZodString;
|
|
53
|
+
priority: z.ZodEnum<{
|
|
54
|
+
high: "high";
|
|
55
|
+
low: "low";
|
|
56
|
+
medium: "medium";
|
|
57
|
+
critical: "critical";
|
|
58
|
+
}>;
|
|
59
|
+
sections: z.ZodArray<z.ZodString>;
|
|
60
|
+
status: z.ZodEnum<{
|
|
61
|
+
covered: "covered";
|
|
62
|
+
"out-of-scope": "out-of-scope";
|
|
63
|
+
planned: "planned";
|
|
64
|
+
uncovered: "uncovered";
|
|
65
|
+
}>;
|
|
66
|
+
taskCount: z.ZodOptional<z.ZodNumber>;
|
|
67
|
+
}, z.core.$strip>;
|
|
68
|
+
/** Inferred TypeScript type for a product feature. */
|
|
69
|
+
export type Feature = z.infer<typeof FeatureSchema>;
|
|
70
|
+
/**
|
|
71
|
+
* Schema for the full config/features.yaml config file.
|
|
72
|
+
*/
|
|
73
|
+
export declare const FeatureRegistrySchema: z.ZodObject<{
|
|
74
|
+
features: z.ZodArray<z.ZodObject<{
|
|
75
|
+
area: z.ZodOptional<z.ZodString>;
|
|
76
|
+
id: z.ZodString;
|
|
77
|
+
name: z.ZodString;
|
|
78
|
+
priority: z.ZodEnum<{
|
|
79
|
+
high: "high";
|
|
80
|
+
low: "low";
|
|
81
|
+
medium: "medium";
|
|
82
|
+
critical: "critical";
|
|
83
|
+
}>;
|
|
84
|
+
sections: z.ZodArray<z.ZodString>;
|
|
85
|
+
status: z.ZodEnum<{
|
|
86
|
+
covered: "covered";
|
|
87
|
+
"out-of-scope": "out-of-scope";
|
|
88
|
+
planned: "planned";
|
|
89
|
+
uncovered: "uncovered";
|
|
90
|
+
}>;
|
|
91
|
+
taskCount: z.ZodOptional<z.ZodNumber>;
|
|
92
|
+
}, z.core.$strip>>;
|
|
93
|
+
}, z.core.$strip>;
|
|
94
|
+
/** Inferred TypeScript type for the feature registry. */
|
|
95
|
+
export type FeatureRegistry = z.infer<typeof FeatureRegistrySchema>;
|
|
96
|
+
/**
|
|
97
|
+
* Templated llm-rubric assertion — references a rubric template by key
|
|
98
|
+
* and provides task-specific criteria.
|
|
99
|
+
*/
|
|
100
|
+
declare const TemplatedLlmRubricAssertSchema: z.ZodObject<{
|
|
101
|
+
criteria: z.ZodArray<z.ZodString>;
|
|
102
|
+
template: z.ZodString;
|
|
103
|
+
type: z.ZodLiteral<"llm-rubric">;
|
|
104
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
105
|
+
}, z.core.$strip>;
|
|
106
|
+
export type TemplatedLlmRubricAssert = z.infer<typeof TemplatedLlmRubricAssertSchema>;
|
|
107
|
+
/**
|
|
108
|
+
* Union of all supported Promptfoo assertion types.
|
|
109
|
+
*
|
|
110
|
+
* llm-rubric assertions use the templated format (template + criteria).
|
|
111
|
+
* Other assertion types use their standard value-based format.
|
|
112
|
+
*
|
|
113
|
+
* Note: we use z.union instead of z.discriminatedUnion because multiple
|
|
114
|
+
* schemas share the "llm-rubric" type literal (the old value-based format
|
|
115
|
+
* is gone, but union is more flexible for future additions).
|
|
116
|
+
*/
|
|
117
|
+
export declare const AssertionSchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
118
|
+
criteria: z.ZodArray<z.ZodString>;
|
|
119
|
+
template: z.ZodString;
|
|
120
|
+
type: z.ZodLiteral<"llm-rubric">;
|
|
121
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
122
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
123
|
+
type: z.ZodLiteral<"contains">;
|
|
124
|
+
value: z.ZodString;
|
|
125
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
126
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
127
|
+
type: z.ZodLiteral<"contains-any">;
|
|
128
|
+
value: z.ZodArray<z.ZodString>;
|
|
129
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
130
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
131
|
+
type: z.ZodLiteral<"not-contains">;
|
|
132
|
+
value: z.ZodString;
|
|
133
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
134
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
135
|
+
type: z.ZodLiteral<"javascript">;
|
|
136
|
+
value: z.ZodString;
|
|
137
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
138
|
+
}, z.core.$strip>]>;
|
|
139
|
+
/** Inferred TypeScript type for a single assertion. */
|
|
140
|
+
export type Assertion = z.infer<typeof AssertionSchema>;
|
|
141
|
+
declare const CanonicalDocSchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
142
|
+
reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
143
|
+
slug: z.ZodString;
|
|
144
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
145
|
+
path: z.ZodString;
|
|
146
|
+
reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
147
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
148
|
+
id: z.ZodString;
|
|
149
|
+
reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
150
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
151
|
+
perspective: z.ZodString;
|
|
152
|
+
reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
153
|
+
}, z.core.$strip>]>;
|
|
154
|
+
/** Inferred TypeScript type for a canonical doc entry. */
|
|
155
|
+
export type CanonicalDoc = z.infer<typeof CanonicalDocSchema>;
|
|
156
|
+
/**
|
|
157
|
+
* Schema for the new single-definition task format.
|
|
158
|
+
* Each task is defined once with an id field; the pipeline auto-generates
|
|
159
|
+
* gold + baseline variants.
|
|
160
|
+
*/
|
|
161
|
+
export declare const SingleTaskSchema: z.ZodObject<{
|
|
162
|
+
assert: z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
|
|
163
|
+
criteria: z.ZodArray<z.ZodString>;
|
|
164
|
+
template: z.ZodString;
|
|
165
|
+
type: z.ZodLiteral<"llm-rubric">;
|
|
166
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
167
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
168
|
+
type: z.ZodLiteral<"contains">;
|
|
169
|
+
value: z.ZodString;
|
|
170
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
171
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
172
|
+
type: z.ZodLiteral<"contains-any">;
|
|
173
|
+
value: z.ZodArray<z.ZodString>;
|
|
174
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
175
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
176
|
+
type: z.ZodLiteral<"not-contains">;
|
|
177
|
+
value: z.ZodString;
|
|
178
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
179
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
180
|
+
type: z.ZodLiteral<"javascript">;
|
|
181
|
+
value: z.ZodString;
|
|
182
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
183
|
+
}, z.core.$strip>]>>;
|
|
184
|
+
baseline: z.ZodOptional<z.ZodObject<{
|
|
185
|
+
enabled: z.ZodOptional<z.ZodBoolean>;
|
|
186
|
+
rubric: z.ZodOptional<z.ZodEnum<{
|
|
187
|
+
full: "full";
|
|
188
|
+
abbreviated: "abbreviated";
|
|
189
|
+
none: "none";
|
|
190
|
+
}>>;
|
|
191
|
+
}, z.core.$strip>>;
|
|
192
|
+
canonical_docs: z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
|
|
193
|
+
reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
194
|
+
slug: z.ZodString;
|
|
195
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
196
|
+
path: z.ZodString;
|
|
197
|
+
reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
198
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
199
|
+
id: z.ZodString;
|
|
200
|
+
reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
201
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
202
|
+
perspective: z.ZodString;
|
|
203
|
+
reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
204
|
+
}, z.core.$strip>]>>;
|
|
205
|
+
description: z.ZodString;
|
|
206
|
+
doc_coverage: z.ZodOptional<z.ZodBoolean>;
|
|
207
|
+
id: z.ZodString;
|
|
208
|
+
reference_solution: z.ZodString;
|
|
209
|
+
vars: z.ZodObject<{
|
|
210
|
+
docs: z.ZodString;
|
|
211
|
+
task: z.ZodString;
|
|
212
|
+
}, z.core.$loose>;
|
|
213
|
+
}, z.core.$strip>;
|
|
214
|
+
/** Inferred TypeScript type for a single task definition. */
|
|
215
|
+
export type SingleTask = z.infer<typeof SingleTaskSchema>;
|
|
216
|
+
/**
|
|
217
|
+
* Schema for legacy task entries (the old paired gold/baseline format).
|
|
218
|
+
* These have description and vars but no id field.
|
|
219
|
+
*/
|
|
220
|
+
export declare const LegacyTaskSchema: z.ZodObject<{
|
|
221
|
+
assert: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
|
|
222
|
+
criteria: z.ZodArray<z.ZodString>;
|
|
223
|
+
template: z.ZodString;
|
|
224
|
+
type: z.ZodLiteral<"llm-rubric">;
|
|
225
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
226
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
227
|
+
type: z.ZodLiteral<"contains">;
|
|
228
|
+
value: z.ZodString;
|
|
229
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
230
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
231
|
+
type: z.ZodLiteral<"contains-any">;
|
|
232
|
+
value: z.ZodArray<z.ZodString>;
|
|
233
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
234
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
235
|
+
type: z.ZodLiteral<"not-contains">;
|
|
236
|
+
value: z.ZodString;
|
|
237
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
238
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
239
|
+
type: z.ZodLiteral<"javascript">;
|
|
240
|
+
value: z.ZodString;
|
|
241
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
242
|
+
}, z.core.$strip>]>>>;
|
|
243
|
+
description: z.ZodString;
|
|
244
|
+
transform: z.ZodOptional<z.ZodString>;
|
|
245
|
+
vars: z.ZodObject<{
|
|
246
|
+
task: z.ZodString;
|
|
247
|
+
}, z.core.$loose>;
|
|
248
|
+
}, z.core.$strip>;
|
|
249
|
+
/** Inferred TypeScript type for a legacy task entry. */
|
|
250
|
+
export type LegacyTask = z.infer<typeof LegacyTaskSchema>;
|
|
251
|
+
/**
|
|
252
|
+
* A single task entry that can be either format.
|
|
253
|
+
* Entries with an id field are validated as SingleTask;
|
|
254
|
+
* entries without are validated as LegacyTask.
|
|
255
|
+
*/
|
|
256
|
+
export declare const TaskEntrySchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
257
|
+
assert: z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
|
|
258
|
+
criteria: z.ZodArray<z.ZodString>;
|
|
259
|
+
template: z.ZodString;
|
|
260
|
+
type: z.ZodLiteral<"llm-rubric">;
|
|
261
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
262
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
263
|
+
type: z.ZodLiteral<"contains">;
|
|
264
|
+
value: z.ZodString;
|
|
265
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
266
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
267
|
+
type: z.ZodLiteral<"contains-any">;
|
|
268
|
+
value: z.ZodArray<z.ZodString>;
|
|
269
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
270
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
271
|
+
type: z.ZodLiteral<"not-contains">;
|
|
272
|
+
value: z.ZodString;
|
|
273
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
274
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
275
|
+
type: z.ZodLiteral<"javascript">;
|
|
276
|
+
value: z.ZodString;
|
|
277
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
278
|
+
}, z.core.$strip>]>>;
|
|
279
|
+
baseline: z.ZodOptional<z.ZodObject<{
|
|
280
|
+
enabled: z.ZodOptional<z.ZodBoolean>;
|
|
281
|
+
rubric: z.ZodOptional<z.ZodEnum<{
|
|
282
|
+
full: "full";
|
|
283
|
+
abbreviated: "abbreviated";
|
|
284
|
+
none: "none";
|
|
285
|
+
}>>;
|
|
286
|
+
}, z.core.$strip>>;
|
|
287
|
+
canonical_docs: z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
|
|
288
|
+
reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
289
|
+
slug: z.ZodString;
|
|
290
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
291
|
+
path: z.ZodString;
|
|
292
|
+
reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
293
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
294
|
+
id: z.ZodString;
|
|
295
|
+
reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
296
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
297
|
+
perspective: z.ZodString;
|
|
298
|
+
reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
299
|
+
}, z.core.$strip>]>>;
|
|
300
|
+
description: z.ZodString;
|
|
301
|
+
doc_coverage: z.ZodOptional<z.ZodBoolean>;
|
|
302
|
+
id: z.ZodString;
|
|
303
|
+
reference_solution: z.ZodString;
|
|
304
|
+
vars: z.ZodObject<{
|
|
305
|
+
docs: z.ZodString;
|
|
306
|
+
task: z.ZodString;
|
|
307
|
+
}, z.core.$loose>;
|
|
308
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
309
|
+
assert: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
|
|
310
|
+
criteria: z.ZodArray<z.ZodString>;
|
|
311
|
+
template: z.ZodString;
|
|
312
|
+
type: z.ZodLiteral<"llm-rubric">;
|
|
313
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
314
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
315
|
+
type: z.ZodLiteral<"contains">;
|
|
316
|
+
value: z.ZodString;
|
|
317
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
318
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
319
|
+
type: z.ZodLiteral<"contains-any">;
|
|
320
|
+
value: z.ZodArray<z.ZodString>;
|
|
321
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
322
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
323
|
+
type: z.ZodLiteral<"not-contains">;
|
|
324
|
+
value: z.ZodString;
|
|
325
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
326
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
327
|
+
type: z.ZodLiteral<"javascript">;
|
|
328
|
+
value: z.ZodString;
|
|
329
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
330
|
+
}, z.core.$strip>]>>>;
|
|
331
|
+
description: z.ZodString;
|
|
332
|
+
transform: z.ZodOptional<z.ZodString>;
|
|
333
|
+
vars: z.ZodObject<{
|
|
334
|
+
task: z.ZodString;
|
|
335
|
+
}, z.core.$loose>;
|
|
336
|
+
}, z.core.$strip>]>;
|
|
337
|
+
/** Inferred TypeScript type for any task entry. */
|
|
338
|
+
export type TaskEntryParsed = z.infer<typeof TaskEntrySchema>;
|
|
339
|
+
/**
|
|
340
|
+
* Schema for an entire task YAML file: an array of task entries.
|
|
341
|
+
*/
|
|
342
|
+
export declare const TaskFileSchema: z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
|
|
343
|
+
assert: z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
|
|
344
|
+
criteria: z.ZodArray<z.ZodString>;
|
|
345
|
+
template: z.ZodString;
|
|
346
|
+
type: z.ZodLiteral<"llm-rubric">;
|
|
347
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
348
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
349
|
+
type: z.ZodLiteral<"contains">;
|
|
350
|
+
value: z.ZodString;
|
|
351
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
352
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
353
|
+
type: z.ZodLiteral<"contains-any">;
|
|
354
|
+
value: z.ZodArray<z.ZodString>;
|
|
355
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
356
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
357
|
+
type: z.ZodLiteral<"not-contains">;
|
|
358
|
+
value: z.ZodString;
|
|
359
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
360
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
361
|
+
type: z.ZodLiteral<"javascript">;
|
|
362
|
+
value: z.ZodString;
|
|
363
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
364
|
+
}, z.core.$strip>]>>;
|
|
365
|
+
baseline: z.ZodOptional<z.ZodObject<{
|
|
366
|
+
enabled: z.ZodOptional<z.ZodBoolean>;
|
|
367
|
+
rubric: z.ZodOptional<z.ZodEnum<{
|
|
368
|
+
full: "full";
|
|
369
|
+
abbreviated: "abbreviated";
|
|
370
|
+
none: "none";
|
|
371
|
+
}>>;
|
|
372
|
+
}, z.core.$strip>>;
|
|
373
|
+
canonical_docs: z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
|
|
374
|
+
reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
375
|
+
slug: z.ZodString;
|
|
376
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
377
|
+
path: z.ZodString;
|
|
378
|
+
reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
379
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
380
|
+
id: z.ZodString;
|
|
381
|
+
reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
382
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
383
|
+
perspective: z.ZodString;
|
|
384
|
+
reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
385
|
+
}, z.core.$strip>]>>;
|
|
386
|
+
description: z.ZodString;
|
|
387
|
+
doc_coverage: z.ZodOptional<z.ZodBoolean>;
|
|
388
|
+
id: z.ZodString;
|
|
389
|
+
reference_solution: z.ZodString;
|
|
390
|
+
vars: z.ZodObject<{
|
|
391
|
+
docs: z.ZodString;
|
|
392
|
+
task: z.ZodString;
|
|
393
|
+
}, z.core.$loose>;
|
|
394
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
395
|
+
assert: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
|
|
396
|
+
criteria: z.ZodArray<z.ZodString>;
|
|
397
|
+
template: z.ZodString;
|
|
398
|
+
type: z.ZodLiteral<"llm-rubric">;
|
|
399
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
400
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
401
|
+
type: z.ZodLiteral<"contains">;
|
|
402
|
+
value: z.ZodString;
|
|
403
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
404
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
405
|
+
type: z.ZodLiteral<"contains-any">;
|
|
406
|
+
value: z.ZodArray<z.ZodString>;
|
|
407
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
408
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
409
|
+
type: z.ZodLiteral<"not-contains">;
|
|
410
|
+
value: z.ZodString;
|
|
411
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
412
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
413
|
+
type: z.ZodLiteral<"javascript">;
|
|
414
|
+
value: z.ZodString;
|
|
415
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
416
|
+
}, z.core.$strip>]>>>;
|
|
417
|
+
description: z.ZodString;
|
|
418
|
+
transform: z.ZodOptional<z.ZodString>;
|
|
419
|
+
vars: z.ZodObject<{
|
|
420
|
+
task: z.ZodString;
|
|
421
|
+
}, z.core.$loose>;
|
|
422
|
+
}, z.core.$strip>]>>;
|
|
423
|
+
/** Inferred TypeScript type for a parsed task file. */
|
|
424
|
+
export type TaskFile = z.infer<typeof TaskFileSchema>;
|
|
425
|
+
/**
|
|
426
|
+
* Schema for per-dimension threshold values.
|
|
427
|
+
* Keys use kebab-case to match YAML convention; the threshold engine
|
|
428
|
+
* normalizes to camelCase for comparison against FeatureScore fields.
|
|
429
|
+
*/
|
|
430
|
+
export declare const ThresholdDimensionsSchema: z.ZodObject<{
|
|
431
|
+
"code-correctness": z.ZodOptional<z.ZodNumber>;
|
|
432
|
+
"doc-coverage": z.ZodOptional<z.ZodNumber>;
|
|
433
|
+
"task-completion": z.ZodOptional<z.ZodNumber>;
|
|
434
|
+
}, z.core.$strip>;
|
|
435
|
+
/** Inferred TypeScript type for threshold dimension overrides. */
|
|
436
|
+
export type ThresholdDimensions = z.infer<typeof ThresholdDimensionsSchema>;
|
|
437
|
+
/**
|
|
438
|
+
* Schema for threshold defaults (and per-area overrides).
|
|
439
|
+
* All fields are optional in per-area overrides; defaults must have composite.
|
|
440
|
+
*/
|
|
441
|
+
export declare const ThresholdDefaultsSchema: z.ZodObject<{
|
|
442
|
+
ceiling: z.ZodOptional<z.ZodNumber>;
|
|
443
|
+
composite: z.ZodNumber;
|
|
444
|
+
dimensions: z.ZodOptional<z.ZodObject<{
|
|
445
|
+
"code-correctness": z.ZodOptional<z.ZodNumber>;
|
|
446
|
+
"doc-coverage": z.ZodOptional<z.ZodNumber>;
|
|
447
|
+
"task-completion": z.ZodOptional<z.ZodNumber>;
|
|
448
|
+
}, z.core.$strip>>;
|
|
449
|
+
"doc-lift": z.ZodOptional<z.ZodNumber>;
|
|
450
|
+
}, z.core.$strip>;
|
|
451
|
+
/** Inferred TypeScript type for threshold defaults. */
|
|
452
|
+
export type ThresholdDefaults = z.infer<typeof ThresholdDefaultsSchema>;
|
|
453
|
+
/**
|
|
454
|
+
* Schema for regression thresholds (all values should be negative or zero).
|
|
455
|
+
*/
|
|
456
|
+
export declare const RegressionThresholdsSchema: z.ZodObject<{
|
|
457
|
+
composite: z.ZodNumber;
|
|
458
|
+
"per-area": z.ZodNumber;
|
|
459
|
+
"per-dimension": z.ZodNumber;
|
|
460
|
+
}, z.core.$strip>;
|
|
461
|
+
/** Inferred TypeScript type for regression thresholds. */
|
|
462
|
+
export type RegressionThresholds = z.infer<typeof RegressionThresholdsSchema>;
|
|
463
|
+
/**
|
|
464
|
+
* Schema for severity classification rules.
|
|
465
|
+
*/
|
|
466
|
+
export declare const SeverityConfigSchema: z.ZodObject<{
|
|
467
|
+
critical: z.ZodOptional<z.ZodObject<{
|
|
468
|
+
"composite-below": z.ZodOptional<z.ZodNumber>;
|
|
469
|
+
"negative-doc-lift": z.ZodOptional<z.ZodBoolean>;
|
|
470
|
+
}, z.core.$strip>>;
|
|
471
|
+
info: z.ZodOptional<z.ZodObject<{
|
|
472
|
+
"composite-below": z.ZodOptional<z.ZodNumber>;
|
|
473
|
+
}, z.core.$strip>>;
|
|
474
|
+
warning: z.ZodOptional<z.ZodObject<{
|
|
475
|
+
"composite-below": z.ZodOptional<z.ZodNumber>;
|
|
476
|
+
"regression-exceeds": z.ZodOptional<z.ZodNumber>;
|
|
477
|
+
}, z.core.$strip>>;
|
|
478
|
+
}, z.core.$strip>;
|
|
479
|
+
/** Inferred TypeScript type for severity classification. */
|
|
480
|
+
export type SeverityConfig = z.infer<typeof SeverityConfigSchema>;
|
|
481
|
+
/**
|
|
482
|
+
* Schema for the complete config/thresholds.yaml file.
|
|
483
|
+
*/
|
|
484
|
+
export declare const ThresholdConfigSchema: z.ZodObject<{
|
|
485
|
+
areas: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
486
|
+
ceiling: z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
|
|
487
|
+
composite: z.ZodOptional<z.ZodNumber>;
|
|
488
|
+
dimensions: z.ZodOptional<z.ZodOptional<z.ZodObject<{
|
|
489
|
+
"code-correctness": z.ZodOptional<z.ZodNumber>;
|
|
490
|
+
"doc-coverage": z.ZodOptional<z.ZodNumber>;
|
|
491
|
+
"task-completion": z.ZodOptional<z.ZodNumber>;
|
|
492
|
+
}, z.core.$strip>>>;
|
|
493
|
+
"doc-lift": z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
|
|
494
|
+
}, z.core.$strip>>>;
|
|
495
|
+
defaults: z.ZodObject<{
|
|
496
|
+
ceiling: z.ZodOptional<z.ZodNumber>;
|
|
497
|
+
composite: z.ZodNumber;
|
|
498
|
+
dimensions: z.ZodOptional<z.ZodObject<{
|
|
499
|
+
"code-correctness": z.ZodOptional<z.ZodNumber>;
|
|
500
|
+
"doc-coverage": z.ZodOptional<z.ZodNumber>;
|
|
501
|
+
"task-completion": z.ZodOptional<z.ZodNumber>;
|
|
502
|
+
}, z.core.$strip>>;
|
|
503
|
+
"doc-lift": z.ZodOptional<z.ZodNumber>;
|
|
504
|
+
}, z.core.$strip>;
|
|
505
|
+
regression: z.ZodOptional<z.ZodObject<{
|
|
506
|
+
composite: z.ZodNumber;
|
|
507
|
+
"per-area": z.ZodNumber;
|
|
508
|
+
"per-dimension": z.ZodNumber;
|
|
509
|
+
}, z.core.$strip>>;
|
|
510
|
+
severity: z.ZodOptional<z.ZodObject<{
|
|
511
|
+
critical: z.ZodOptional<z.ZodObject<{
|
|
512
|
+
"composite-below": z.ZodOptional<z.ZodNumber>;
|
|
513
|
+
"negative-doc-lift": z.ZodOptional<z.ZodBoolean>;
|
|
514
|
+
}, z.core.$strip>>;
|
|
515
|
+
info: z.ZodOptional<z.ZodObject<{
|
|
516
|
+
"composite-below": z.ZodOptional<z.ZodNumber>;
|
|
517
|
+
}, z.core.$strip>>;
|
|
518
|
+
warning: z.ZodOptional<z.ZodObject<{
|
|
519
|
+
"composite-below": z.ZodOptional<z.ZodNumber>;
|
|
520
|
+
"regression-exceeds": z.ZodOptional<z.ZodNumber>;
|
|
521
|
+
}, z.core.$strip>>;
|
|
522
|
+
}, z.core.$strip>>;
|
|
523
|
+
}, z.core.$strip>;
|
|
524
|
+
/** Inferred TypeScript type for the full threshold config. */
|
|
525
|
+
export type ThresholdConfig = z.infer<typeof ThresholdConfigSchema>;
|
|
526
|
+
/**
|
|
527
|
+
* Format a Zod error into human-readable lines suitable for CLI output.
|
|
528
|
+
* Each line includes the path (e.g., "[2].vars.task") and the error message.
|
|
529
|
+
*/
|
|
530
|
+
export declare function formatZodErrors(error: z.ZodError): string[];
|
|
531
|
+
export {};
|