@sanity/ailf 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +89 -0
- package/bin/ailf.js +64 -0
- package/canonical/grader-references/README.md +88 -0
- package/canonical/grader-references/groq.yaml +234 -0
- package/canonical/grader-references/studio-setup.yaml +275 -0
- package/canonical/reference-solutions/.gitkeep +1 -0
- package/canonical/reference-solutions/frameworks/nuxt.ts +119 -0
- package/canonical/reference-solutions/frameworks/remix.tsx +100 -0
- package/canonical/reference-solutions/functions/publish-webhook.ts +60 -0
- package/canonical/reference-solutions/groq/advanced-filtering.ts +379 -0
- package/canonical/reference-solutions/groq/blog-queries.ts +137 -0
- package/canonical/reference-solutions/groq/joins-references.ts +300 -0
- package/canonical/reference-solutions/nextjs/app-router-integration.tsx +128 -0
- package/canonical/reference-solutions/studio-setup/blog-schema.ts +143 -0
- package/canonical/reference-solutions/studio-setup/custom-tool.tsx +78 -0
- package/canonical/reference-solutions/visual-editing/live-preview.tsx +137 -0
- package/canonical/reference-solutions/visual-editing/presentation-nextjs.tsx +130 -0
- package/config/airbyte/ai_literacy_framework.connector.yaml +639 -0
- package/config/bigquery/README.md +74 -0
- package/config/bigquery/views/area_scores.sql +87 -0
- package/config/bigquery/views/reports.sql +49 -0
- package/config/features.yaml +116 -0
- package/config/models.yaml +115 -0
- package/config/prompts.yaml +75 -0
- package/config/rubrics.yaml +62 -0
- package/config/schedules.yaml +43 -0
- package/config/sinks.yaml +54 -0
- package/config/sources.yaml +51 -0
- package/config/thresholds.yaml +49 -0
- package/dist/_vendor/ailf-core/examples/index.d.ts +190 -0
- package/dist/_vendor/ailf-core/examples/index.js +285 -0
- package/dist/_vendor/ailf-core/index.d.ts +17 -0
- package/dist/_vendor/ailf-core/index.js +17 -0
- package/dist/_vendor/ailf-core/ports/cache-store.d.ts +72 -0
- package/dist/_vendor/ailf-core/ports/cache-store.js +17 -0
- package/dist/_vendor/ailf-core/ports/config-source.d.ts +33 -0
- package/dist/_vendor/ailf-core/ports/config-source.js +15 -0
- package/dist/_vendor/ailf-core/ports/context.d.ts +172 -0
- package/dist/_vendor/ailf-core/ports/context.js +14 -0
- package/dist/_vendor/ailf-core/ports/doc-fetcher.d.ts +131 -0
- package/dist/_vendor/ailf-core/ports/doc-fetcher.js +12 -0
- package/dist/_vendor/ailf-core/ports/eval-runner.d.ts +24 -0
- package/dist/_vendor/ailf-core/ports/eval-runner.js +8 -0
- package/dist/_vendor/ailf-core/ports/index.d.ts +15 -0
- package/dist/_vendor/ailf-core/ports/index.js +7 -0
- package/dist/_vendor/ailf-core/ports/logger.d.ts +36 -0
- package/dist/_vendor/ailf-core/ports/logger.js +11 -0
- package/dist/_vendor/ailf-core/ports/pipeline-step.d.ts +46 -0
- package/dist/_vendor/ailf-core/ports/pipeline-step.js +8 -0
- package/dist/_vendor/ailf-core/ports/task-source.d.ts +159 -0
- package/dist/_vendor/ailf-core/ports/task-source.js +72 -0
- package/dist/_vendor/ailf-core/schemas/callback-payload.d.ts +24 -0
- package/dist/_vendor/ailf-core/schemas/callback-payload.js +29 -0
- package/dist/_vendor/ailf-core/schemas/eval-config.d.ts +55 -0
- package/dist/_vendor/ailf-core/schemas/eval-config.js +78 -0
- package/dist/_vendor/ailf-core/schemas/index.d.ts +16 -0
- package/dist/_vendor/ailf-core/schemas/index.js +16 -0
- package/dist/_vendor/ailf-core/schemas/pipeline-request.d.ts +125 -0
- package/dist/_vendor/ailf-core/schemas/pipeline-request.js +67 -0
- package/dist/_vendor/ailf-core/schemas/pipeline.d.ts +531 -0
- package/dist/_vendor/ailf-core/schemas/pipeline.js +318 -0
- package/dist/_vendor/ailf-core/schemas/schedules.d.ts +68 -0
- package/dist/_vendor/ailf-core/schemas/schedules.js +74 -0
- package/dist/_vendor/ailf-core/schemas/sinks.d.ts +207 -0
- package/dist/_vendor/ailf-core/schemas/sinks.js +108 -0
- package/dist/_vendor/ailf-core/services/comparison-formatters.d.ts +18 -0
- package/dist/_vendor/ailf-core/services/comparison-formatters.js +189 -0
- package/dist/_vendor/ailf-core/services/config-helpers.d.ts +41 -0
- package/dist/_vendor/ailf-core/services/config-helpers.js +86 -0
- package/dist/_vendor/ailf-core/services/index.d.ts +12 -0
- package/dist/_vendor/ailf-core/services/index.js +12 -0
- package/dist/_vendor/ailf-core/services/scoring.d.ts +49 -0
- package/dist/_vendor/ailf-core/services/scoring.js +222 -0
- package/dist/_vendor/ailf-core/types/index.d.ts +1082 -0
- package/dist/_vendor/ailf-core/types/index.js +21 -0
- package/dist/_vendor/ailf-core/types/scoring-input.d.ts +54 -0
- package/dist/_vendor/ailf-core/types/scoring-input.js +9 -0
- package/dist/_vendor/ailf-shared/dimension-names.d.ts +21 -0
- package/dist/_vendor/ailf-shared/dimension-names.js +27 -0
- package/dist/_vendor/ailf-shared/document-ref.d.ts +29 -0
- package/dist/_vendor/ailf-shared/document-ref.js +1 -0
- package/dist/_vendor/ailf-shared/eval-modes.d.ts +12 -0
- package/dist/_vendor/ailf-shared/eval-modes.js +8 -0
- package/dist/_vendor/ailf-shared/index.d.ts +16 -0
- package/dist/_vendor/ailf-shared/index.js +16 -0
- package/dist/_vendor/ailf-shared/noise-threshold.d.ts +9 -0
- package/dist/_vendor/ailf-shared/noise-threshold.js +9 -0
- package/dist/_vendor/ailf-shared/score-grades.d.ts +17 -0
- package/dist/_vendor/ailf-shared/score-grades.js +23 -0
- package/dist/adapters/cache/content-lake-cache.d.ts +24 -0
- package/dist/adapters/cache/content-lake-cache.js +59 -0
- package/dist/adapters/cache/filesystem-cache.d.ts +18 -0
- package/dist/adapters/cache/filesystem-cache.js +54 -0
- package/dist/adapters/cache/index.d.ts +2 -0
- package/dist/adapters/cache/index.js +2 -0
- package/dist/adapters/config-sources/cli-config-adapter.d.ts +17 -0
- package/dist/adapters/config-sources/cli-config-adapter.js +23 -0
- package/dist/adapters/config-sources/file-config-adapter.d.ts +26 -0
- package/dist/adapters/config-sources/file-config-adapter.js +96 -0
- package/dist/adapters/config-sources/index.d.ts +2 -0
- package/dist/adapters/config-sources/index.js +2 -0
- package/dist/adapters/doc-fetchers/index.d.ts +1 -0
- package/dist/adapters/doc-fetchers/index.js +1 -0
- package/dist/adapters/doc-fetchers/sanity-doc-fetcher.d.ts +76 -0
- package/dist/adapters/doc-fetchers/sanity-doc-fetcher.js +620 -0
- package/dist/adapters/eval-runners/index.d.ts +1 -0
- package/dist/adapters/eval-runners/index.js +1 -0
- package/dist/adapters/eval-runners/promptfoo-eval-adapter.d.ts +14 -0
- package/dist/adapters/eval-runners/promptfoo-eval-adapter.js +63 -0
- package/dist/adapters/index.d.ts +12 -0
- package/dist/adapters/index.js +12 -0
- package/dist/adapters/loggers/console-logger.d.ts +22 -0
- package/dist/adapters/loggers/console-logger.js +54 -0
- package/dist/adapters/loggers/index.d.ts +9 -0
- package/dist/adapters/loggers/index.js +9 -0
- package/dist/adapters/loggers/json-logger.d.ts +18 -0
- package/dist/adapters/loggers/json-logger.js +33 -0
- package/dist/adapters/loggers/quiet-logger.d.ts +16 -0
- package/dist/adapters/loggers/quiet-logger.js +30 -0
- package/dist/adapters/task-sources/composite-task-source.d.ts +20 -0
- package/dist/adapters/task-sources/composite-task-source.js +59 -0
- package/dist/adapters/task-sources/content-lake-task-source.d.ts +20 -0
- package/dist/adapters/task-sources/content-lake-task-source.js +219 -0
- package/dist/adapters/task-sources/index.d.ts +7 -0
- package/dist/adapters/task-sources/index.js +7 -0
- package/dist/adapters/task-sources/repo-schemas.d.ts +245 -0
- package/dist/adapters/task-sources/repo-schemas.js +234 -0
- package/dist/adapters/task-sources/repo-task-source.d.ts +22 -0
- package/dist/adapters/task-sources/repo-task-source.js +104 -0
- package/dist/adapters/task-sources/repo-trigger.d.ts +52 -0
- package/dist/adapters/task-sources/repo-trigger.js +153 -0
- package/dist/adapters/task-sources/repo-validation.d.ts +49 -0
- package/dist/adapters/task-sources/repo-validation.js +164 -0
- package/dist/adapters/task-sources/yaml-task-source.d.ts +18 -0
- package/dist/adapters/task-sources/yaml-task-source.js +136 -0
- package/dist/agent-observer/agentic-provider.d.ts +132 -0
- package/dist/agent-observer/agentic-provider.js +983 -0
- package/dist/agent-observer/classifier.d.ts +62 -0
- package/dist/agent-observer/classifier.js +269 -0
- package/dist/agent-observer/index.d.ts +7 -0
- package/dist/agent-observer/index.js +4 -0
- package/dist/agent-observer/pricing.d.ts +35 -0
- package/dist/agent-observer/pricing.js +82 -0
- package/dist/agent-observer/provider.d.ts +77 -0
- package/dist/agent-observer/provider.js +151 -0
- package/dist/agent-observer/proxy.d.ts +91 -0
- package/dist/agent-observer/proxy.js +321 -0
- package/dist/agent-observer/test-imports.d.ts +7 -0
- package/dist/agent-observer/test-imports.js +185 -0
- package/dist/agent-observer/types.d.ts +137 -0
- package/dist/agent-observer/types.js +16 -0
- package/dist/assertions/source-isolation.d.ts +72 -0
- package/dist/assertions/source-isolation.js +117 -0
- package/dist/cli.d.ts +24 -0
- package/dist/cli.js +199 -0
- package/dist/commands/agent-report.d.ts +5 -0
- package/dist/commands/agent-report.js +69 -0
- package/dist/commands/baseline.d.ts +9 -0
- package/dist/commands/baseline.js +141 -0
- package/dist/commands/cache.d.ts +13 -0
- package/dist/commands/cache.js +135 -0
- package/dist/commands/calculate-scores.d.ts +8 -0
- package/dist/commands/calculate-scores.js +48 -0
- package/dist/commands/compare.d.ts +8 -0
- package/dist/commands/compare.js +120 -0
- package/dist/commands/completion.d.ts +18 -0
- package/dist/commands/completion.js +260 -0
- package/dist/commands/coverage-audit.d.ts +7 -0
- package/dist/commands/coverage-audit.js +40 -0
- package/dist/commands/discovery-report.d.ts +10 -0
- package/dist/commands/discovery-report.js +44 -0
- package/dist/commands/eval.d.ts +9 -0
- package/dist/commands/eval.js +35 -0
- package/dist/commands/explain-handler.d.ts +34 -0
- package/dist/commands/explain-handler.js +719 -0
- package/dist/commands/fetch-docs.d.ts +8 -0
- package/dist/commands/fetch-docs.js +128 -0
- package/dist/commands/generate-configs.d.ts +8 -0
- package/dist/commands/generate-configs.js +46 -0
- package/dist/commands/grader/index.d.ts +11 -0
- package/dist/commands/grader/index.js +118 -0
- package/dist/commands/init.d.ts +19 -0
- package/dist/commands/init.js +150 -0
- package/dist/commands/interactive.d.ts +12 -0
- package/dist/commands/interactive.js +238 -0
- package/dist/commands/lookup-doc.d.ts +15 -0
- package/dist/commands/lookup-doc.js +84 -0
- package/dist/commands/measure-retrieval.d.ts +5 -0
- package/dist/commands/measure-retrieval.js +65 -0
- package/dist/commands/pipeline-action.d.ts +71 -0
- package/dist/commands/pipeline-action.js +305 -0
- package/dist/commands/pipeline.d.ts +62 -0
- package/dist/commands/pipeline.js +53 -0
- package/dist/commands/pr-comment.d.ts +8 -0
- package/dist/commands/pr-comment.js +47 -0
- package/dist/commands/publish.d.ts +26 -0
- package/dist/commands/publish.js +253 -0
- package/dist/commands/readiness-report.d.ts +10 -0
- package/dist/commands/readiness-report.js +104 -0
- package/dist/commands/shared/options.d.ts +29 -0
- package/dist/commands/shared/options.js +57 -0
- package/dist/commands/update-quality-scores.d.ts +5 -0
- package/dist/commands/update-quality-scores.js +20 -0
- package/dist/commands/validate-tasks.d.ts +16 -0
- package/dist/commands/validate-tasks.js +93 -0
- package/dist/commands/validate.d.ts +9 -0
- package/dist/commands/validate.js +73 -0
- package/dist/commands/webhook-server.d.ts +5 -0
- package/dist/commands/webhook-server.js +30 -0
- package/dist/commands/weekly-digest.d.ts +10 -0
- package/dist/commands/weekly-digest.js +104 -0
- package/dist/composition-root.d.ts +26 -0
- package/dist/composition-root.js +107 -0
- package/dist/interpolate.d.ts +26 -0
- package/dist/interpolate.js +70 -0
- package/dist/job-store.d.ts +104 -0
- package/dist/job-store.js +188 -0
- package/dist/lib/agent-behavior-report.d.ts +8 -0
- package/dist/lib/agent-behavior-report.js +185 -0
- package/dist/lib/baseline.d.ts +19 -0
- package/dist/lib/baseline.js +153 -0
- package/dist/lib/calculate-scores.d.ts +23 -0
- package/dist/lib/calculate-scores.js +42 -0
- package/dist/lib/compare.d.ts +18 -0
- package/dist/lib/compare.js +170 -0
- package/dist/lib/coverage-audit.d.ts +4 -0
- package/dist/lib/coverage-audit.js +42 -0
- package/dist/lib/discovery-report.d.ts +13 -0
- package/dist/lib/discovery-report.js +57 -0
- package/dist/lib/fetch-docs.d.ts +30 -0
- package/dist/lib/fetch-docs.js +171 -0
- package/dist/lib/generate-configs.d.ts +25 -0
- package/dist/lib/generate-configs.js +42 -0
- package/dist/lib/grader-api.d.ts +21 -0
- package/dist/lib/grader-api.js +34 -0
- package/dist/lib/grader-compare.d.ts +19 -0
- package/dist/lib/grader-compare.js +91 -0
- package/dist/lib/grader-consistency.d.ts +27 -0
- package/dist/lib/grader-consistency.js +79 -0
- package/dist/lib/grader-sensitivity.d.ts +19 -0
- package/dist/lib/grader-sensitivity.js +75 -0
- package/dist/lib/grader-validate.d.ts +19 -0
- package/dist/lib/grader-validate.js +78 -0
- package/dist/lib/measure-retrieval.d.ts +14 -0
- package/dist/lib/measure-retrieval.js +71 -0
- package/dist/lib/pr-comment.d.ts +16 -0
- package/dist/lib/pr-comment.js +28 -0
- package/dist/lib/readiness-report.d.ts +13 -0
- package/dist/lib/readiness-report.js +108 -0
- package/dist/lib/webhook-server.d.ts +11 -0
- package/dist/lib/webhook-server.js +24 -0
- package/dist/lib/weekly-digest.d.ts +24 -0
- package/dist/lib/weekly-digest.js +148 -0
- package/dist/orchestration/build-app-context.d.ts +27 -0
- package/dist/orchestration/build-app-context.js +81 -0
- package/dist/orchestration/build-step-sequence.d.ts +15 -0
- package/dist/orchestration/build-step-sequence.js +84 -0
- package/dist/orchestration/config-to-source-overrides.d.ts +9 -0
- package/dist/orchestration/config-to-source-overrides.js +28 -0
- package/dist/orchestration/env-bridge.d.ts +21 -0
- package/dist/orchestration/env-bridge.js +66 -0
- package/dist/orchestration/index.d.ts +11 -0
- package/dist/orchestration/index.js +11 -0
- package/dist/orchestration/pipeline-orchestrator.d.ts +24 -0
- package/dist/orchestration/pipeline-orchestrator.js +153 -0
- package/dist/orchestration/step-runner.d.ts +20 -0
- package/dist/orchestration/step-runner.js +88 -0
- package/dist/orchestration/steps/calculate-scores-step.d.ts +13 -0
- package/dist/orchestration/steps/calculate-scores-step.js +95 -0
- package/dist/orchestration/steps/callback-step.d.ts +24 -0
- package/dist/orchestration/steps/callback-step.js +76 -0
- package/dist/orchestration/steps/compare-step.d.ts +14 -0
- package/dist/orchestration/steps/compare-step.js +92 -0
- package/dist/orchestration/steps/discovery-report-step.d.ts +13 -0
- package/dist/orchestration/steps/discovery-report-step.js +55 -0
- package/dist/orchestration/steps/fetch-docs-shell.d.ts +17 -0
- package/dist/orchestration/steps/fetch-docs-shell.js +30 -0
- package/dist/orchestration/steps/fetch-docs-step.d.ts +14 -0
- package/dist/orchestration/steps/fetch-docs-step.js +135 -0
- package/dist/orchestration/steps/gap-analysis-step.d.ts +16 -0
- package/dist/orchestration/steps/gap-analysis-step.js +136 -0
- package/dist/orchestration/steps/generate-configs-step.d.ts +14 -0
- package/dist/orchestration/steps/generate-configs-step.js +85 -0
- package/dist/orchestration/steps/grader-consistency-step.d.ts +13 -0
- package/dist/orchestration/steps/grader-consistency-step.js +64 -0
- package/dist/orchestration/steps/index.d.ts +19 -0
- package/dist/orchestration/steps/index.js +19 -0
- package/dist/orchestration/steps/mirror-repo-tasks-step.d.ts +21 -0
- package/dist/orchestration/steps/mirror-repo-tasks-step.js +94 -0
- package/dist/orchestration/steps/publish-report-step.d.ts +26 -0
- package/dist/orchestration/steps/publish-report-step.js +216 -0
- package/dist/orchestration/steps/readiness-step.d.ts +13 -0
- package/dist/orchestration/steps/readiness-step.js +91 -0
- package/dist/orchestration/steps/report-step.d.ts +12 -0
- package/dist/orchestration/steps/report-step.js +49 -0
- package/dist/orchestration/steps/run-eval-step.d.ts +17 -0
- package/dist/orchestration/steps/run-eval-step.js +195 -0
- package/dist/orchestration/steps/validate-step.d.ts +12 -0
- package/dist/orchestration/steps/validate-step.js +41 -0
- package/dist/pipeline/agent-behavior-report.d.ts +53 -0
- package/dist/pipeline/agent-behavior-report.js +132 -0
- package/dist/pipeline/attribution.d.ts +47 -0
- package/dist/pipeline/attribution.js +226 -0
- package/dist/pipeline/baseline.d.ts +37 -0
- package/dist/pipeline/baseline.js +141 -0
- package/dist/pipeline/cache.d.ts +101 -0
- package/dist/pipeline/cache.js +283 -0
- package/dist/pipeline/calculate-scores.d.ts +102 -0
- package/dist/pipeline/calculate-scores.js +1128 -0
- package/dist/pipeline/callback-delivery.d.ts +50 -0
- package/dist/pipeline/callback-delivery.js +89 -0
- package/dist/pipeline/checks.d.ts +39 -0
- package/dist/pipeline/checks.js +280 -0
- package/dist/pipeline/classify-url.d.ts +61 -0
- package/dist/pipeline/classify-url.js +93 -0
- package/dist/pipeline/compare.d.ts +31 -0
- package/dist/pipeline/compare.js +208 -0
- package/dist/pipeline/coverage-audit.d.ts +39 -0
- package/dist/pipeline/coverage-audit.js +165 -0
- package/dist/pipeline/degradations.d.ts +85 -0
- package/dist/pipeline/degradations.js +242 -0
- package/dist/pipeline/discovery-report.d.ts +55 -0
- package/dist/pipeline/discovery-report.js +178 -0
- package/dist/pipeline/eval-constants.d.ts +68 -0
- package/dist/pipeline/eval-constants.js +111 -0
- package/dist/pipeline/eval-fingerprint.d.ts +66 -0
- package/dist/pipeline/eval-fingerprint.js +175 -0
- package/dist/pipeline/expand-tasks.d.ts +220 -0
- package/dist/pipeline/expand-tasks.js +421 -0
- package/dist/pipeline/failure-modes.d.ts +46 -0
- package/dist/pipeline/failure-modes.js +348 -0
- package/dist/pipeline/fetch-url-content.d.ts +44 -0
- package/dist/pipeline/fetch-url-content.js +93 -0
- package/dist/pipeline/gap-analysis.d.ts +48 -0
- package/dist/pipeline/gap-analysis.js +231 -0
- package/dist/pipeline/generate-configs.d.ts +72 -0
- package/dist/pipeline/generate-configs.js +395 -0
- package/dist/pipeline/grader-api.d.ts +49 -0
- package/dist/pipeline/grader-api.js +200 -0
- package/dist/pipeline/grader-compare-runner.d.ts +44 -0
- package/dist/pipeline/grader-compare-runner.js +301 -0
- package/dist/pipeline/grader-comparison.d.ts +111 -0
- package/dist/pipeline/grader-comparison.js +161 -0
- package/dist/pipeline/grader-consistency-runner.d.ts +60 -0
- package/dist/pipeline/grader-consistency-runner.js +270 -0
- package/dist/pipeline/grader-consistency.d.ts +103 -0
- package/dist/pipeline/grader-consistency.js +146 -0
- package/dist/pipeline/grader-sensitivity-runner.d.ts +40 -0
- package/dist/pipeline/grader-sensitivity-runner.js +282 -0
- package/dist/pipeline/grader-sensitivity.d.ts +94 -0
- package/dist/pipeline/grader-sensitivity.js +144 -0
- package/dist/pipeline/grader-validate-runner.d.ts +38 -0
- package/dist/pipeline/grader-validate-runner.js +229 -0
- package/dist/pipeline/grader-validation.d.ts +107 -0
- package/dist/pipeline/grader-validation.js +169 -0
- package/dist/pipeline/map-request-to-config.d.ts +19 -0
- package/dist/pipeline/map-request-to-config.js +80 -0
- package/dist/pipeline/measure-retrieval.d.ts +59 -0
- package/dist/pipeline/measure-retrieval.js +111 -0
- package/dist/pipeline/mirror-repo-tasks.d.ts +86 -0
- package/dist/pipeline/mirror-repo-tasks.js +350 -0
- package/dist/pipeline/plan-format.d.ts +33 -0
- package/dist/pipeline/plan-format.js +202 -0
- package/dist/pipeline/plan.d.ts +169 -0
- package/dist/pipeline/plan.js +708 -0
- package/dist/pipeline/pr-comment.d.ts +19 -0
- package/dist/pipeline/pr-comment.js +502 -0
- package/dist/pipeline/probe.d.ts +52 -0
- package/dist/pipeline/probe.js +390 -0
- package/dist/pipeline/provenance.d.ts +47 -0
- package/dist/pipeline/provenance.js +146 -0
- package/dist/pipeline/readiness-report.d.ts +87 -0
- package/dist/pipeline/readiness-report.js +205 -0
- package/dist/pipeline/release-classification.d.ts +54 -0
- package/dist/pipeline/release-classification.js +238 -0
- package/dist/pipeline/release-report.d.ts +37 -0
- package/dist/pipeline/release-report.js +222 -0
- package/dist/pipeline/repo-eval-comment.d.ts +37 -0
- package/dist/pipeline/repo-eval-comment.js +165 -0
- package/dist/pipeline/repo-threshold-evaluator.d.ts +89 -0
- package/dist/pipeline/repo-threshold-evaluator.js +162 -0
- package/dist/pipeline/resolve-mappings.d.ts +35 -0
- package/dist/pipeline/resolve-mappings.js +72 -0
- package/dist/pipeline/retrieval-metrics.d.ts +39 -0
- package/dist/pipeline/retrieval-metrics.js +136 -0
- package/dist/pipeline/reverse-mapping.d.ts +67 -0
- package/dist/pipeline/reverse-mapping.js +88 -0
- package/dist/pipeline/schemas.d.ts +9 -0
- package/dist/pipeline/schemas.js +9 -0
- package/dist/pipeline/steps/calculate-scores-step.d.ts +11 -0
- package/dist/pipeline/steps/calculate-scores-step.js +89 -0
- package/dist/pipeline/steps/compare-step.d.ts +18 -0
- package/dist/pipeline/steps/compare-step.js +90 -0
- package/dist/pipeline/steps/eval-step.d.ts +53 -0
- package/dist/pipeline/steps/eval-step.js +347 -0
- package/dist/pipeline/steps/fetch-docs-step.d.ts +11 -0
- package/dist/pipeline/steps/fetch-docs-step.js +84 -0
- package/dist/pipeline/steps/generate-configs-step.d.ts +11 -0
- package/dist/pipeline/steps/generate-configs-step.js +98 -0
- package/dist/pipeline/steps/grader-consistency-step.d.ts +21 -0
- package/dist/pipeline/steps/grader-consistency-step.js +74 -0
- package/dist/pipeline/steps/publish-report-step.d.ts +57 -0
- package/dist/pipeline/steps/publish-report-step.js +243 -0
- package/dist/pipeline/steps/report-step.d.ts +13 -0
- package/dist/pipeline/steps/report-step.js +56 -0
- package/dist/pipeline/steps/update-scores-step.d.ts +11 -0
- package/dist/pipeline/steps/update-scores-step.js +42 -0
- package/dist/pipeline/targeted-loo.d.ts +88 -0
- package/dist/pipeline/targeted-loo.js +203 -0
- package/dist/pipeline/thresholds.d.ts +27 -0
- package/dist/pipeline/thresholds.js +245 -0
- package/dist/pipeline/types.d.ts +10 -0
- package/dist/pipeline/types.js +10 -0
- package/dist/pipeline/validate.d.ts +67 -0
- package/dist/pipeline/validate.js +406 -0
- package/dist/pipeline/webhook-server.d.ts +37 -0
- package/dist/pipeline/webhook-server.js +133 -0
- package/dist/report-store.d.ts +84 -0
- package/dist/report-store.js +208 -0
- package/dist/sanity/client.d.ts +38 -0
- package/dist/sanity/client.js +86 -0
- package/dist/sanity/portable-text.d.ts +11 -0
- package/dist/sanity/portable-text.js +211 -0
- package/dist/sanity/queries.d.ts +133 -0
- package/dist/sanity/queries.js +300 -0
- package/dist/schedules/digest.d.ts +116 -0
- package/dist/schedules/digest.js +156 -0
- package/dist/schedules/index.d.ts +12 -0
- package/dist/schedules/index.js +10 -0
- package/dist/schedules/loader.d.ts +31 -0
- package/dist/schedules/loader.js +73 -0
- package/dist/schedules/schema.d.ts +9 -0
- package/dist/schedules/schema.js +9 -0
- package/dist/scripts/agent-behavior-report.d.ts +19 -0
- package/dist/scripts/agent-behavior-report.js +315 -0
- package/dist/scripts/baseline.d.ts +43 -0
- package/dist/scripts/baseline.js +267 -0
- package/dist/scripts/calculate-scores.d.ts +166 -0
- package/dist/scripts/calculate-scores.js +1296 -0
- package/dist/scripts/compare.d.ts +22 -0
- package/dist/scripts/compare.js +334 -0
- package/dist/scripts/coverage-audit.d.ts +44 -0
- package/dist/scripts/coverage-audit.js +209 -0
- package/dist/scripts/debug-eval.d.ts +19 -0
- package/dist/scripts/debug-eval.js +73 -0
- package/dist/scripts/discovery-report.d.ts +58 -0
- package/dist/scripts/discovery-report.js +250 -0
- package/dist/scripts/fetch-docs.d.ts +35 -0
- package/dist/scripts/fetch-docs.js +472 -0
- package/dist/scripts/generate-configs.d.ts +66 -0
- package/dist/scripts/generate-configs.js +459 -0
- package/dist/scripts/grader-api.d.ts +27 -0
- package/dist/scripts/grader-api.js +206 -0
- package/dist/scripts/grader-compare.d.ts +22 -0
- package/dist/scripts/grader-compare.js +368 -0
- package/dist/scripts/grader-consistency.d.ts +20 -0
- package/dist/scripts/grader-consistency.js +313 -0
- package/dist/scripts/grader-sensitivity.d.ts +22 -0
- package/dist/scripts/grader-sensitivity.js +354 -0
- package/dist/scripts/grader-validate.d.ts +19 -0
- package/dist/scripts/grader-validate.js +267 -0
- package/dist/scripts/measure-retrieval.d.ts +10 -0
- package/dist/scripts/measure-retrieval.js +145 -0
- package/dist/scripts/migrate-tasks-to-content-lake.d.ts +24 -0
- package/dist/scripts/migrate-tasks-to-content-lake.js +327 -0
- package/dist/scripts/pipeline.d.ts +76 -0
- package/dist/scripts/pipeline.js +1031 -0
- package/dist/scripts/pr-comment.d.ts +10 -0
- package/dist/scripts/pr-comment.js +510 -0
- package/dist/scripts/readiness-report.d.ts +88 -0
- package/dist/scripts/readiness-report.js +342 -0
- package/dist/scripts/update-quality-scores.d.ts +15 -0
- package/dist/scripts/update-quality-scores.js +184 -0
- package/dist/scripts/validate-task-sources.d.ts +21 -0
- package/dist/scripts/validate-task-sources.js +210 -0
- package/dist/scripts/validate.d.ts +13 -0
- package/dist/scripts/validate.js +79 -0
- package/dist/scripts/webhook-server.d.ts +26 -0
- package/dist/scripts/webhook-server.js +147 -0
- package/dist/scripts/weekly-digest.d.ts +24 -0
- package/dist/scripts/weekly-digest.js +144 -0
- package/dist/sinks/bigquery/index.d.ts +131 -0
- package/dist/sinks/bigquery/index.js +222 -0
- package/dist/sinks/format-slack.d.ts +64 -0
- package/dist/sinks/format-slack.js +306 -0
- package/dist/sinks/index.d.ts +23 -0
- package/dist/sinks/index.js +18 -0
- package/dist/sinks/loader.d.ts +18 -0
- package/dist/sinks/loader.js +82 -0
- package/dist/sinks/retry.d.ts +24 -0
- package/dist/sinks/retry.js +52 -0
- package/dist/sinks/schema.d.ts +9 -0
- package/dist/sinks/schema.js +9 -0
- package/dist/sinks/slack/format.d.ts +65 -0
- package/dist/sinks/slack/format.js +327 -0
- package/dist/sinks/slack/index.d.ts +27 -0
- package/dist/sinks/slack/index.js +78 -0
- package/dist/sinks/slack-sink.d.ts +27 -0
- package/dist/sinks/slack-sink.js +78 -0
- package/dist/sinks/types.d.ts +59 -0
- package/dist/sinks/types.js +44 -0
- package/dist/sinks/webhook/index.d.ts +19 -0
- package/dist/sinks/webhook/index.js +50 -0
- package/dist/sinks/webhook-sink.d.ts +19 -0
- package/dist/sinks/webhook-sink.js +50 -0
- package/dist/sources.d.ts +104 -0
- package/dist/sources.js +292 -0
- package/dist/webhook/budget.d.ts +42 -0
- package/dist/webhook/budget.js +60 -0
- package/dist/webhook/debounce.d.ts +67 -0
- package/dist/webhook/debounce.js +76 -0
- package/dist/webhook/dispatch.d.ts +45 -0
- package/dist/webhook/dispatch.js +84 -0
- package/dist/webhook/eval-request-handler.d.ts +87 -0
- package/dist/webhook/eval-request-handler.js +181 -0
- package/dist/webhook/handler.d.ts +88 -0
- package/dist/webhook/handler.js +203 -0
- package/dist/webhook/index.d.ts +17 -0
- package/dist/webhook/index.js +12 -0
- package/dist/webhook/types.d.ts +109 -0
- package/dist/webhook/types.js +10 -0
- package/package.json +72 -0
- package/tasks/.expanded.agentic.yaml +51 -0
- package/tasks/.expanded.yaml +66 -0
- package/tasks/frameworks.yaml +98 -0
- package/tasks/functions.yaml +51 -0
- package/tasks/groq.yaml +216 -0
- package/tasks/nextjs-live.yaml +62 -0
- package/tasks/studio-setup.yaml +111 -0
- package/tasks/visual-editing.yaml +120 -0
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* lib/webhook-server.ts — DEPRECATED re-export shim.
|
|
3
|
+
*
|
|
4
|
+
* The real implementation has moved to pipeline/webhook-server.ts.
|
|
5
|
+
* This shim preserves backward compatibility for direct CLI invocation.
|
|
6
|
+
*
|
|
7
|
+
* TODO: Update all importers to use pipeline/webhook-server.ts, then delete this file.
|
|
8
|
+
*
|
|
9
|
+
* @deprecated Import from ../pipeline/webhook-server.js instead.
|
|
10
|
+
*/
|
|
11
|
+
import { dirname, resolve } from "path";
|
|
12
|
+
import { fileURLToPath } from "url";
|
|
13
|
+
import { startWebhookServer } from "../pipeline/webhook-server.js";
|
|
14
|
+
export { startWebhookServer, } from "../pipeline/webhook-server.js";
|
|
15
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
16
|
+
const ROOT = resolve(__dirname, "..", "..");
|
|
17
|
+
// When imported directly (e.g., `tsx src/lib/webhook-server.ts`), start the server
|
|
18
|
+
startWebhookServer({
|
|
19
|
+
dailyBudget: parseInt(process.env.WEBHOOK_DAILY_BUDGET ?? "20", 10),
|
|
20
|
+
debounceMs: parseInt(process.env.WEBHOOK_DEBOUNCE_MS ?? "10000", 10),
|
|
21
|
+
githubToken: process.env.GITHUB_TOKEN ?? "",
|
|
22
|
+
port: parseInt(process.env.WEBHOOK_PORT ?? "3333", 10),
|
|
23
|
+
rootDir: ROOT,
|
|
24
|
+
});
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* weekly-digest.ts
|
|
3
|
+
*
|
|
4
|
+
* CLI script to generate and deliver a weekly evaluation digest.
|
|
5
|
+
*
|
|
6
|
+
* Queries the Sanity Content Lake for all reports within the configured
|
|
7
|
+
* lookback window, computes trend analysis, and delivers the digest
|
|
8
|
+
* via configured channels (Slack, stdout, or both).
|
|
9
|
+
*
|
|
10
|
+
* Usage:
|
|
11
|
+
* pnpm weekly-digest # send to configured Slack webhook
|
|
12
|
+
* pnpm weekly-digest --dry-run # print to stdout only
|
|
13
|
+
* pnpm weekly-digest --lookback 14 # 14-day lookback window
|
|
14
|
+
* pnpm weekly-digest --json # output raw JSON
|
|
15
|
+
*
|
|
16
|
+
* Environment variables:
|
|
17
|
+
* SLACK_WEBHOOK_URL — Slack incoming webhook URL
|
|
18
|
+
* SANITY_API_TOKEN — Sanity read token
|
|
19
|
+
* AILF_TRIGGER_TYPE — set to "scheduled" by the cron workflow
|
|
20
|
+
* AILF_SCHEDULE — the schedule name (e.g., "weekly-digest")
|
|
21
|
+
*
|
|
22
|
+
* @see docs/design-docs/report-store/implementation.md — Phase 5
|
|
23
|
+
*/
|
|
24
|
+
export declare function main(): Promise<void>;
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* weekly-digest.ts
|
|
3
|
+
*
|
|
4
|
+
* CLI script to generate and deliver a weekly evaluation digest.
|
|
5
|
+
*
|
|
6
|
+
* Queries the Sanity Content Lake for all reports within the configured
|
|
7
|
+
* lookback window, computes trend analysis, and delivers the digest
|
|
8
|
+
* via configured channels (Slack, stdout, or both).
|
|
9
|
+
*
|
|
10
|
+
* Usage:
|
|
11
|
+
* pnpm weekly-digest # send to configured Slack webhook
|
|
12
|
+
* pnpm weekly-digest --dry-run # print to stdout only
|
|
13
|
+
* pnpm weekly-digest --lookback 14 # 14-day lookback window
|
|
14
|
+
* pnpm weekly-digest --json # output raw JSON
|
|
15
|
+
*
|
|
16
|
+
* Environment variables:
|
|
17
|
+
* SLACK_WEBHOOK_URL — Slack incoming webhook URL
|
|
18
|
+
* SANITY_API_TOKEN — Sanity read token
|
|
19
|
+
* AILF_TRIGGER_TYPE — set to "scheduled" by the cron workflow
|
|
20
|
+
* AILF_SCHEDULE — the schedule name (e.g., "weekly-digest")
|
|
21
|
+
*
|
|
22
|
+
* @see docs/design-docs/report-store/implementation.md — Phase 5
|
|
23
|
+
*/
|
|
24
|
+
import { config as dotenvConfig } from "dotenv";
|
|
25
|
+
import { existsSync } from "fs";
|
|
26
|
+
import { dirname, resolve } from "path";
|
|
27
|
+
import { fileURLToPath } from "url";
|
|
28
|
+
import { generateDigest } from "../schedules/digest.js";
|
|
29
|
+
import { getDigestConfig } from "../schedules/loader.js";
|
|
30
|
+
import { formatWeeklyDigest } from "../sinks/slack/format.js";
|
|
31
|
+
// Load root .env (same override behavior as pipeline.ts)
|
|
32
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
33
|
+
const envPath = resolve(__dirname, "..", "..", "..", "..", ".env");
|
|
34
|
+
if (existsSync(envPath)) {
|
|
35
|
+
dotenvConfig({ override: true, path: envPath });
|
|
36
|
+
}
|
|
37
|
+
// ---------------------------------------------------------------------------
|
|
38
|
+
// CLI argument parsing
|
|
39
|
+
// ---------------------------------------------------------------------------
|
|
40
|
+
const args = process.argv.slice(2);
|
|
41
|
+
function getOption(name) {
|
|
42
|
+
const idx = args.indexOf(`--${name}`);
|
|
43
|
+
return idx >= 0 && idx + 1 < args.length ? args[idx + 1] : undefined;
|
|
44
|
+
}
|
|
45
|
+
function hasFlag(name) {
|
|
46
|
+
return args.includes(`--${name}`);
|
|
47
|
+
}
|
|
48
|
+
const DRY_RUN = hasFlag("dry-run");
|
|
49
|
+
const JSON_OUTPUT = hasFlag("json");
|
|
50
|
+
const lookbackOverride = getOption("lookback");
|
|
51
|
+
// ---------------------------------------------------------------------------
|
|
52
|
+
// Main
|
|
53
|
+
// ---------------------------------------------------------------------------
|
|
54
|
+
export async function main() {
|
|
55
|
+
console.log();
|
|
56
|
+
console.log("=== AI Literacy Weekly Digest ===");
|
|
57
|
+
console.log();
|
|
58
|
+
// Load digest config
|
|
59
|
+
const digestConfig = getDigestConfig();
|
|
60
|
+
const lookbackDays = lookbackOverride
|
|
61
|
+
? parseInt(lookbackOverride, 10)
|
|
62
|
+
: (digestConfig?.lookbackDays ?? 7);
|
|
63
|
+
console.log(` Lookback: ${lookbackDays} days`);
|
|
64
|
+
console.log(` Mode: ${DRY_RUN ? "dry run (stdout only)" : "live"}`);
|
|
65
|
+
console.log();
|
|
66
|
+
// Generate digest — uses AILF_REPORT_* env vars for report store access,
|
|
67
|
+
// independent of SANITY_DATASET/SANITY_PROJECT_ID (which control doc evaluation)
|
|
68
|
+
const digest = await generateDigest({
|
|
69
|
+
dataset: process.env.AILF_REPORT_DATASET,
|
|
70
|
+
lookbackDays,
|
|
71
|
+
projectId: process.env.AILF_REPORT_PROJECT_ID,
|
|
72
|
+
token: process.env.AILF_REPORT_SANITY_API_TOKEN ?? process.env.SANITY_API_TOKEN,
|
|
73
|
+
});
|
|
74
|
+
if (!digest) {
|
|
75
|
+
console.log(" No reports found in the lookback window. Nothing to send.");
|
|
76
|
+
process.exit(0);
|
|
77
|
+
}
|
|
78
|
+
// Output
|
|
79
|
+
console.log(` Reports found: ${digest.reportCount}`);
|
|
80
|
+
console.log(` Overall: ${Math.round(digest.overallLatest)} (${digest.overallTrend})`);
|
|
81
|
+
console.log(` Improved: ${digest.improved.join(", ") || "none"}`);
|
|
82
|
+
console.log(` Regressed: ${digest.regressed.join(", ") || "none"}`);
|
|
83
|
+
console.log(` Stable: ${digest.stable.join(", ") || "none"}`);
|
|
84
|
+
console.log();
|
|
85
|
+
if (JSON_OUTPUT) {
|
|
86
|
+
console.log(JSON.stringify(digest, null, 2));
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
// Format for Slack
|
|
90
|
+
const message = formatWeeklyDigest(digest);
|
|
91
|
+
if (DRY_RUN) {
|
|
92
|
+
console.log(" --- Slack Message Preview ---");
|
|
93
|
+
console.log(` Text: ${message.text}`);
|
|
94
|
+
console.log();
|
|
95
|
+
for (const block of message.blocks) {
|
|
96
|
+
if (block.text) {
|
|
97
|
+
console.log(` [${block.type}] ${block.text.text}`);
|
|
98
|
+
}
|
|
99
|
+
if (block.fields) {
|
|
100
|
+
for (const field of block.fields) {
|
|
101
|
+
console.log(` [field] ${field.text}`);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
if (block.elements) {
|
|
105
|
+
for (const el of block.elements) {
|
|
106
|
+
console.log(` [element] ${el.text}`);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
console.log();
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
// Deliver via Slack
|
|
114
|
+
const webhookUrl = digestConfig?.slackWebhookUrl ?? process.env.SLACK_WEBHOOK_URL;
|
|
115
|
+
if (!webhookUrl) {
|
|
116
|
+
console.warn(" ⚠️ No Slack webhook URL configured. Set SLACK_WEBHOOK_URL or configure in schedules.yaml");
|
|
117
|
+
console.log(" Printing digest to stdout instead:");
|
|
118
|
+
console.log();
|
|
119
|
+
console.log(` ${message.text}`);
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
console.log(" Sending to Slack...");
|
|
123
|
+
try {
|
|
124
|
+
const response = await fetch(webhookUrl, {
|
|
125
|
+
body: JSON.stringify(message),
|
|
126
|
+
headers: { "Content-Type": "application/json" },
|
|
127
|
+
method: "POST",
|
|
128
|
+
});
|
|
129
|
+
if (response.ok) {
|
|
130
|
+
console.log(" ✅ Digest sent successfully");
|
|
131
|
+
}
|
|
132
|
+
else {
|
|
133
|
+
const text = await response.text();
|
|
134
|
+
console.warn(` ⚠️ Slack delivery failed: ${response.status} ${text}`);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
catch (error) {
|
|
138
|
+
console.warn(` ⚠️ Slack delivery error: ${error instanceof Error ? error.message : String(error)}`);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
// Only run when invoked directly (not when imported)
|
|
142
|
+
if (process.argv[1]?.endsWith("weekly-digest.ts") ||
|
|
143
|
+
process.argv[1]?.endsWith("weekly-digest.js")) {
|
|
144
|
+
main().catch((error) => {
|
|
145
|
+
console.error("Fatal error:", error);
|
|
146
|
+
process.exit(1);
|
|
147
|
+
});
|
|
148
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bridge from legacy ResolvedOptions to the new AppContext.
|
|
3
|
+
*
|
|
4
|
+
* This mapping function exists during the Phase 3→7 migration. It converts
|
|
5
|
+
* the old ResolvedOptions shape to ResolvedConfig, then delegates to
|
|
6
|
+
* createAppContext() (the composition root).
|
|
7
|
+
*
|
|
8
|
+
* Once all commands construct ResolvedConfig directly (or use --config),
|
|
9
|
+
* this bridge can be deleted.
|
|
10
|
+
*/
|
|
11
|
+
import type { AppContext, ResolvedConfig } from "../_vendor/ailf-core/index.d.ts";
|
|
12
|
+
import type { ResolvedOptions } from "../commands/pipeline-action.js";
|
|
13
|
+
/**
|
|
14
|
+
* Map legacy ResolvedOptions to ResolvedConfig.
|
|
15
|
+
*
|
|
16
|
+
* This is a mechanical mapping — every field in ResolvedConfig has a
|
|
17
|
+
* corresponding field in ResolvedOptions. Fields that don't map directly
|
|
18
|
+
* are derived (e.g., areas from areaOption).
|
|
19
|
+
*/
|
|
20
|
+
export declare function mapToResolvedConfig(opts: ResolvedOptions, rootDir: string): ResolvedConfig;
|
|
21
|
+
/**
|
|
22
|
+
* Build an AppContext from legacy ResolvedOptions.
|
|
23
|
+
*
|
|
24
|
+
* Delegates to createAppContext() (composition root) after mapping
|
|
25
|
+
* ResolvedOptions → ResolvedConfig.
|
|
26
|
+
*/
|
|
27
|
+
export declare function buildAppContext(opts: ResolvedOptions, rootDir: string): AppContext;
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bridge from legacy ResolvedOptions to the new AppContext.
|
|
3
|
+
*
|
|
4
|
+
* This mapping function exists during the Phase 3→7 migration. It converts
|
|
5
|
+
* the old ResolvedOptions shape to ResolvedConfig, then delegates to
|
|
6
|
+
* createAppContext() (the composition root).
|
|
7
|
+
*
|
|
8
|
+
* Once all commands construct ResolvedConfig directly (or use --config),
|
|
9
|
+
* this bridge can be deleted.
|
|
10
|
+
*/
|
|
11
|
+
import { createAppContext } from "../composition-root.js";
|
|
12
|
+
/**
|
|
13
|
+
* Map legacy ResolvedOptions to ResolvedConfig.
|
|
14
|
+
*
|
|
15
|
+
* This is a mechanical mapping — every field in ResolvedConfig has a
|
|
16
|
+
* corresponding field in ResolvedOptions. Fields that don't map directly
|
|
17
|
+
* are derived (e.g., areas from areaOption).
|
|
18
|
+
*/
|
|
19
|
+
export function mapToResolvedConfig(opts, rootDir) {
|
|
20
|
+
return {
|
|
21
|
+
rootDir,
|
|
22
|
+
mode: opts.mode,
|
|
23
|
+
debug: opts.debug,
|
|
24
|
+
areas: opts.areaOption
|
|
25
|
+
?.split(",")
|
|
26
|
+
.map((s) => s.trim())
|
|
27
|
+
.filter(Boolean),
|
|
28
|
+
tasks: opts.taskOption
|
|
29
|
+
?.split(",")
|
|
30
|
+
.map((s) => s.trim())
|
|
31
|
+
.filter(Boolean),
|
|
32
|
+
changedDocs: opts.changedDocsOption
|
|
33
|
+
?.split(",")
|
|
34
|
+
.map((s) => s.trim())
|
|
35
|
+
.filter(Boolean),
|
|
36
|
+
source: opts.source,
|
|
37
|
+
skipFetch: opts.skipFetch,
|
|
38
|
+
skipEval: opts.skipEval,
|
|
39
|
+
compareEnabled: opts.compareEnabled,
|
|
40
|
+
compareThreshold: opts.compareThreshold,
|
|
41
|
+
compareBaseline: opts.compareBaseline,
|
|
42
|
+
gapAnalysisEnabled: opts.gapAnalysisEnabled,
|
|
43
|
+
readinessEnabled: opts.readinessEnabled,
|
|
44
|
+
discoveryReportEnabled: opts.discoveryReportEnabled,
|
|
45
|
+
publishEnabled: opts.publishEnabled,
|
|
46
|
+
publishTag: opts.publishTag,
|
|
47
|
+
noCache: opts.noCache,
|
|
48
|
+
noRemoteCache: opts.noRemoteCache,
|
|
49
|
+
graderReplications: opts.graderReplications,
|
|
50
|
+
outputPath: opts.outputPath,
|
|
51
|
+
urls: opts.urlArgs.length > 0 ? opts.urlArgs : undefined,
|
|
52
|
+
headers: opts.headerArgs.length > 0
|
|
53
|
+
? Object.fromEntries(opts.headerArgs.map((h) => {
|
|
54
|
+
const [key, ...rest] = h.split(":");
|
|
55
|
+
return [key.trim(), rest.join(":").trim()];
|
|
56
|
+
}))
|
|
57
|
+
: undefined,
|
|
58
|
+
allowedOrigins: opts.allowedOriginArgs.length > 0 ? opts.allowedOriginArgs : undefined,
|
|
59
|
+
searchMode: opts.searchMode ?? "open",
|
|
60
|
+
concurrency: opts.concurrency,
|
|
61
|
+
promptfooUrl: opts.promptfooUrl,
|
|
62
|
+
datasetOverride: opts.datasetOverride,
|
|
63
|
+
projectIdOverride: opts.projectIdOverride,
|
|
64
|
+
perspectiveOverride: opts.perspectiveOverride,
|
|
65
|
+
studioOriginOverride: opts.studioOriginOverride,
|
|
66
|
+
sanityDocumentArgs: opts.sanityDocumentArgs.length > 0 ? opts.sanityDocumentArgs : undefined,
|
|
67
|
+
beforeOption: opts.beforeOption,
|
|
68
|
+
taskSourceType: opts.taskSourceType,
|
|
69
|
+
repoTasksPath: opts.repoTasksPath,
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Build an AppContext from legacy ResolvedOptions.
|
|
74
|
+
*
|
|
75
|
+
* Delegates to createAppContext() (composition root) after mapping
|
|
76
|
+
* ResolvedOptions → ResolvedConfig.
|
|
77
|
+
*/
|
|
78
|
+
export function buildAppContext(opts, rootDir) {
|
|
79
|
+
const config = mapToResolvedConfig(opts, rootDir);
|
|
80
|
+
return createAppContext(config);
|
|
81
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Build the step sequence based on resolved config.
|
|
3
|
+
*
|
|
4
|
+
* This is the declarative description of the pipeline — a list of
|
|
5
|
+
* PipelineStep objects determined by config flags like skipFetch,
|
|
6
|
+
* skipEval, compareEnabled, etc.
|
|
7
|
+
*/
|
|
8
|
+
import type { AppContext, PipelineStep } from "../_vendor/ailf-core/index.d.ts";
|
|
9
|
+
/**
|
|
10
|
+
* Build the pipeline step sequence based on config.
|
|
11
|
+
*
|
|
12
|
+
* @param ctx - The application context (used for config)
|
|
13
|
+
* @param pipelineStart - Pipeline start timestamp (for publish step)
|
|
14
|
+
*/
|
|
15
|
+
export declare function buildStepSequence(ctx: AppContext, pipelineStart?: number): PipelineStep[];
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Build the step sequence based on resolved config.
|
|
3
|
+
*
|
|
4
|
+
* This is the declarative description of the pipeline — a list of
|
|
5
|
+
* PipelineStep objects determined by config flags like skipFetch,
|
|
6
|
+
* skipEval, compareEnabled, etc.
|
|
7
|
+
*/
|
|
8
|
+
import { FULL_MODE_SUBMODES } from "../_vendor/ailf-shared/index.js";
|
|
9
|
+
import { CallbackStep } from "./steps/callback-step.js";
|
|
10
|
+
import { CalculateScoresStep } from "./steps/calculate-scores-step.js";
|
|
11
|
+
import { CompareStep } from "./steps/compare-step.js";
|
|
12
|
+
import { DiscoveryReportStep } from "./steps/discovery-report-step.js";
|
|
13
|
+
import { FetchDocsStep } from "./steps/fetch-docs-step.js";
|
|
14
|
+
import { GapAnalysisStep } from "./steps/gap-analysis-step.js";
|
|
15
|
+
import { GenerateConfigsStep } from "./steps/generate-configs-step.js";
|
|
16
|
+
import { GraderConsistencyStep } from "./steps/grader-consistency-step.js";
|
|
17
|
+
import { PublishReportStep } from "./steps/publish-report-step.js";
|
|
18
|
+
import { ReadinessStep } from "./steps/readiness-step.js";
|
|
19
|
+
import { ReportStep } from "./steps/report-step.js";
|
|
20
|
+
import { RunEvalStep } from "./steps/run-eval-step.js";
|
|
21
|
+
import { MirrorRepoTasksStep } from "./steps/mirror-repo-tasks-step.js";
|
|
22
|
+
import { ValidateStep } from "./steps/validate-step.js";
|
|
23
|
+
/**
|
|
24
|
+
* Build the pipeline step sequence based on config.
|
|
25
|
+
*
|
|
26
|
+
* @param ctx - The application context (used for config)
|
|
27
|
+
* @param pipelineStart - Pipeline start timestamp (for publish step)
|
|
28
|
+
*/
|
|
29
|
+
export function buildStepSequence(ctx, pipelineStart = Date.now()) {
|
|
30
|
+
const config = ctx.config;
|
|
31
|
+
const steps = [];
|
|
32
|
+
// Step 0: Validate configuration
|
|
33
|
+
steps.push(new ValidateStep());
|
|
34
|
+
// Step 0b: Mirror repo tasks to Content Lake (optional, when --repo-tasks-path)
|
|
35
|
+
if (config.repoTasksPath) {
|
|
36
|
+
steps.push(new MirrorRepoTasksStep());
|
|
37
|
+
}
|
|
38
|
+
// Step 1: Fetch documentation (step handles --skip-fetch internally)
|
|
39
|
+
steps.push(new FetchDocsStep());
|
|
40
|
+
// Step 2: Generate Promptfoo configs
|
|
41
|
+
steps.push(new GenerateConfigsStep());
|
|
42
|
+
// Step 3: Run evaluation (steps handle --skip-eval internally)
|
|
43
|
+
const modes = config.mode === "full"
|
|
44
|
+
? [...FULL_MODE_SUBMODES]
|
|
45
|
+
: [config.mode];
|
|
46
|
+
for (const mode of modes) {
|
|
47
|
+
steps.push(new RunEvalStep(mode));
|
|
48
|
+
}
|
|
49
|
+
// Step 3c: Grader consistency (optional, conditional)
|
|
50
|
+
if (config.graderReplications) {
|
|
51
|
+
steps.push(new GraderConsistencyStep());
|
|
52
|
+
}
|
|
53
|
+
// Step 4: Calculate scores
|
|
54
|
+
steps.push(new CalculateScoresStep());
|
|
55
|
+
// Step 4b: Gap analysis (on by default, opt-out via --no-gap-analysis)
|
|
56
|
+
if (config.gapAnalysisEnabled) {
|
|
57
|
+
steps.push(new GapAnalysisStep());
|
|
58
|
+
}
|
|
59
|
+
// Step 4b: Publish report (optional, when token is configured)
|
|
60
|
+
if (config.publishEnabled) {
|
|
61
|
+
steps.push(new PublishReportStep(pipelineStart, {
|
|
62
|
+
publishTag: config.publishTag,
|
|
63
|
+
}));
|
|
64
|
+
}
|
|
65
|
+
// Step 5: Generate report
|
|
66
|
+
steps.push(new ReportStep());
|
|
67
|
+
// Step 5b: Compare against baseline (optional)
|
|
68
|
+
if (config.compareEnabled) {
|
|
69
|
+
steps.push(new CompareStep());
|
|
70
|
+
}
|
|
71
|
+
// Step 6b: Readiness report (optional)
|
|
72
|
+
if (config.readinessEnabled) {
|
|
73
|
+
steps.push(new ReadinessStep());
|
|
74
|
+
}
|
|
75
|
+
// Step 6c: Discovery report (optional)
|
|
76
|
+
if (config.discoveryReportEnabled) {
|
|
77
|
+
steps.push(new DiscoveryReportStep());
|
|
78
|
+
}
|
|
79
|
+
// Step 7: Callback delivery (optional, API-triggered evaluations)
|
|
80
|
+
if (config.callback?.url) {
|
|
81
|
+
steps.push(new CallbackStep(config.callback, config.jobId));
|
|
82
|
+
}
|
|
83
|
+
return steps;
|
|
84
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Extract source-relevant overrides from ResolvedConfig.
|
|
3
|
+
*
|
|
4
|
+
* Used by orchestration steps and command handlers to call loadSource()
|
|
5
|
+
* with typed overrides instead of relying on process.env.
|
|
6
|
+
*/
|
|
7
|
+
import type { ResolvedConfig } from "../_vendor/ailf-core/index.d.ts";
|
|
8
|
+
import type { SourceOverrides } from "../sources.js";
|
|
9
|
+
export declare function configToSourceOverrides(config: ResolvedConfig): SourceOverrides;
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Extract source-relevant overrides from ResolvedConfig.
|
|
3
|
+
*
|
|
4
|
+
* Used by orchestration steps and command handlers to call loadSource()
|
|
5
|
+
* with typed overrides instead of relying on process.env.
|
|
6
|
+
*/
|
|
7
|
+
export function configToSourceOverrides(config) {
|
|
8
|
+
return {
|
|
9
|
+
...(config.allowedOrigins?.length
|
|
10
|
+
? { allowedOrigins: config.allowedOrigins }
|
|
11
|
+
: {}),
|
|
12
|
+
...(config.urls?.[0] ? { baseUrl: config.urls[0] } : {}),
|
|
13
|
+
...(config.datasetOverride ? { dataset: config.datasetOverride } : {}),
|
|
14
|
+
...(config.sanityDocumentArgs?.length
|
|
15
|
+
? { documentIds: config.sanityDocumentArgs }
|
|
16
|
+
: {}),
|
|
17
|
+
...(config.headers ? { headers: config.headers } : {}),
|
|
18
|
+
...(config.perspectiveOverride
|
|
19
|
+
? { perspective: config.perspectiveOverride }
|
|
20
|
+
: {}),
|
|
21
|
+
...(config.projectIdOverride
|
|
22
|
+
? { projectId: config.projectIdOverride }
|
|
23
|
+
: {}),
|
|
24
|
+
...(config.studioOriginOverride
|
|
25
|
+
? { studioOrigin: config.studioOriginOverride }
|
|
26
|
+
: {}),
|
|
27
|
+
};
|
|
28
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Environment variable bridge — writes ResolvedConfig values to process.env
|
|
3
|
+
* so that lib/*.ts modules (which still read process.env) work correctly.
|
|
4
|
+
*
|
|
5
|
+
* This replaces the former global applyEnvironment() with an explicit
|
|
6
|
+
* per-step bridge. Each orchestration step calls this before invoking
|
|
7
|
+
* its lib/*.ts main() function.
|
|
8
|
+
*
|
|
9
|
+
* Phase 9 will eliminate this file entirely by giving lib/*.ts main()
|
|
10
|
+
* functions typed option parameters.
|
|
11
|
+
*
|
|
12
|
+
* @see docs/exec-plans/active/ports-and-adapters/phase-8-delete-legacy-step-layer.md
|
|
13
|
+
*/
|
|
14
|
+
import type { ResolvedConfig } from "../_vendor/ailf-core/index.d.ts";
|
|
15
|
+
/**
|
|
16
|
+
* Bridge ResolvedConfig values to process.env.
|
|
17
|
+
*
|
|
18
|
+
* Idempotent — safe to call multiple times. Only sets env vars for
|
|
19
|
+
* config values that are defined (never deletes or resets).
|
|
20
|
+
*/
|
|
21
|
+
export declare function bridgeConfigToEnv(config: ResolvedConfig): void;
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Environment variable bridge — writes ResolvedConfig values to process.env
|
|
3
|
+
* so that lib/*.ts modules (which still read process.env) work correctly.
|
|
4
|
+
*
|
|
5
|
+
* This replaces the former global applyEnvironment() with an explicit
|
|
6
|
+
* per-step bridge. Each orchestration step calls this before invoking
|
|
7
|
+
* its lib/*.ts main() function.
|
|
8
|
+
*
|
|
9
|
+
* Phase 9 will eliminate this file entirely by giving lib/*.ts main()
|
|
10
|
+
* functions typed option parameters.
|
|
11
|
+
*
|
|
12
|
+
* @see docs/exec-plans/active/ports-and-adapters/phase-8-delete-legacy-step-layer.md
|
|
13
|
+
*/
|
|
14
|
+
/**
|
|
15
|
+
* Bridge ResolvedConfig values to process.env.
|
|
16
|
+
*
|
|
17
|
+
* Idempotent — safe to call multiple times. Only sets env vars for
|
|
18
|
+
* config values that are defined (never deletes or resets).
|
|
19
|
+
*/
|
|
20
|
+
export function bridgeConfigToEnv(config) {
|
|
21
|
+
// Mode
|
|
22
|
+
process.env.EVAL_MODE = config.mode;
|
|
23
|
+
// Search mode
|
|
24
|
+
if (config.searchMode !== "open") {
|
|
25
|
+
process.env.EVAL_SEARCH_MODE = config.searchMode;
|
|
26
|
+
}
|
|
27
|
+
// Source
|
|
28
|
+
if (config.source) {
|
|
29
|
+
process.env.DOC_SOURCE = config.source;
|
|
30
|
+
}
|
|
31
|
+
// URL-derived overrides
|
|
32
|
+
if (config.urls?.[0]) {
|
|
33
|
+
process.env.DOC_BASE_URL = config.urls[0];
|
|
34
|
+
}
|
|
35
|
+
// Sanity overrides
|
|
36
|
+
if (config.datasetOverride) {
|
|
37
|
+
process.env.SANITY_DATASET = config.datasetOverride;
|
|
38
|
+
}
|
|
39
|
+
if (config.projectIdOverride) {
|
|
40
|
+
process.env.SANITY_PROJECT_ID = config.projectIdOverride;
|
|
41
|
+
}
|
|
42
|
+
if (config.perspectiveOverride) {
|
|
43
|
+
process.env.SANITY_PERSPECTIVE = config.perspectiveOverride;
|
|
44
|
+
}
|
|
45
|
+
if (config.studioOriginOverride) {
|
|
46
|
+
process.env.SANITY_STUDIO_ORIGIN = config.studioOriginOverride;
|
|
47
|
+
}
|
|
48
|
+
if (config.sanityDocumentArgs?.length) {
|
|
49
|
+
process.env.SANITY_DOCUMENT_IDS = config.sanityDocumentArgs.join(",");
|
|
50
|
+
}
|
|
51
|
+
// Custom headers
|
|
52
|
+
if (config.headers) {
|
|
53
|
+
process.env.DOC_HEADERS = JSON.stringify(config.headers);
|
|
54
|
+
}
|
|
55
|
+
// Allowed origins
|
|
56
|
+
if (config.allowedOrigins?.length) {
|
|
57
|
+
process.env.DOC_ALLOWED_ORIGINS = config.allowedOrigins.join(",");
|
|
58
|
+
}
|
|
59
|
+
// Scoping filters
|
|
60
|
+
if (config.areas) {
|
|
61
|
+
process.env.EVAL_FILTER_AREAS = config.areas.join(",");
|
|
62
|
+
}
|
|
63
|
+
if (config.tasks) {
|
|
64
|
+
process.env.EVAL_FILTER_TASKS = config.tasks.join(",");
|
|
65
|
+
}
|
|
66
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Orchestration barrel export.
|
|
3
|
+
*
|
|
4
|
+
* The new pipeline execution model: composable steps, uniform middleware,
|
|
5
|
+
* and declarative sequencing.
|
|
6
|
+
*/
|
|
7
|
+
export { buildAppContext, mapToResolvedConfig } from "./build-app-context.js";
|
|
8
|
+
export { buildStepSequence } from "./build-step-sequence.js";
|
|
9
|
+
export { orchestratePipeline } from "./pipeline-orchestrator.js";
|
|
10
|
+
export { runStep } from "./step-runner.js";
|
|
11
|
+
export * from "./steps/index.js";
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Orchestration barrel export.
|
|
3
|
+
*
|
|
4
|
+
* The new pipeline execution model: composable steps, uniform middleware,
|
|
5
|
+
* and declarative sequencing.
|
|
6
|
+
*/
|
|
7
|
+
export { buildAppContext, mapToResolvedConfig } from "./build-app-context.js";
|
|
8
|
+
export { buildStepSequence } from "./build-step-sequence.js";
|
|
9
|
+
export { orchestratePipeline } from "./pipeline-orchestrator.js";
|
|
10
|
+
export { runStep } from "./step-runner.js";
|
|
11
|
+
export * from "./steps/index.js";
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pipeline orchestrator — sequences steps and short-circuits on failure.
|
|
3
|
+
*
|
|
4
|
+
* This replaces the 1,672-line executePipeline() in pipeline-action.ts
|
|
5
|
+
* with declarative step sequencing. Each step is run through the
|
|
6
|
+
* StepRunner which provides uniform timing, precondition checking,
|
|
7
|
+
* error handling, and logging.
|
|
8
|
+
*
|
|
9
|
+
* When a jobId is present in the config (API-triggered evaluations),
|
|
10
|
+
* the orchestrator emits progress updates to the Content Lake after
|
|
11
|
+
* each step completes. This enables the GET /v1/jobs/:jobId polling
|
|
12
|
+
* endpoint to show real-time progress.
|
|
13
|
+
*/
|
|
14
|
+
import type { AppContext, PipelineResult, PipelineStep } from "../_vendor/ailf-core/index.d.ts";
|
|
15
|
+
/**
|
|
16
|
+
* Run a sequence of pipeline steps, short-circuiting on required step failure.
|
|
17
|
+
*
|
|
18
|
+
* Optional steps (step.optional === true) may fail without stopping
|
|
19
|
+
* the pipeline. Required step failures cause an immediate abort.
|
|
20
|
+
*
|
|
21
|
+
* When ctx.config.jobId is set, emits progress updates to the Content
|
|
22
|
+
* Lake after each step completes.
|
|
23
|
+
*/
|
|
24
|
+
export declare function orchestratePipeline(ctx: AppContext, steps: PipelineStep[]): Promise<PipelineResult>;
|