@sanity/ailf 2.0.0 โ 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/canonical/grader-references/README.md +2 -2
- package/canonical/reference-solutions/content-lake/mutations.ts +160 -0
- package/canonical/reference-solutions/content-lake/realtime.ts +187 -0
- package/canonical/reference-solutions/image-handling/asset-pipeline.tsx +166 -0
- package/canonical/reference-solutions/portable-text/custom-blocks.ts +204 -0
- package/canonical/reference-solutions/portable-text/rendering.tsx +163 -0
- package/config/features.ts +1 -1
- package/config/models.ts +28 -23
- package/config/sources.ts +1 -1
- package/config/thresholds.ts +1 -1
- package/dist/_vendor/ailf-core/__tests__/comparison-formatters.test.d.ts +10 -0
- package/dist/_vendor/ailf-core/__tests__/comparison-formatters.test.js +185 -0
- package/dist/_vendor/ailf-core/artifact-capture/__tests__/noop-collector.test.d.ts +6 -0
- package/dist/_vendor/ailf-core/artifact-capture/__tests__/noop-collector.test.js +42 -0
- package/dist/_vendor/ailf-core/artifact-capture/noop-collector.d.ts +14 -0
- package/dist/_vendor/ailf-core/artifact-capture/noop-collector.js +25 -0
- package/dist/_vendor/ailf-core/config-helpers.d.ts +6 -0
- package/dist/_vendor/ailf-core/config-helpers.js +29 -0
- package/dist/_vendor/ailf-core/examples/index.d.ts +164 -94
- package/dist/_vendor/ailf-core/examples/index.js +208 -114
- package/dist/_vendor/ailf-core/index.d.ts +1 -0
- package/dist/_vendor/ailf-core/index.js +1 -0
- package/dist/_vendor/ailf-core/ports/artifact-collector.d.ts +94 -0
- package/dist/_vendor/ailf-core/ports/artifact-collector.js +13 -0
- package/dist/_vendor/ailf-core/ports/capture-comparator.d.ts +138 -0
- package/dist/_vendor/ailf-core/ports/capture-comparator.js +10 -0
- package/dist/_vendor/ailf-core/ports/context.d.ts +20 -1
- package/dist/_vendor/ailf-core/ports/eval-runner.d.ts +6 -0
- package/dist/_vendor/ailf-core/ports/index.d.ts +2 -0
- package/dist/_vendor/ailf-core/ports/pipeline-step.d.ts +11 -0
- package/dist/_vendor/ailf-core/ports/task-source.d.ts +3 -3
- package/dist/_vendor/ailf-core/ports/task-source.js +3 -3
- package/dist/_vendor/ailf-core/schemas/pipeline-request.d.ts +6 -1
- package/dist/_vendor/ailf-core/schemas/pipeline-request.js +14 -2
- package/dist/_vendor/ailf-core/services/config-helpers.d.ts +16 -1
- package/dist/_vendor/ailf-core/services/config-helpers.js +21 -0
- package/dist/_vendor/ailf-core/services/index.d.ts +1 -1
- package/dist/_vendor/ailf-core/services/index.js +1 -1
- package/dist/_vendor/ailf-core/services/scoring.js +9 -0
- package/dist/_vendor/ailf-core/types/generalized-task.d.ts +12 -1
- package/dist/_vendor/ailf-core/types/generalized-task.js +1 -1
- package/dist/_vendor/ailf-core/types/index.d.ts +47 -4
- package/dist/_vendor/ailf-core/types/plugin-registry.d.ts +27 -0
- package/dist/_vendor/ailf-shared/eval-modes.d.ts +15 -0
- package/dist/_vendor/ailf-shared/eval-modes.js +18 -0
- package/dist/_vendor/ailf-tasks/cli.d.ts +8 -0
- package/dist/_vendor/ailf-tasks/cli.js +61 -0
- package/dist/_vendor/ailf-tasks/index.d.ts +13 -0
- package/dist/_vendor/ailf-tasks/index.js +16 -0
- package/dist/_vendor/ailf-tasks/parser.d.ts +27 -0
- package/dist/_vendor/ailf-tasks/parser.js +73 -0
- package/dist/_vendor/ailf-tasks/schemas.d.ts +198 -0
- package/dist/_vendor/ailf-tasks/schemas.js +180 -0
- package/dist/_vendor/ailf-tasks/validation.d.ts +47 -0
- package/dist/_vendor/ailf-tasks/validation.js +162 -0
- package/dist/adapters/api-client/remediation.js +2 -2
- package/dist/adapters/config-sources/file-config-adapter.js +6 -1
- package/dist/adapters/eval-runners/promptfoo-eval-adapter.js +8 -2
- package/dist/adapters/index.d.ts +0 -1
- package/dist/adapters/index.js +0 -1
- package/dist/adapters/task-sources/composite-task-source.d.ts +1 -1
- package/dist/adapters/task-sources/composite-task-source.js +1 -1
- package/dist/adapters/task-sources/content-lake-task-source.d.ts +4 -6
- package/dist/adapters/task-sources/content-lake-task-source.js +4 -6
- package/dist/adapters/task-sources/index.d.ts +1 -2
- package/dist/adapters/task-sources/index.js +1 -2
- package/dist/adapters/task-sources/repo-schemas.d.ts +1 -1
- package/dist/adapters/task-sources/repo-schemas.js +2 -2
- package/dist/adapters/task-sources/repo-task-source.js +1 -1
- package/dist/adapters/task-sources/repo-trigger.d.ts +1 -1
- package/dist/adapters/task-sources/repo-trigger.js +1 -1
- package/dist/adapters/task-sources/task-file-loader.d.ts +9 -6
- package/dist/adapters/task-sources/task-file-loader.js +20 -6
- package/dist/agent-observer/test-imports.d.ts +7 -0
- package/dist/agent-observer/test-imports.js +185 -0
- package/dist/artifact-capture/comparator.d.ts +22 -0
- package/dist/artifact-capture/comparator.js +493 -0
- package/dist/artifact-capture/filesystem-collector.d.ts +42 -0
- package/dist/artifact-capture/filesystem-collector.js +237 -0
- package/dist/artifact-capture/redact-artifact.d.ts +20 -0
- package/dist/artifact-capture/redact-artifact.js +115 -0
- package/dist/assertions/source-isolation.d.ts +1 -1
- package/dist/assertions/source-isolation.js +1 -1
- package/dist/cli.js +4 -0
- package/dist/commands/calculate-scores.js +1 -0
- package/dist/commands/capture-compare.d.ts +15 -0
- package/dist/commands/capture-compare.js +253 -0
- package/dist/commands/capture-list.d.ts +12 -0
- package/dist/commands/capture-list.js +147 -0
- package/dist/commands/capture.d.ts +9 -0
- package/dist/commands/capture.js +16 -0
- package/dist/commands/chronic-failures.d.ts +8 -0
- package/dist/commands/chronic-failures.js +33 -0
- package/dist/commands/explain-handler.d.ts +1 -1
- package/dist/commands/explain-handler.js +37 -8
- package/dist/commands/fetch-docs.js +1 -0
- package/dist/commands/generate-configs.d.ts +3 -3
- package/dist/commands/generate-configs.js +20 -8
- package/dist/commands/init.d.ts +2 -3
- package/dist/commands/init.js +56 -170
- package/dist/commands/pipeline-action.d.ts +7 -1
- package/dist/commands/pipeline-action.js +43 -19
- package/dist/commands/pipeline.d.ts +6 -1
- package/dist/commands/pipeline.js +7 -2
- package/dist/commands/pr-comment.js +1 -0
- package/dist/commands/publish.js +1 -0
- package/dist/commands/shared/help.js +2 -2
- package/dist/commands/update-quality-scores.d.ts +5 -0
- package/dist/commands/update-quality-scores.js +20 -0
- package/dist/composition-root.d.ts +2 -3
- package/dist/composition-root.js +27 -14
- package/dist/config/features.ts +23 -0
- package/dist/config/models.ts +100 -0
- package/dist/config/prompts.ts +16 -0
- package/dist/config/rubrics.ts +225 -0
- package/dist/config/schedules.ts +47 -0
- package/dist/config/sinks.ts +37 -0
- package/dist/config/sources.ts +21 -0
- package/dist/config/thresholds.ts +61 -0
- package/dist/lib/agent-behavior-report.d.ts +8 -0
- package/dist/lib/agent-behavior-report.js +185 -0
- package/dist/lib/baseline.d.ts +19 -0
- package/dist/lib/baseline.js +153 -0
- package/dist/lib/calculate-scores.d.ts +23 -0
- package/dist/lib/calculate-scores.js +42 -0
- package/dist/lib/compare.d.ts +18 -0
- package/dist/lib/compare.js +170 -0
- package/dist/lib/coverage-audit.d.ts +4 -0
- package/dist/lib/coverage-audit.js +42 -0
- package/dist/lib/discovery-report.d.ts +13 -0
- package/dist/lib/discovery-report.js +57 -0
- package/dist/lib/fetch-docs.d.ts +30 -0
- package/dist/lib/fetch-docs.js +171 -0
- package/dist/lib/generate-configs.d.ts +25 -0
- package/dist/lib/generate-configs.js +42 -0
- package/dist/lib/grader-api.d.ts +21 -0
- package/dist/lib/grader-api.js +34 -0
- package/dist/lib/grader-compare.d.ts +19 -0
- package/dist/lib/grader-compare.js +91 -0
- package/dist/lib/grader-consistency.d.ts +27 -0
- package/dist/lib/grader-consistency.js +79 -0
- package/dist/lib/grader-sensitivity.d.ts +19 -0
- package/dist/lib/grader-sensitivity.js +75 -0
- package/dist/lib/grader-validate.d.ts +19 -0
- package/dist/lib/grader-validate.js +78 -0
- package/dist/lib/measure-retrieval.d.ts +14 -0
- package/dist/lib/measure-retrieval.js +71 -0
- package/dist/lib/pr-comment.d.ts +16 -0
- package/dist/lib/pr-comment.js +28 -0
- package/dist/lib/readiness-report.d.ts +13 -0
- package/dist/lib/readiness-report.js +108 -0
- package/dist/lib/webhook-server.d.ts +11 -0
- package/dist/lib/webhook-server.js +24 -0
- package/dist/lib/weekly-digest.d.ts +24 -0
- package/dist/lib/weekly-digest.js +148 -0
- package/dist/orchestration/build-app-context.js +13 -0
- package/dist/orchestration/cache-context.d.ts +23 -0
- package/dist/orchestration/cache-context.js +43 -0
- package/dist/orchestration/env-bridge.d.ts +21 -0
- package/dist/orchestration/env-bridge.js +66 -0
- package/dist/orchestration/load-pipeline-tasks.d.ts +34 -0
- package/dist/orchestration/load-pipeline-tasks.js +52 -0
- package/dist/orchestration/pipeline-orchestrator.js +75 -5
- package/dist/orchestration/step-runner.js +5 -1
- package/dist/orchestration/steps/calculate-scores-step.d.ts +1 -0
- package/dist/orchestration/steps/calculate-scores-step.js +13 -0
- package/dist/orchestration/steps/callback-step.js +10 -1
- package/dist/orchestration/steps/compare-step.js +6 -3
- package/dist/orchestration/steps/discovery-report-step.js +6 -2
- package/dist/orchestration/steps/fetch-docs-shell.d.ts +17 -0
- package/dist/orchestration/steps/fetch-docs-shell.js +30 -0
- package/dist/orchestration/steps/fetch-docs-step.d.ts +1 -0
- package/dist/orchestration/steps/fetch-docs-step.js +30 -16
- package/dist/orchestration/steps/gap-analysis-step.js +13 -2
- package/dist/orchestration/steps/generate-configs-step.d.ts +1 -0
- package/dist/orchestration/steps/generate-configs-step.js +50 -15
- package/dist/orchestration/steps/mirror-repo-tasks-step.d.ts +1 -1
- package/dist/orchestration/steps/mirror-repo-tasks-step.js +1 -1
- package/dist/orchestration/steps/publish-report-step.js +19 -0
- package/dist/orchestration/steps/readiness-step.js +8 -3
- package/dist/orchestration/steps/report-step.js +17 -4
- package/dist/orchestration/steps/run-eval-step.d.ts +1 -0
- package/dist/orchestration/steps/run-eval-step.js +51 -31
- package/dist/pipeline/agent-behavior-report.js +6 -0
- package/dist/pipeline/attribution.d.ts +1 -1
- package/dist/pipeline/attribution.js +1 -1
- package/dist/pipeline/cache.js +29 -15
- package/dist/pipeline/calculate-scores.d.ts +2 -0
- package/dist/pipeline/calculate-scores.js +70 -33
- package/dist/pipeline/chronic-failures.d.ts +55 -0
- package/dist/pipeline/chronic-failures.js +110 -0
- package/dist/pipeline/compiler/__tests__/mcp-server-handler.test.js +33 -0
- package/dist/pipeline/compiler/__tests__/promptfoo-compiler.test.js +2 -3
- package/dist/pipeline/compiler/__tests__/task-bridge.test.d.ts +9 -0
- package/dist/pipeline/compiler/__tests__/task-bridge.test.js +339 -0
- package/dist/pipeline/compiler/__tests__/tool-loop-openai.test.d.ts +10 -0
- package/dist/pipeline/compiler/__tests__/tool-loop-openai.test.js +509 -0
- package/dist/pipeline/compiler/assertion-mapper.d.ts +1 -1
- package/dist/pipeline/compiler/assertion-mapper.js +1 -1
- package/dist/pipeline/compiler/compiler-to-yaml.d.ts +2 -7
- package/dist/pipeline/compiler/compiler-to-yaml.js +2 -7
- package/dist/pipeline/compiler/config-loader.d.ts +14 -0
- package/dist/pipeline/compiler/config-loader.js +42 -2
- package/dist/pipeline/compiler/fixture-resolver.d.ts +1 -1
- package/dist/pipeline/compiler/fixture-resolver.js +1 -1
- package/dist/pipeline/compiler/ignore-fields.d.ts +1 -1
- package/dist/pipeline/compiler/ignore-fields.js +1 -1
- package/dist/pipeline/compiler/index.d.ts +2 -5
- package/dist/pipeline/compiler/index.js +2 -5
- package/dist/pipeline/compiler/literacy-bridge.d.ts +1 -1
- package/dist/pipeline/compiler/literacy-bridge.js +1 -1
- package/dist/pipeline/compiler/mode-bases/agent-harness.d.ts +1 -1
- package/dist/pipeline/compiler/mode-bases/agent-harness.js +1 -1
- package/dist/pipeline/compiler/mode-bases/knowledge-probe.d.ts +1 -1
- package/dist/pipeline/compiler/mode-bases/knowledge-probe.js +1 -1
- package/dist/pipeline/compiler/mode-bases/literacy.d.ts +13 -2
- package/dist/pipeline/compiler/mode-bases/literacy.js +55 -1
- package/dist/pipeline/compiler/mode-bases/mcp-server.d.ts +1 -1
- package/dist/pipeline/compiler/mode-bases/mcp-server.js +1 -1
- package/dist/pipeline/compiler/mode-handlers/agent-harness/index.d.ts +1 -1
- package/dist/pipeline/compiler/mode-handlers/agent-harness/index.js +1 -1
- package/dist/pipeline/compiler/mode-handlers/agent-harness-handler.d.ts +70 -0
- package/dist/pipeline/compiler/mode-handlers/agent-harness-handler.js +485 -0
- package/dist/pipeline/compiler/mode-handlers/index.d.ts +2 -2
- package/dist/pipeline/compiler/mode-handlers/index.js +2 -2
- package/dist/pipeline/compiler/mode-handlers/knowledge-probe/index.d.ts +1 -1
- package/dist/pipeline/compiler/mode-handlers/knowledge-probe/index.js +1 -1
- package/dist/pipeline/compiler/mode-handlers/knowledge-probe-handler.d.ts +76 -0
- package/dist/pipeline/compiler/mode-handlers/knowledge-probe-handler.js +245 -0
- package/dist/pipeline/compiler/mode-handlers/literacy/index.d.ts +1 -1
- package/dist/pipeline/compiler/mode-handlers/literacy/index.js +1 -1
- package/dist/pipeline/compiler/mode-handlers/literacy-handler.d.ts +89 -0
- package/dist/pipeline/compiler/mode-handlers/literacy-handler.js +379 -0
- package/dist/pipeline/compiler/mode-handlers/mcp-assertions.d.ts +50 -0
- package/dist/pipeline/compiler/mode-handlers/mcp-assertions.js +334 -0
- package/dist/pipeline/compiler/mode-handlers/mcp-server/assertions.d.ts +1 -1
- package/dist/pipeline/compiler/mode-handlers/mcp-server/assertions.js +1 -1
- package/dist/pipeline/compiler/mode-handlers/mcp-server/index.d.ts +1 -1
- package/dist/pipeline/compiler/mode-handlers/mcp-server/index.js +1 -1
- package/dist/pipeline/compiler/mode-handlers/mcp-server/provider-config.js +4 -0
- package/dist/pipeline/compiler/mode-handlers/mcp-server-handler.d.ts +69 -0
- package/dist/pipeline/compiler/mode-handlers/mcp-server-handler.js +307 -0
- package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/index.js +22 -5
- package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/mcp-connection.js +6 -0
- package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/tool-loop-openai.d.ts +10 -5
- package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/tool-loop-openai.js +314 -7
- package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/types.d.ts +10 -0
- package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider.d.ts +65 -0
- package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider.js +368 -0
- package/dist/pipeline/compiler/presets/sanity-literacy.d.ts +1 -1
- package/dist/pipeline/compiler/presets/sanity-literacy.js +1 -1
- package/dist/pipeline/compiler/promptfoo-compiler.d.ts +1 -4
- package/dist/pipeline/compiler/promptfoo-compiler.js +3 -12
- package/dist/pipeline/compiler/provider-assembler.js +13 -7
- package/dist/pipeline/compiler/sandbox/docker-sandbox.d.ts +1 -1
- package/dist/pipeline/compiler/sandbox/docker-sandbox.js +1 -1
- package/dist/pipeline/compiler/sandbox/fixture-provisioner.d.ts +1 -1
- package/dist/pipeline/compiler/sandbox/fixture-provisioner.js +1 -1
- package/dist/pipeline/compiler/sandbox/git-worktree-sandbox.d.ts +1 -1
- package/dist/pipeline/compiler/sandbox/git-worktree-sandbox.js +1 -1
- package/dist/pipeline/compiler/sandbox/index.d.ts +1 -1
- package/dist/pipeline/compiler/sandbox/index.js +1 -1
- package/dist/pipeline/compiler/sandbox/sandbox-selector.d.ts +1 -1
- package/dist/pipeline/compiler/sandbox/sandbox-selector.js +1 -1
- package/dist/pipeline/compiler/sandbox/sandbox-strategy.d.ts +1 -1
- package/dist/pipeline/compiler/sandbox/sandbox-strategy.js +1 -1
- package/dist/pipeline/compiler/sandbox/tempdir-sandbox.d.ts +1 -1
- package/dist/pipeline/compiler/sandbox/tempdir-sandbox.js +1 -1
- package/dist/pipeline/compiler/scoring-bridge.d.ts +1 -1
- package/dist/pipeline/compiler/scoring-bridge.js +1 -1
- package/dist/pipeline/compiler/task-bridge.d.ts +41 -0
- package/dist/pipeline/compiler/task-bridge.js +92 -0
- package/dist/pipeline/compiler/task-graph-builder.d.ts +1 -4
- package/dist/pipeline/compiler/task-graph-builder.js +1 -4
- package/dist/pipeline/compiler/telemetry/index.d.ts +1 -1
- package/dist/pipeline/compiler/telemetry/index.js +1 -1
- package/dist/pipeline/compiler/variable-resolver.d.ts +1 -1
- package/dist/pipeline/compiler/variable-resolver.js +1 -1
- package/dist/pipeline/coverage-audit.d.ts +1 -1
- package/dist/pipeline/coverage-audit.js +1 -1
- package/dist/pipeline/degradations.d.ts +1 -1
- package/dist/pipeline/degradations.js +1 -1
- package/dist/pipeline/failure-modes.d.ts +1 -1
- package/dist/pipeline/failure-modes.js +13 -1
- package/dist/pipeline/gap-analysis.d.ts +1 -1
- package/dist/pipeline/gap-analysis.js +3 -1
- package/dist/pipeline/generate-configs.d.ts +2 -2
- package/dist/pipeline/generate-configs.js +15 -8
- package/dist/pipeline/grader-compare-runner.d.ts +1 -1
- package/dist/pipeline/grader-compare-runner.js +7 -1
- package/dist/pipeline/grader-comparison.d.ts +1 -1
- package/dist/pipeline/grader-comparison.js +1 -1
- package/dist/pipeline/grader-consistency-runner.d.ts +1 -1
- package/dist/pipeline/grader-consistency-runner.js +7 -1
- package/dist/pipeline/grader-consistency.d.ts +1 -1
- package/dist/pipeline/grader-consistency.js +1 -1
- package/dist/pipeline/grader-sensitivity-runner.d.ts +1 -1
- package/dist/pipeline/grader-sensitivity-runner.js +1 -1
- package/dist/pipeline/grader-sensitivity.d.ts +1 -1
- package/dist/pipeline/grader-sensitivity.js +1 -1
- package/dist/pipeline/grader-validate-runner.d.ts +1 -1
- package/dist/pipeline/grader-validate-runner.js +2 -2
- package/dist/pipeline/grader-validation.d.ts +1 -1
- package/dist/pipeline/grader-validation.js +1 -1
- package/dist/pipeline/map-request-to-config.js +15 -2
- package/dist/pipeline/mirror-repo-tasks.d.ts +1 -1
- package/dist/pipeline/mirror-repo-tasks.js +1 -1
- package/dist/pipeline/plan-format.d.ts +1 -1
- package/dist/pipeline/plan-format.js +1 -1
- package/dist/pipeline/plan.d.ts +1 -1
- package/dist/pipeline/plan.js +67 -29
- package/dist/pipeline/probe.d.ts +1 -1
- package/dist/pipeline/probe.js +1 -1
- package/dist/pipeline/readiness-report.d.ts +2 -2
- package/dist/pipeline/readiness-report.js +2 -2
- package/dist/pipeline/release-classification.d.ts +1 -1
- package/dist/pipeline/release-classification.js +1 -1
- package/dist/pipeline/release-report.d.ts +1 -1
- package/dist/pipeline/release-report.js +1 -1
- package/dist/pipeline/repo-eval-comment.d.ts +1 -1
- package/dist/pipeline/repo-eval-comment.js +1 -1
- package/dist/pipeline/repo-threshold-evaluator.d.ts +1 -1
- package/dist/pipeline/repo-threshold-evaluator.js +1 -1
- package/dist/pipeline/resolve-mappings.d.ts +6 -6
- package/dist/pipeline/resolve-mappings.js +44 -44
- package/dist/pipeline/retrieval-metrics.d.ts +3 -3
- package/dist/pipeline/retrieval-metrics.js +28 -20
- package/dist/pipeline/steps/calculate-scores-step.d.ts +11 -0
- package/dist/pipeline/steps/calculate-scores-step.js +89 -0
- package/dist/pipeline/steps/compare-step.d.ts +18 -0
- package/dist/pipeline/steps/compare-step.js +90 -0
- package/dist/pipeline/steps/eval-step.d.ts +53 -0
- package/dist/pipeline/steps/eval-step.js +347 -0
- package/dist/pipeline/steps/fetch-docs-step.d.ts +11 -0
- package/dist/pipeline/steps/fetch-docs-step.js +84 -0
- package/dist/pipeline/steps/generate-configs-step.d.ts +11 -0
- package/dist/pipeline/steps/generate-configs-step.js +98 -0
- package/dist/pipeline/steps/grader-consistency-step.d.ts +21 -0
- package/dist/pipeline/steps/grader-consistency-step.js +74 -0
- package/dist/pipeline/steps/publish-report-step.d.ts +57 -0
- package/dist/pipeline/steps/publish-report-step.js +243 -0
- package/dist/pipeline/steps/report-step.d.ts +13 -0
- package/dist/pipeline/steps/report-step.js +56 -0
- package/dist/pipeline/steps/update-scores-step.d.ts +11 -0
- package/dist/pipeline/steps/update-scores-step.js +42 -0
- package/dist/pipeline/targeted-loo.d.ts +1 -1
- package/dist/pipeline/targeted-loo.js +1 -1
- package/dist/pipeline/thresholds.d.ts +1 -1
- package/dist/pipeline/thresholds.js +1 -1
- package/dist/pipeline/validate.js +13 -0
- package/dist/report-store.d.ts +17 -0
- package/dist/report-store.js +24 -0
- package/dist/scripts/agent-behavior-report.d.ts +19 -0
- package/dist/scripts/agent-behavior-report.js +315 -0
- package/dist/scripts/baseline.d.ts +43 -0
- package/dist/scripts/baseline.js +267 -0
- package/dist/scripts/calculate-scores.d.ts +166 -0
- package/dist/scripts/calculate-scores.js +1296 -0
- package/dist/scripts/compare.d.ts +22 -0
- package/dist/scripts/compare.js +334 -0
- package/dist/scripts/coverage-audit.d.ts +44 -0
- package/dist/scripts/coverage-audit.js +209 -0
- package/dist/scripts/debug-eval.d.ts +19 -0
- package/dist/scripts/debug-eval.js +73 -0
- package/dist/scripts/discovery-report.d.ts +58 -0
- package/dist/scripts/discovery-report.js +250 -0
- package/dist/scripts/fetch-docs.d.ts +35 -0
- package/dist/scripts/fetch-docs.js +472 -0
- package/dist/scripts/generate-configs.d.ts +66 -0
- package/dist/scripts/generate-configs.js +459 -0
- package/dist/scripts/grader-api.d.ts +27 -0
- package/dist/scripts/grader-api.js +206 -0
- package/dist/scripts/grader-compare.d.ts +22 -0
- package/dist/scripts/grader-compare.js +368 -0
- package/dist/scripts/grader-consistency.d.ts +20 -0
- package/dist/scripts/grader-consistency.js +313 -0
- package/dist/scripts/grader-sensitivity.d.ts +22 -0
- package/dist/scripts/grader-sensitivity.js +354 -0
- package/dist/scripts/grader-validate.d.ts +19 -0
- package/dist/scripts/grader-validate.js +267 -0
- package/dist/scripts/measure-retrieval.d.ts +10 -0
- package/dist/scripts/measure-retrieval.js +145 -0
- package/dist/scripts/migrate-task-mode.d.ts +1 -1
- package/dist/scripts/migrate-task-mode.js +1 -1
- package/dist/scripts/migrate-tasks-to-content-lake.d.ts +1 -1
- package/dist/scripts/migrate-tasks-to-content-lake.js +1 -1
- package/dist/scripts/pipeline.d.ts +76 -0
- package/dist/scripts/pipeline.js +1031 -0
- package/dist/scripts/pr-comment.d.ts +10 -0
- package/dist/scripts/pr-comment.js +510 -0
- package/dist/scripts/readiness-report.d.ts +88 -0
- package/dist/scripts/readiness-report.js +342 -0
- package/dist/scripts/update-quality-scores.d.ts +15 -0
- package/dist/scripts/update-quality-scores.js +184 -0
- package/dist/scripts/validate-task-sources.d.ts +1 -1
- package/dist/scripts/validate-task-sources.js +1 -1
- package/dist/scripts/validate.d.ts +13 -0
- package/dist/scripts/validate.js +79 -0
- package/dist/scripts/webhook-server.d.ts +26 -0
- package/dist/scripts/webhook-server.js +147 -0
- package/dist/scripts/weekly-digest.d.ts +24 -0
- package/dist/scripts/weekly-digest.js +144 -0
- package/dist/sinks/format-slack.d.ts +64 -0
- package/dist/sinks/format-slack.js +306 -0
- package/dist/sinks/slack-sink.d.ts +27 -0
- package/dist/sinks/slack-sink.js +78 -0
- package/dist/sinks/types.d.ts +1 -1
- package/dist/sinks/types.js +1 -1
- package/dist/sinks/webhook-sink.d.ts +19 -0
- package/dist/sinks/webhook-sink.js +50 -0
- package/dist/tasks/knowledge-probe/define-type-api.task.ts +66 -0
- package/dist/tasks/knowledge-probe/groq-projections.task.ts +62 -0
- package/dist/tasks/literacy/content-lake.task.ts +181 -0
- package/dist/tasks/literacy/frameworks.task.ts +129 -0
- package/dist/tasks/literacy/functions.task.ts +70 -0
- package/dist/tasks/literacy/groq.task.ts +259 -0
- package/dist/tasks/literacy/image-handling.task.ts +95 -0
- package/dist/tasks/literacy/nextjs-live.task.ts +76 -0
- package/dist/tasks/literacy/portable-text.task.ts +169 -0
- package/dist/tasks/literacy/studio-setup.task.ts +134 -0
- package/dist/tasks/literacy/visual-editing.task.ts +147 -0
- package/package.json +24 -24
- package/tasks/.expanded.agentic.yaml +280 -0
- package/tasks/.expanded.yaml +565 -0
- package/tasks/knowledge-probe/define-type-api.task.ts +11 -0
- package/tasks/knowledge-probe/groq-projections.task.ts +3 -0
- package/tasks/literacy/content-lake.task.ts +181 -0
- package/tasks/literacy/frameworks.task.ts +1 -0
- package/tasks/literacy/functions.task.ts +1 -0
- package/tasks/literacy/groq.task.ts +1 -0
- package/tasks/literacy/image-handling.task.ts +95 -0
- package/tasks/literacy/nextjs-live.task.ts +2 -1
- package/tasks/literacy/portable-text.task.ts +169 -0
- package/tasks/literacy/studio-setup.task.ts +5 -2
- package/tasks/literacy/visual-editing.task.ts +1 -0
- package/LICENSE +0 -21
- package/tasks/frameworks.yaml +0 -98
- package/tasks/functions.yaml +0 -51
- package/tasks/groq.yaml +0 -216
- package/tasks/nextjs-live.yaml +0 -62
- package/tasks/studio-setup.yaml +0 -111
- package/tasks/visual-editing.yaml +0 -120
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* webhook-server.ts
|
|
3
|
+
*
|
|
4
|
+
* Local development server for testing the webhook handler.
|
|
5
|
+
*
|
|
6
|
+
* Starts an HTTP server that receives Sanity webhook payloads, processes
|
|
7
|
+
* them through the WebhookHandler, and logs results. Useful for local
|
|
8
|
+
* development and testing the full event-driven trigger flow.
|
|
9
|
+
*
|
|
10
|
+
* Usage:
|
|
11
|
+
* pnpm webhook-server # start on port 3333
|
|
12
|
+
* WEBHOOK_PORT=8080 pnpm webhook-server # custom port
|
|
13
|
+
*
|
|
14
|
+
* Test with curl:
|
|
15
|
+
* curl -X POST http://localhost:3333/webhook \
|
|
16
|
+
* -H "Content-Type: application/json" \
|
|
17
|
+
* -d '{"operation":"update","result":{"_id":"abc","_type":"article","slug":{"current":"groq-introduction"}}}'
|
|
18
|
+
*
|
|
19
|
+
* Endpoints:
|
|
20
|
+
* POST /webhook โ handle a Sanity webhook payload
|
|
21
|
+
* GET /health โ handler diagnostics (budget, pending, tracked slugs)
|
|
22
|
+
* GET /mappings โ list all tracked document slugs and their areas
|
|
23
|
+
*
|
|
24
|
+
* @see docs/design-docs/report-store/visibility-workflows.md
|
|
25
|
+
*/
|
|
26
|
+
import { createServer } from "http";
|
|
27
|
+
import { dirname, resolve } from "path";
|
|
28
|
+
import { fileURLToPath } from "url";
|
|
29
|
+
import { allTrackedSlugs, buildReverseMapping, } from "../pipeline/reverse-mapping.js";
|
|
30
|
+
import { WebhookHandler } from "../webhook/handler.js";
|
|
31
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
32
|
+
const ROOT = resolve(__dirname, "..", "..");
|
|
33
|
+
// ---------------------------------------------------------------------------
|
|
34
|
+
// Configuration
|
|
35
|
+
// ---------------------------------------------------------------------------
|
|
36
|
+
const PORT = parseInt(process.env.WEBHOOK_PORT ?? "3333", 10);
|
|
37
|
+
const GITHUB_TOKEN = process.env.GITHUB_TOKEN ?? "";
|
|
38
|
+
const DRY_RUN = !GITHUB_TOKEN;
|
|
39
|
+
// ---------------------------------------------------------------------------
|
|
40
|
+
// Handler
|
|
41
|
+
// ---------------------------------------------------------------------------
|
|
42
|
+
const handler = new WebhookHandler({
|
|
43
|
+
dailyBudget: parseInt(process.env.WEBHOOK_DAILY_BUDGET ?? "20", 10),
|
|
44
|
+
debounceMs: parseInt(process.env.WEBHOOK_DEBOUNCE_MS ?? "10000", 10), // 10s for local dev
|
|
45
|
+
githubToken: GITHUB_TOKEN,
|
|
46
|
+
rootDir: ROOT,
|
|
47
|
+
});
|
|
48
|
+
// ---------------------------------------------------------------------------
|
|
49
|
+
// HTTP Server
|
|
50
|
+
// ---------------------------------------------------------------------------
|
|
51
|
+
const server = createServer((req, res) => {
|
|
52
|
+
void handleRequest(req, res);
|
|
53
|
+
});
|
|
54
|
+
async function handleRequest(req, res) {
|
|
55
|
+
const url = new URL(req.url ?? "/", `http://localhost:${PORT}`);
|
|
56
|
+
// CORS headers for local dev
|
|
57
|
+
res.setHeader("Access-Control-Allow-Origin", "*");
|
|
58
|
+
res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
|
|
59
|
+
res.setHeader("Access-Control-Allow-Headers", "Content-Type");
|
|
60
|
+
if (req.method === "OPTIONS") {
|
|
61
|
+
res.writeHead(200);
|
|
62
|
+
res.end();
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
65
|
+
// Health check
|
|
66
|
+
if (url.pathname === "/health" && req.method === "GET") {
|
|
67
|
+
const diagnostics = handler.diagnostics();
|
|
68
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
69
|
+
res.end(JSON.stringify({ dryRun: DRY_RUN, ...diagnostics }, null, 2));
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
// Mappings
|
|
73
|
+
if (url.pathname === "/mappings" && req.method === "GET") {
|
|
74
|
+
const reverseMapping = buildReverseMapping(ROOT);
|
|
75
|
+
const slugs = allTrackedSlugs(reverseMapping);
|
|
76
|
+
const mappings = Object.fromEntries(slugs.map((slug) => [slug, reverseMapping.get(slug)]));
|
|
77
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
78
|
+
res.end(JSON.stringify({ mappings, slugCount: slugs.length }, null, 2));
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
// Webhook handler
|
|
82
|
+
if (url.pathname === "/webhook" && req.method === "POST") {
|
|
83
|
+
const body = await readBody(req);
|
|
84
|
+
let payload;
|
|
85
|
+
try {
|
|
86
|
+
payload = JSON.parse(body);
|
|
87
|
+
}
|
|
88
|
+
catch {
|
|
89
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
90
|
+
res.end(JSON.stringify({ error: "Invalid JSON" }));
|
|
91
|
+
return;
|
|
92
|
+
}
|
|
93
|
+
if (DRY_RUN) {
|
|
94
|
+
console.log("\n ๐ถ DRY RUN โ would dispatch (no GITHUB_TOKEN set)");
|
|
95
|
+
}
|
|
96
|
+
const result = handler.handle(payload);
|
|
97
|
+
console.log(` โ ${result.status}:`, JSON.stringify(result));
|
|
98
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
99
|
+
res.end(JSON.stringify(result, null, 2));
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
// 404
|
|
103
|
+
res.writeHead(404, { "Content-Type": "application/json" });
|
|
104
|
+
res.end(JSON.stringify({
|
|
105
|
+
endpoints: ["POST /webhook", "GET /health", "GET /mappings"],
|
|
106
|
+
error: "Not found",
|
|
107
|
+
}));
|
|
108
|
+
}
|
|
109
|
+
// ---------------------------------------------------------------------------
|
|
110
|
+
// Start
|
|
111
|
+
// ---------------------------------------------------------------------------
|
|
112
|
+
server.listen(PORT, () => {
|
|
113
|
+
const reverseMapping = buildReverseMapping(ROOT);
|
|
114
|
+
const slugCount = allTrackedSlugs(reverseMapping).length;
|
|
115
|
+
console.log();
|
|
116
|
+
console.log("=== AILF Webhook Server ===");
|
|
117
|
+
console.log();
|
|
118
|
+
console.log(` Port: ${PORT}`);
|
|
119
|
+
console.log(` Mode: ${DRY_RUN ? "DRY RUN (set GITHUB_TOKEN to dispatch)" : "LIVE"}`);
|
|
120
|
+
console.log(` Tracked slugs: ${slugCount}`);
|
|
121
|
+
console.log(` Debounce: ${handler.diagnostics().pendingSlugs}`);
|
|
122
|
+
console.log();
|
|
123
|
+
console.log(" Endpoints:");
|
|
124
|
+
console.log(` POST http://localhost:${PORT}/webhook โ handle webhook`);
|
|
125
|
+
console.log(` GET http://localhost:${PORT}/health โ diagnostics`);
|
|
126
|
+
console.log(` GET http://localhost:${PORT}/mappings โ slug โ area map`);
|
|
127
|
+
console.log();
|
|
128
|
+
});
|
|
129
|
+
// Graceful shutdown
|
|
130
|
+
process.on("SIGINT", () => {
|
|
131
|
+
console.log("\n Shutting down โ flushing debounce window...");
|
|
132
|
+
void handler.shutdown().then(() => {
|
|
133
|
+
server.close();
|
|
134
|
+
process.exit(0);
|
|
135
|
+
});
|
|
136
|
+
});
|
|
137
|
+
// ---------------------------------------------------------------------------
|
|
138
|
+
// Helpers
|
|
139
|
+
// ---------------------------------------------------------------------------
|
|
140
|
+
function readBody(req) {
|
|
141
|
+
return new Promise((resolve, reject) => {
|
|
142
|
+
const chunks = [];
|
|
143
|
+
req.on("data", (chunk) => chunks.push(chunk));
|
|
144
|
+
req.on("end", () => resolve(Buffer.concat(chunks).toString()));
|
|
145
|
+
req.on("error", reject);
|
|
146
|
+
});
|
|
147
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* weekly-digest.ts
|
|
3
|
+
*
|
|
4
|
+
* CLI script to generate and deliver a weekly evaluation digest.
|
|
5
|
+
*
|
|
6
|
+
* Queries the Sanity Content Lake for all reports within the configured
|
|
7
|
+
* lookback window, computes trend analysis, and delivers the digest
|
|
8
|
+
* via configured channels (Slack, stdout, or both).
|
|
9
|
+
*
|
|
10
|
+
* Usage:
|
|
11
|
+
* pnpm weekly-digest # send to configured Slack webhook
|
|
12
|
+
* pnpm weekly-digest --dry-run # print to stdout only
|
|
13
|
+
* pnpm weekly-digest --lookback 14 # 14-day lookback window
|
|
14
|
+
* pnpm weekly-digest --json # output raw JSON
|
|
15
|
+
*
|
|
16
|
+
* Environment variables:
|
|
17
|
+
* SLACK_WEBHOOK_URL โ Slack incoming webhook URL
|
|
18
|
+
* SANITY_API_TOKEN โ Sanity read token
|
|
19
|
+
* AILF_TRIGGER_TYPE โ set to "scheduled" by the cron workflow
|
|
20
|
+
* AILF_SCHEDULE โ the schedule name (e.g., "weekly-digest")
|
|
21
|
+
*
|
|
22
|
+
* @see docs/design-docs/report-store/implementation.md โ Phase 5
|
|
23
|
+
*/
|
|
24
|
+
export {};
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* weekly-digest.ts
|
|
3
|
+
*
|
|
4
|
+
* CLI script to generate and deliver a weekly evaluation digest.
|
|
5
|
+
*
|
|
6
|
+
* Queries the Sanity Content Lake for all reports within the configured
|
|
7
|
+
* lookback window, computes trend analysis, and delivers the digest
|
|
8
|
+
* via configured channels (Slack, stdout, or both).
|
|
9
|
+
*
|
|
10
|
+
* Usage:
|
|
11
|
+
* pnpm weekly-digest # send to configured Slack webhook
|
|
12
|
+
* pnpm weekly-digest --dry-run # print to stdout only
|
|
13
|
+
* pnpm weekly-digest --lookback 14 # 14-day lookback window
|
|
14
|
+
* pnpm weekly-digest --json # output raw JSON
|
|
15
|
+
*
|
|
16
|
+
* Environment variables:
|
|
17
|
+
* SLACK_WEBHOOK_URL โ Slack incoming webhook URL
|
|
18
|
+
* SANITY_API_TOKEN โ Sanity read token
|
|
19
|
+
* AILF_TRIGGER_TYPE โ set to "scheduled" by the cron workflow
|
|
20
|
+
* AILF_SCHEDULE โ the schedule name (e.g., "weekly-digest")
|
|
21
|
+
*
|
|
22
|
+
* @see docs/design-docs/report-store/implementation.md โ Phase 5
|
|
23
|
+
*/
|
|
24
|
+
import { config as dotenvConfig } from "dotenv";
|
|
25
|
+
import { existsSync } from "fs";
|
|
26
|
+
import { dirname, resolve } from "path";
|
|
27
|
+
import { fileURLToPath } from "url";
|
|
28
|
+
import { generateDigest } from "../schedules/digest.js";
|
|
29
|
+
import { getDigestConfig } from "../schedules/loader.js";
|
|
30
|
+
import { formatWeeklyDigest } from "../sinks/slack/format.js";
|
|
31
|
+
// Load root .env (same override behavior as pipeline.ts)
|
|
32
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
33
|
+
const envPath = resolve(__dirname, "..", "..", "..", "..", ".env");
|
|
34
|
+
if (existsSync(envPath)) {
|
|
35
|
+
dotenvConfig({ override: true, path: envPath });
|
|
36
|
+
}
|
|
37
|
+
// ---------------------------------------------------------------------------
|
|
38
|
+
// CLI argument parsing
|
|
39
|
+
// ---------------------------------------------------------------------------
|
|
40
|
+
const args = process.argv.slice(2);
|
|
41
|
+
function getOption(name) {
|
|
42
|
+
const idx = args.indexOf(`--${name}`);
|
|
43
|
+
return idx >= 0 && idx + 1 < args.length ? args[idx + 1] : undefined;
|
|
44
|
+
}
|
|
45
|
+
function hasFlag(name) {
|
|
46
|
+
return args.includes(`--${name}`);
|
|
47
|
+
}
|
|
48
|
+
const DRY_RUN = hasFlag("dry-run");
|
|
49
|
+
const JSON_OUTPUT = hasFlag("json");
|
|
50
|
+
const lookbackOverride = getOption("lookback");
|
|
51
|
+
// ---------------------------------------------------------------------------
|
|
52
|
+
// Main
|
|
53
|
+
// ---------------------------------------------------------------------------
|
|
54
|
+
async function main() {
|
|
55
|
+
console.log();
|
|
56
|
+
console.log("=== AI Literacy Weekly Digest ===");
|
|
57
|
+
console.log();
|
|
58
|
+
// Load digest config
|
|
59
|
+
const digestConfig = getDigestConfig();
|
|
60
|
+
const lookbackDays = lookbackOverride
|
|
61
|
+
? parseInt(lookbackOverride, 10)
|
|
62
|
+
: (digestConfig?.lookbackDays ?? 7);
|
|
63
|
+
console.log(` Lookback: ${lookbackDays} days`);
|
|
64
|
+
console.log(` Mode: ${DRY_RUN ? "dry run (stdout only)" : "live"}`);
|
|
65
|
+
console.log();
|
|
66
|
+
// Generate digest โ uses AILF_REPORT_* env vars for report store access,
|
|
67
|
+
// independent of SANITY_DATASET/SANITY_PROJECT_ID (which control doc evaluation)
|
|
68
|
+
const digest = await generateDigest({
|
|
69
|
+
dataset: process.env.AILF_REPORT_DATASET,
|
|
70
|
+
lookbackDays,
|
|
71
|
+
projectId: process.env.AILF_REPORT_PROJECT_ID,
|
|
72
|
+
token: process.env.AILF_REPORT_SANITY_API_TOKEN ?? process.env.SANITY_API_TOKEN,
|
|
73
|
+
});
|
|
74
|
+
if (!digest) {
|
|
75
|
+
console.log(" No reports found in the lookback window. Nothing to send.");
|
|
76
|
+
process.exit(0);
|
|
77
|
+
}
|
|
78
|
+
// Output
|
|
79
|
+
console.log(` Reports found: ${digest.reportCount}`);
|
|
80
|
+
console.log(` Overall: ${Math.round(digest.overallLatest)} (${digest.overallTrend})`);
|
|
81
|
+
console.log(` Improved: ${digest.improved.join(", ") || "none"}`);
|
|
82
|
+
console.log(` Regressed: ${digest.regressed.join(", ") || "none"}`);
|
|
83
|
+
console.log(` Stable: ${digest.stable.join(", ") || "none"}`);
|
|
84
|
+
console.log();
|
|
85
|
+
if (JSON_OUTPUT) {
|
|
86
|
+
console.log(JSON.stringify(digest, null, 2));
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
// Format for Slack
|
|
90
|
+
const message = formatWeeklyDigest(digest);
|
|
91
|
+
if (DRY_RUN) {
|
|
92
|
+
console.log(" --- Slack Message Preview ---");
|
|
93
|
+
console.log(` Text: ${message.text}`);
|
|
94
|
+
console.log();
|
|
95
|
+
for (const block of message.blocks) {
|
|
96
|
+
if (block.text) {
|
|
97
|
+
console.log(` [${block.type}] ${block.text.text}`);
|
|
98
|
+
}
|
|
99
|
+
if (block.fields) {
|
|
100
|
+
for (const field of block.fields) {
|
|
101
|
+
console.log(` [field] ${field.text}`);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
if (block.elements) {
|
|
105
|
+
for (const el of block.elements) {
|
|
106
|
+
console.log(` [element] ${el.text}`);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
console.log();
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
// Deliver via Slack
|
|
114
|
+
const webhookUrl = digestConfig?.slackWebhookUrl ?? process.env.SLACK_WEBHOOK_URL;
|
|
115
|
+
if (!webhookUrl) {
|
|
116
|
+
console.warn(" โ ๏ธ No Slack webhook URL configured. Set SLACK_WEBHOOK_URL or configure in schedules.yaml");
|
|
117
|
+
console.log(" Printing digest to stdout instead:");
|
|
118
|
+
console.log();
|
|
119
|
+
console.log(` ${message.text}`);
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
console.log(" Sending to Slack...");
|
|
123
|
+
try {
|
|
124
|
+
const response = await fetch(webhookUrl, {
|
|
125
|
+
body: JSON.stringify(message),
|
|
126
|
+
headers: { "Content-Type": "application/json" },
|
|
127
|
+
method: "POST",
|
|
128
|
+
});
|
|
129
|
+
if (response.ok) {
|
|
130
|
+
console.log(" โ
Digest sent successfully");
|
|
131
|
+
}
|
|
132
|
+
else {
|
|
133
|
+
const text = await response.text();
|
|
134
|
+
console.warn(` โ ๏ธ Slack delivery failed: ${response.status} ${text}`);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
catch (error) {
|
|
138
|
+
console.warn(` โ ๏ธ Slack delivery error: ${error instanceof Error ? error.message : String(error)}`);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
main().catch((error) => {
|
|
142
|
+
console.error("Fatal error:", error);
|
|
143
|
+
process.exit(1);
|
|
144
|
+
});
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* sinks/format-slack.ts
|
|
3
|
+
*
|
|
4
|
+
* Formats evaluation report data into Slack Block Kit structures for the
|
|
5
|
+
* SlackSink. Provides two message formats:
|
|
6
|
+
*
|
|
7
|
+
* - `formatRegressionAlert` โ detailed regression notification with
|
|
8
|
+
* per-area dimension breakdowns
|
|
9
|
+
* - `formatScoreSummary` โ compact score overview for general reporting
|
|
10
|
+
*
|
|
11
|
+
* @see docs/design-docs/report-store/sink-architecture.md
|
|
12
|
+
*/
|
|
13
|
+
import type { Report } from "../pipeline/types.js";
|
|
14
|
+
import type { DigestSummary } from "../schedules/digest.js";
|
|
15
|
+
export interface SlackMessage {
|
|
16
|
+
blocks: SlackBlock[];
|
|
17
|
+
text: string;
|
|
18
|
+
}
|
|
19
|
+
interface SlackBlock {
|
|
20
|
+
elements?: Array<{
|
|
21
|
+
text: string;
|
|
22
|
+
type: "mrkdwn" | "plain_text";
|
|
23
|
+
}>;
|
|
24
|
+
fields?: Array<{
|
|
25
|
+
text: string;
|
|
26
|
+
type: "mrkdwn" | "plain_text";
|
|
27
|
+
}>;
|
|
28
|
+
text?: {
|
|
29
|
+
text: string;
|
|
30
|
+
type: "mrkdwn" | "plain_text";
|
|
31
|
+
};
|
|
32
|
+
type: "context" | "divider" | "header" | "section";
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Format a regression alert for areas that have regressed.
|
|
36
|
+
*
|
|
37
|
+
* Produces a rich Slack message with:
|
|
38
|
+
* - Header with overall score change
|
|
39
|
+
* - Context metadata (mode, source, timestamp, promptfoo link)
|
|
40
|
+
* - Per-area regression details with dimension breakdowns
|
|
41
|
+
* - Brief mentions of improved and unchanged areas
|
|
42
|
+
*/
|
|
43
|
+
export declare function formatRegressionAlert(report: Report): SlackMessage;
|
|
44
|
+
/**
|
|
45
|
+
* Format a general score summary for Slack reporting.
|
|
46
|
+
*
|
|
47
|
+
* Produces a compact overview with:
|
|
48
|
+
* - Overall score with grade emoji
|
|
49
|
+
* - Per-area score table
|
|
50
|
+
* - Cost summary (if available)
|
|
51
|
+
* - Promptfoo link (if available)
|
|
52
|
+
*/
|
|
53
|
+
export declare function formatScoreSummary(report: Report): SlackMessage;
|
|
54
|
+
/**
|
|
55
|
+
* Format a weekly digest summary for Slack.
|
|
56
|
+
*
|
|
57
|
+
* Produces a summary message covering score trends over a time window:
|
|
58
|
+
* - Header with overall trend direction and score
|
|
59
|
+
* - Per-area trend table with arrows
|
|
60
|
+
* - Lists of improved, regressed, and stable areas
|
|
61
|
+
* - Report count and time window metadata
|
|
62
|
+
*/
|
|
63
|
+
export declare function formatWeeklyDigest(digest: DigestSummary): SlackMessage;
|
|
64
|
+
export {};
|
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* sinks/format-slack.ts
|
|
3
|
+
*
|
|
4
|
+
* Formats evaluation report data into Slack Block Kit structures for the
|
|
5
|
+
* SlackSink. Provides two message formats:
|
|
6
|
+
*
|
|
7
|
+
* - `formatRegressionAlert` โ detailed regression notification with
|
|
8
|
+
* per-area dimension breakdowns
|
|
9
|
+
* - `formatScoreSummary` โ compact score overview for general reporting
|
|
10
|
+
*
|
|
11
|
+
* @see docs/design-docs/report-store/sink-architecture.md
|
|
12
|
+
*/
|
|
13
|
+
// ---------------------------------------------------------------------------
|
|
14
|
+
// Helpers
|
|
15
|
+
// ---------------------------------------------------------------------------
|
|
16
|
+
/**
|
|
17
|
+
* Format a regression alert for areas that have regressed.
|
|
18
|
+
*
|
|
19
|
+
* Produces a rich Slack message with:
|
|
20
|
+
* - Header with overall score change
|
|
21
|
+
* - Context metadata (mode, source, timestamp, promptfoo link)
|
|
22
|
+
* - Per-area regression details with dimension breakdowns
|
|
23
|
+
* - Brief mentions of improved and unchanged areas
|
|
24
|
+
*/
|
|
25
|
+
export function formatRegressionAlert(report) {
|
|
26
|
+
const { comparison, provenance, summary } = report;
|
|
27
|
+
if (!comparison) {
|
|
28
|
+
return {
|
|
29
|
+
blocks: [
|
|
30
|
+
{
|
|
31
|
+
text: { text: "โ ๏ธ No comparison data available", type: "mrkdwn" },
|
|
32
|
+
type: "section",
|
|
33
|
+
},
|
|
34
|
+
],
|
|
35
|
+
text: "No comparison data available",
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
const baselineScore = Math.round(comparison.baseline.overall.avgScore);
|
|
39
|
+
const experimentScore = Math.round(comparison.experiment.overall.avgScore);
|
|
40
|
+
const delta = Math.round(comparison.deltas.overall);
|
|
41
|
+
const blocks = [];
|
|
42
|
+
// Header โ emoji + title + overall score change
|
|
43
|
+
blocks.push({
|
|
44
|
+
text: {
|
|
45
|
+
text: `๐ *AI Literacy Score Regression*\n` +
|
|
46
|
+
`Overall: ${baselineScore} โ ${experimentScore} (${formatDelta(delta)})`,
|
|
47
|
+
type: "mrkdwn",
|
|
48
|
+
},
|
|
49
|
+
type: "section",
|
|
50
|
+
});
|
|
51
|
+
// Context โ mode, source, timestamp, promptfoo link
|
|
52
|
+
const contextElements = [
|
|
53
|
+
{ text: `*Mode:* ${provenance.mode}`, type: "mrkdwn" },
|
|
54
|
+
{ text: `*Source:* ${provenance.source.name}`, type: "mrkdwn" },
|
|
55
|
+
{ text: `*Date:* ${readableDate(summary.timestamp)}`, type: "mrkdwn" },
|
|
56
|
+
];
|
|
57
|
+
if (provenance.promptfooUrl) {
|
|
58
|
+
contextElements.push({
|
|
59
|
+
text: `<${provenance.promptfooUrl}|View in Promptfoo>`,
|
|
60
|
+
type: "mrkdwn",
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
blocks.push({ elements: contextElements, type: "context" });
|
|
64
|
+
// Divider
|
|
65
|
+
blocks.push({ type: "divider" });
|
|
66
|
+
// Regressed areas โ detailed fields with dimension breakdowns
|
|
67
|
+
const regressedAreas = comparison.areas.filter((a) => a.change === "regressed");
|
|
68
|
+
if (regressedAreas.length > 0) {
|
|
69
|
+
const fields = regressedAreas.map((area) => ({
|
|
70
|
+
text: `*${area.area}:* ${Math.round(area.baseline)} โ ` +
|
|
71
|
+
`${Math.round(area.experiment)} (${formatDelta(Math.round(area.delta))})\n` +
|
|
72
|
+
dimensionBreakdown(area.dimensions),
|
|
73
|
+
type: "mrkdwn",
|
|
74
|
+
}));
|
|
75
|
+
blocks.push({
|
|
76
|
+
fields,
|
|
77
|
+
text: {
|
|
78
|
+
text: `*Regressed Areas (${regressedAreas.length})*`,
|
|
79
|
+
type: "mrkdwn",
|
|
80
|
+
},
|
|
81
|
+
type: "section",
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
// Improved areas โ compact mention
|
|
85
|
+
if (comparison.improved.length > 0) {
|
|
86
|
+
blocks.push({
|
|
87
|
+
text: {
|
|
88
|
+
text: `๐ ${comparison.improved.length} area${comparison.improved.length === 1 ? "" : "s"} improved: ${comparison.improved.join(", ")}`,
|
|
89
|
+
type: "mrkdwn",
|
|
90
|
+
},
|
|
91
|
+
type: "section",
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
// Unchanged areas โ brief mention
|
|
95
|
+
if (comparison.unchanged.length > 0) {
|
|
96
|
+
blocks.push({
|
|
97
|
+
text: {
|
|
98
|
+
text: `โก๏ธ ${comparison.unchanged.length} area${comparison.unchanged.length === 1 ? "" : "s"} unchanged`,
|
|
99
|
+
type: "mrkdwn",
|
|
100
|
+
},
|
|
101
|
+
type: "section",
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
return {
|
|
105
|
+
blocks,
|
|
106
|
+
text: `๐ AI Literacy Score Regression: ${baselineScore} โ ${experimentScore} (${formatDelta(delta)})`,
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Format a general score summary for Slack reporting.
|
|
111
|
+
*
|
|
112
|
+
* Produces a compact overview with:
|
|
113
|
+
* - Overall score with grade emoji
|
|
114
|
+
* - Per-area score table
|
|
115
|
+
* - Cost summary (if available)
|
|
116
|
+
* - Promptfoo link (if available)
|
|
117
|
+
*/
|
|
118
|
+
export function formatScoreSummary(report) {
|
|
119
|
+
const { provenance, summary } = report;
|
|
120
|
+
const overall = Math.round(summary.overall.avgScore);
|
|
121
|
+
const blocks = [];
|
|
122
|
+
// Header โ overall score with emoji
|
|
123
|
+
blocks.push({
|
|
124
|
+
text: {
|
|
125
|
+
text: `${gradeEmoji(overall)} *AI Literacy Score: ${overall}*`,
|
|
126
|
+
type: "mrkdwn",
|
|
127
|
+
},
|
|
128
|
+
type: "section",
|
|
129
|
+
});
|
|
130
|
+
// Context โ mode, source
|
|
131
|
+
const contextElements = [
|
|
132
|
+
{ text: `*Mode:* ${provenance.mode}`, type: "mrkdwn" },
|
|
133
|
+
{ text: `*Source:* ${provenance.source.name}`, type: "mrkdwn" },
|
|
134
|
+
{ text: `*Date:* ${readableDate(summary.timestamp)}`, type: "mrkdwn" },
|
|
135
|
+
];
|
|
136
|
+
blocks.push({ elements: contextElements, type: "context" });
|
|
137
|
+
// Divider
|
|
138
|
+
blocks.push({ type: "divider" });
|
|
139
|
+
// Per-area score table as markdown
|
|
140
|
+
const rows = summary.scores
|
|
141
|
+
.map((s) => {
|
|
142
|
+
const emoji = gradeEmoji(s.totalScore);
|
|
143
|
+
return `${emoji} *${s.feature}*: ${Math.round(s.totalScore)} _(T:${Math.round(s.taskCompletion)} ยท C:${Math.round(s.codeCorrectness)} ยท D:${Math.round(s.docCoverage)})_`;
|
|
144
|
+
})
|
|
145
|
+
.join("\n");
|
|
146
|
+
blocks.push({
|
|
147
|
+
text: { text: rows, type: "mrkdwn" },
|
|
148
|
+
type: "section",
|
|
149
|
+
});
|
|
150
|
+
// Cost summary
|
|
151
|
+
if (summary.overall.cost) {
|
|
152
|
+
blocks.push({
|
|
153
|
+
text: {
|
|
154
|
+
text: `๐ฐ Total cost: $${summary.overall.cost.total.toFixed(2)} ($${summary.overall.cost.perTest.toFixed(3)}/test)`,
|
|
155
|
+
type: "mrkdwn",
|
|
156
|
+
},
|
|
157
|
+
type: "section",
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
// Promptfoo link
|
|
161
|
+
if (provenance.promptfooUrl) {
|
|
162
|
+
blocks.push({
|
|
163
|
+
text: {
|
|
164
|
+
text: `๐ <${provenance.promptfooUrl}|View full results in Promptfoo>`,
|
|
165
|
+
type: "mrkdwn",
|
|
166
|
+
},
|
|
167
|
+
type: "section",
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
return {
|
|
171
|
+
blocks,
|
|
172
|
+
text: `${gradeEmoji(overall)} AI Literacy Score: ${overall}`,
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Format a weekly digest summary for Slack.
|
|
177
|
+
*
|
|
178
|
+
* Produces a summary message covering score trends over a time window:
|
|
179
|
+
* - Header with overall trend direction and score
|
|
180
|
+
* - Per-area trend table with arrows
|
|
181
|
+
* - Lists of improved, regressed, and stable areas
|
|
182
|
+
* - Report count and time window metadata
|
|
183
|
+
*/
|
|
184
|
+
export function formatWeeklyDigest(digest) {
|
|
185
|
+
const trendEmoji = digest.overallTrend === "improving"
|
|
186
|
+
? "๐"
|
|
187
|
+
: digest.overallTrend === "regressing"
|
|
188
|
+
? "๐"
|
|
189
|
+
: "โก๏ธ";
|
|
190
|
+
const blocks = [];
|
|
191
|
+
// Header โ overall trend
|
|
192
|
+
blocks.push({
|
|
193
|
+
text: {
|
|
194
|
+
text: `${trendEmoji} *Weekly AI Literacy Digest*\n` +
|
|
195
|
+
`Overall: ${Math.round(digest.overallLatest)} (${formatDelta(Math.round(digest.overallDelta))} this week)`,
|
|
196
|
+
type: "mrkdwn",
|
|
197
|
+
},
|
|
198
|
+
type: "section",
|
|
199
|
+
});
|
|
200
|
+
// Context โ time window and report count
|
|
201
|
+
blocks.push({
|
|
202
|
+
elements: [
|
|
203
|
+
{
|
|
204
|
+
text: `*Period:* ${readableDate(digest.lookbackStart)} โ ${readableDate(digest.lookbackEnd)}`,
|
|
205
|
+
type: "mrkdwn",
|
|
206
|
+
},
|
|
207
|
+
{
|
|
208
|
+
text: `*Reports:* ${digest.reportCount}`,
|
|
209
|
+
type: "mrkdwn",
|
|
210
|
+
},
|
|
211
|
+
],
|
|
212
|
+
type: "context",
|
|
213
|
+
});
|
|
214
|
+
blocks.push({ type: "divider" });
|
|
215
|
+
// Per-area trend table
|
|
216
|
+
if (digest.areaTrends.length > 0) {
|
|
217
|
+
const rows = digest.areaTrends
|
|
218
|
+
.map((t) => {
|
|
219
|
+
const arrow = t.trend === "improving" ? "โ" : t.trend === "regressing" ? "โ" : "โ";
|
|
220
|
+
const emoji = gradeEmoji(t.lastScore);
|
|
221
|
+
return `${emoji} *${t.area}*: ${Math.round(t.lastScore)} ${arrow} (${formatDelta(Math.round(t.scoreDelta))})`;
|
|
222
|
+
})
|
|
223
|
+
.join("\n");
|
|
224
|
+
blocks.push({
|
|
225
|
+
text: { text: rows, type: "mrkdwn" },
|
|
226
|
+
type: "section",
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
// Summary badges
|
|
230
|
+
if (digest.improved.length > 0) {
|
|
231
|
+
blocks.push({
|
|
232
|
+
text: {
|
|
233
|
+
text: `๐ *Improved:* ${digest.improved.join(", ")}`,
|
|
234
|
+
type: "mrkdwn",
|
|
235
|
+
},
|
|
236
|
+
type: "section",
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
if (digest.regressed.length > 0) {
|
|
240
|
+
blocks.push({
|
|
241
|
+
text: {
|
|
242
|
+
text: `๐ *Regressed:* ${digest.regressed.join(", ")}`,
|
|
243
|
+
type: "mrkdwn",
|
|
244
|
+
},
|
|
245
|
+
type: "section",
|
|
246
|
+
});
|
|
247
|
+
}
|
|
248
|
+
if (digest.stable.length > 0) {
|
|
249
|
+
blocks.push({
|
|
250
|
+
text: {
|
|
251
|
+
text: `โก๏ธ *Stable:* ${digest.stable.join(", ")}`,
|
|
252
|
+
type: "mrkdwn",
|
|
253
|
+
},
|
|
254
|
+
type: "section",
|
|
255
|
+
});
|
|
256
|
+
}
|
|
257
|
+
// Cost summary if available
|
|
258
|
+
if (digest.totalCost !== undefined) {
|
|
259
|
+
blocks.push({
|
|
260
|
+
text: {
|
|
261
|
+
text: `๐ฐ Total evaluation cost this week: ${digest.totalCost.toFixed(2)}`,
|
|
262
|
+
type: "mrkdwn",
|
|
263
|
+
},
|
|
264
|
+
type: "section",
|
|
265
|
+
});
|
|
266
|
+
}
|
|
267
|
+
return {
|
|
268
|
+
blocks,
|
|
269
|
+
text: `${trendEmoji} Weekly AI Literacy Digest: ${Math.round(digest.overallLatest)} (${formatDelta(Math.round(digest.overallDelta))})`,
|
|
270
|
+
};
|
|
271
|
+
}
|
|
272
|
+
/** Build a dimension breakdown string for an area delta */
|
|
273
|
+
function dimensionBreakdown(dimensions) {
|
|
274
|
+
return [
|
|
275
|
+
`Task: ${formatDelta(dimensions.taskCompletion.delta)}`,
|
|
276
|
+
`Code: ${formatDelta(dimensions.codeCorrectness.delta)}`,
|
|
277
|
+
`Docs: ${formatDelta(dimensions.docCoverage.delta)}`,
|
|
278
|
+
].join(" ยท ");
|
|
279
|
+
}
|
|
280
|
+
/** Format a numeric delta with explicit sign: "+4", "-2", or "0" */
|
|
281
|
+
function formatDelta(n) {
|
|
282
|
+
if (n > 0)
|
|
283
|
+
return `+${n}`;
|
|
284
|
+
if (n < 0)
|
|
285
|
+
return `${n}`;
|
|
286
|
+
return "0";
|
|
287
|
+
}
|
|
288
|
+
/** Score-tier emoji: โ
(โฅ80), ๐ก (โฅ70), ๐ (โฅ50), ๐ด (<50) */
|
|
289
|
+
function gradeEmoji(score) {
|
|
290
|
+
if (score >= 80)
|
|
291
|
+
return "โ
";
|
|
292
|
+
if (score >= 70)
|
|
293
|
+
return "๐ก";
|
|
294
|
+
if (score >= 50)
|
|
295
|
+
return "๐ ";
|
|
296
|
+
return "๐ด";
|
|
297
|
+
}
|
|
298
|
+
/** Format an ISO timestamp into a readable date string */
|
|
299
|
+
function readableDate(iso) {
|
|
300
|
+
const d = new Date(iso);
|
|
301
|
+
return d.toLocaleDateString("en-US", {
|
|
302
|
+
day: "numeric",
|
|
303
|
+
month: "short",
|
|
304
|
+
year: "numeric",
|
|
305
|
+
});
|
|
306
|
+
}
|