@sanity/ailf 0.4.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config/features.ts +23 -0
- package/config/models.ts +83 -0
- package/config/prompts.ts +16 -0
- package/config/rubrics.ts +225 -0
- package/config/schedules.ts +47 -0
- package/config/sinks.ts +37 -0
- package/config/sources.ts +21 -0
- package/config/thresholds.ts +61 -0
- package/dist/_vendor/ailf-core/config-helpers.d.ts +174 -0
- package/dist/_vendor/ailf-core/config-helpers.js +150 -0
- package/dist/_vendor/ailf-core/env-helper.d.ts +35 -0
- package/dist/_vendor/ailf-core/env-helper.js +45 -0
- package/dist/_vendor/ailf-core/examples/index.d.ts +10 -10
- package/dist/_vendor/ailf-core/examples/index.js +10 -10
- package/dist/_vendor/ailf-core/index.d.ts +3 -0
- package/dist/_vendor/ailf-core/index.js +5 -0
- package/dist/_vendor/ailf-core/ports/context.d.ts +15 -2
- package/dist/_vendor/ailf-core/ports/doc-fetcher.d.ts +2 -2
- package/dist/_vendor/ailf-core/ports/index.d.ts +2 -1
- package/dist/_vendor/ailf-core/ports/mode-handler.d.ts +129 -0
- package/dist/_vendor/ailf-core/ports/mode-handler.js +19 -0
- package/dist/_vendor/ailf-core/ports/task-source.d.ts +16 -122
- package/dist/_vendor/ailf-core/ports/task-source.js +7 -7
- package/dist/_vendor/ailf-core/schemas/eval-config.d.ts +7 -2
- package/dist/_vendor/ailf-core/schemas/eval-config.js +7 -2
- package/dist/_vendor/ailf-core/schemas/pipeline-request.d.ts +8 -3
- package/dist/_vendor/ailf-core/schemas/pipeline-request.js +6 -1
- package/dist/_vendor/ailf-core/schemas/pipeline.d.ts +32 -31
- package/dist/_vendor/ailf-core/schemas/pipeline.js +52 -12
- package/dist/_vendor/ailf-core/schemas/schedules.d.ts +14 -4
- package/dist/_vendor/ailf-core/schemas/schedules.js +6 -2
- package/dist/_vendor/ailf-core/schemas/sinks.d.ts +1 -1
- package/dist/_vendor/ailf-core/services/comparison-formatters.js +57 -19
- package/dist/_vendor/ailf-core/services/index.d.ts +2 -1
- package/dist/_vendor/ailf-core/services/index.js +2 -1
- package/dist/_vendor/ailf-core/services/scoring-engine.d.ts +153 -0
- package/dist/_vendor/ailf-core/services/scoring-engine.js +237 -0
- package/dist/_vendor/ailf-core/services/scoring.d.ts +15 -2
- package/dist/_vendor/ailf-core/services/scoring.js +25 -15
- package/dist/_vendor/ailf-core/types/branded-ids.d.ts +137 -0
- package/dist/_vendor/ailf-core/types/branded-ids.js +136 -0
- package/dist/_vendor/ailf-core/types/eval-mode-config.d.ts +150 -0
- package/dist/_vendor/ailf-core/types/eval-mode-config.js +24 -0
- package/dist/_vendor/ailf-core/types/generalized-task.d.ts +319 -0
- package/dist/_vendor/ailf-core/types/generalized-task.js +13 -0
- package/dist/_vendor/ailf-core/types/index.d.ts +45 -81
- package/dist/_vendor/ailf-core/types/index.js +8 -1
- package/dist/_vendor/ailf-core/types/plugin-registry.d.ts +202 -0
- package/dist/_vendor/ailf-core/types/plugin-registry.js +132 -0
- package/dist/_vendor/ailf-core/types/storage-schema.d.ts +199 -0
- package/dist/_vendor/ailf-core/types/storage-schema.js +39 -0
- package/dist/_vendor/ailf-core/types/task-graph.d.ts +86 -0
- package/dist/_vendor/ailf-core/types/task-graph.js +20 -0
- package/dist/_vendor/ailf-core/types/trace.d.ts +118 -0
- package/dist/_vendor/ailf-core/types/trace.js +18 -0
- package/dist/_vendor/ailf-core/types/variable-envelope.d.ts +80 -0
- package/dist/_vendor/ailf-core/types/variable-envelope.js +16 -0
- package/dist/_vendor/ailf-shared/dimension-names.d.ts +5 -18
- package/dist/_vendor/ailf-shared/dimension-names.js +6 -24
- package/dist/_vendor/ailf-shared/eval-modes.d.ts +38 -6
- package/dist/_vendor/ailf-shared/eval-modes.js +26 -2
- package/dist/_vendor/ailf-shared/index.d.ts +0 -1
- package/dist/_vendor/ailf-shared/index.js +0 -1
- package/dist/adapters/api-client/build-request.js +14 -13
- package/dist/adapters/config-sources/file-config-adapter.d.ts +20 -11
- package/dist/adapters/config-sources/file-config-adapter.js +38 -12
- package/dist/adapters/config-sources/index.d.ts +2 -0
- package/dist/adapters/config-sources/index.js +1 -0
- package/dist/adapters/config-sources/ts-config-loader.d.ts +59 -0
- package/dist/adapters/config-sources/ts-config-loader.js +133 -0
- package/dist/adapters/doc-fetchers/sanity-doc-fetcher.d.ts +3 -2
- package/dist/adapters/doc-fetchers/sanity-doc-fetcher.js +7 -2
- package/dist/adapters/task-sources/composite-task-source.d.ts +3 -3
- package/dist/adapters/task-sources/composite-task-source.js +1 -1
- package/dist/adapters/task-sources/content-lake-task-source.d.ts +7 -6
- package/dist/adapters/task-sources/content-lake-task-source.js +22 -23
- package/dist/adapters/task-sources/index.d.ts +1 -0
- package/dist/adapters/task-sources/index.js +1 -0
- package/dist/adapters/task-sources/repo-task-source.d.ts +4 -4
- package/dist/adapters/task-sources/repo-task-source.js +69 -16
- package/dist/adapters/task-sources/task-file-loader.d.ts +64 -0
- package/dist/adapters/task-sources/task-file-loader.js +83 -0
- package/dist/adapters/task-sources/yaml-task-source.d.ts +6 -6
- package/dist/adapters/task-sources/yaml-task-source.js +19 -16
- package/dist/cli.js +0 -2
- package/dist/commands/baseline.js +4 -1
- package/dist/commands/calculate-scores.js +1 -1
- package/dist/commands/coverage-audit.js +7 -1
- package/dist/commands/explain-handler.js +25 -23
- package/dist/commands/fetch-docs.js +3 -2
- package/dist/commands/generate-configs.js +1 -1
- package/dist/commands/interactive.js +11 -7
- package/dist/commands/pipeline-action.d.ts +2 -0
- package/dist/commands/pipeline-action.js +16 -6
- package/dist/commands/pipeline.d.ts +1 -0
- package/dist/commands/pipeline.js +4 -2
- package/dist/commands/pr-comment.js +1 -1
- package/dist/commands/publish.js +2 -2
- package/dist/commands/readiness-report.js +13 -6
- package/dist/composition-root.d.ts +1 -1
- package/dist/composition-root.js +67 -4
- package/dist/orchestration/build-app-context.js +1 -0
- package/dist/orchestration/build-step-sequence.js +24 -6
- package/dist/orchestration/steps/calculate-scores-step.js +24 -11
- package/dist/orchestration/steps/fetch-docs-step.js +6 -4
- package/dist/orchestration/steps/gap-analysis-step.js +8 -7
- package/dist/orchestration/steps/generate-configs-step.d.ts +16 -3
- package/dist/orchestration/steps/generate-configs-step.js +245 -51
- package/dist/orchestration/steps/grader-consistency-step.js +7 -4
- package/dist/orchestration/steps/mirror-repo-tasks-step.js +1 -1
- package/dist/orchestration/steps/readiness-step.js +5 -6
- package/dist/orchestration/steps/run-eval-step.d.ts +1 -2
- package/dist/orchestration/steps/run-eval-step.js +8 -7
- package/dist/pipeline/cache.d.ts +1 -1
- package/dist/pipeline/cache.js +36 -8
- package/dist/pipeline/calculate-scores.d.ts +5 -7
- package/dist/pipeline/calculate-scores.js +74 -153
- package/dist/pipeline/checks.js +2 -2
- package/dist/pipeline/compare.js +8 -8
- package/dist/pipeline/compiler/__tests__/agent-harness-handler.test.d.ts +10 -0
- package/dist/pipeline/compiler/__tests__/agent-harness-handler.test.js +288 -0
- package/dist/pipeline/compiler/__tests__/assertion-mapper.test.d.ts +9 -0
- package/dist/pipeline/compiler/__tests__/assertion-mapper.test.js +145 -0
- package/dist/pipeline/compiler/__tests__/knowledge-probe-handler.test.d.ts +10 -0
- package/dist/pipeline/compiler/__tests__/knowledge-probe-handler.test.js +314 -0
- package/dist/pipeline/compiler/__tests__/literacy-handler.test.d.ts +10 -0
- package/dist/pipeline/compiler/__tests__/literacy-handler.test.js +486 -0
- package/dist/pipeline/compiler/__tests__/mcp-server-handler.test.d.ts +10 -0
- package/dist/pipeline/compiler/__tests__/mcp-server-handler.test.js +355 -0
- package/dist/pipeline/compiler/__tests__/promptfoo-compiler.test.d.ts +9 -0
- package/dist/pipeline/compiler/__tests__/promptfoo-compiler.test.js +333 -0
- package/dist/pipeline/compiler/__tests__/sandbox-and-fixtures.test.d.ts +12 -0
- package/dist/pipeline/compiler/__tests__/sandbox-and-fixtures.test.js +210 -0
- package/dist/pipeline/compiler/__tests__/scoring-and-presets.test.d.ts +7 -0
- package/dist/pipeline/compiler/__tests__/scoring-and-presets.test.js +471 -0
- package/dist/pipeline/compiler/__tests__/scoring-bridge.test.d.ts +10 -0
- package/dist/pipeline/compiler/__tests__/scoring-bridge.test.js +184 -0
- package/dist/pipeline/compiler/__tests__/task-graph-builder.test.d.ts +8 -0
- package/dist/pipeline/compiler/__tests__/task-graph-builder.test.js +301 -0
- package/dist/pipeline/compiler/__tests__/telemetry.test.d.ts +9 -0
- package/dist/pipeline/compiler/__tests__/telemetry.test.js +503 -0
- package/dist/pipeline/compiler/assertion-mapper.d.ts +58 -0
- package/dist/pipeline/compiler/assertion-mapper.js +175 -0
- package/dist/pipeline/compiler/compiler-to-yaml.d.ts +51 -0
- package/dist/pipeline/compiler/compiler-to-yaml.js +222 -0
- package/dist/pipeline/compiler/config-loader.d.ts +56 -0
- package/dist/pipeline/compiler/config-loader.js +111 -0
- package/dist/pipeline/compiler/fixture-resolver.d.ts +41 -0
- package/dist/pipeline/compiler/fixture-resolver.js +113 -0
- package/dist/pipeline/compiler/hash.d.ts +11 -0
- package/dist/pipeline/compiler/hash.js +18 -0
- package/dist/pipeline/compiler/ignore-fields.d.ts +53 -0
- package/dist/pipeline/compiler/ignore-fields.js +113 -0
- package/dist/pipeline/compiler/index.d.ts +29 -0
- package/dist/pipeline/compiler/index.js +45 -0
- package/dist/pipeline/compiler/literacy-bridge.d.ts +102 -0
- package/dist/pipeline/compiler/literacy-bridge.js +172 -0
- package/dist/pipeline/compiler/mode-handlers/__fixtures__/agent-harness-example-tasks.d.ts +14 -0
- package/dist/pipeline/compiler/mode-handlers/__fixtures__/agent-harness-example-tasks.js +152 -0
- package/dist/pipeline/compiler/mode-handlers/__fixtures__/knowledge-probe-example-tasks.d.ts +32 -0
- package/dist/pipeline/compiler/mode-handlers/__fixtures__/knowledge-probe-example-tasks.js +176 -0
- package/dist/pipeline/compiler/mode-handlers/__fixtures__/mcp-example-tasks.d.ts +49 -0
- package/dist/pipeline/compiler/mode-handlers/__fixtures__/mcp-example-tasks.js +259 -0
- package/dist/pipeline/compiler/mode-handlers/agent-harness-handler.d.ts +70 -0
- package/dist/pipeline/compiler/mode-handlers/agent-harness-handler.js +485 -0
- package/dist/pipeline/compiler/mode-handlers/index.d.ts +16 -0
- package/dist/pipeline/compiler/mode-handlers/index.js +21 -0
- package/dist/pipeline/compiler/mode-handlers/knowledge-probe-handler.d.ts +76 -0
- package/dist/pipeline/compiler/mode-handlers/knowledge-probe-handler.js +245 -0
- package/dist/pipeline/compiler/mode-handlers/literacy-handler.d.ts +89 -0
- package/dist/pipeline/compiler/mode-handlers/literacy-handler.js +379 -0
- package/dist/pipeline/compiler/mode-handlers/mcp-assertions.d.ts +50 -0
- package/dist/pipeline/compiler/mode-handlers/mcp-assertions.js +277 -0
- package/dist/pipeline/compiler/mode-handlers/mcp-server-handler.d.ts +67 -0
- package/dist/pipeline/compiler/mode-handlers/mcp-server-handler.js +309 -0
- package/dist/pipeline/compiler/presets/index.d.ts +9 -0
- package/dist/pipeline/compiler/presets/index.js +8 -0
- package/dist/pipeline/compiler/presets/sanity-literacy.d.ts +45 -0
- package/dist/pipeline/compiler/presets/sanity-literacy.js +354 -0
- package/dist/pipeline/compiler/promptfoo-compiler.d.ts +96 -0
- package/dist/pipeline/compiler/promptfoo-compiler.js +230 -0
- package/dist/pipeline/compiler/provider-assembler.d.ts +39 -0
- package/dist/pipeline/compiler/provider-assembler.js +137 -0
- package/dist/pipeline/compiler/sandbox/docker-sandbox.d.ts +21 -0
- package/dist/pipeline/compiler/sandbox/docker-sandbox.js +136 -0
- package/dist/pipeline/compiler/sandbox/fixture-provisioner.d.ts +69 -0
- package/dist/pipeline/compiler/sandbox/fixture-provisioner.js +189 -0
- package/dist/pipeline/compiler/sandbox/git-worktree-sandbox.d.ts +20 -0
- package/dist/pipeline/compiler/sandbox/git-worktree-sandbox.js +114 -0
- package/dist/pipeline/compiler/sandbox/index.d.ts +10 -0
- package/dist/pipeline/compiler/sandbox/index.js +11 -0
- package/dist/pipeline/compiler/sandbox/sandbox-selector.d.ts +35 -0
- package/dist/pipeline/compiler/sandbox/sandbox-selector.js +86 -0
- package/dist/pipeline/compiler/sandbox/sandbox-strategy.d.ts +81 -0
- package/dist/pipeline/compiler/sandbox/sandbox-strategy.js +15 -0
- package/dist/pipeline/compiler/sandbox/tempdir-sandbox.d.ts +20 -0
- package/dist/pipeline/compiler/sandbox/tempdir-sandbox.js +74 -0
- package/dist/pipeline/compiler/scoring-bridge.d.ts +49 -0
- package/dist/pipeline/compiler/scoring-bridge.js +114 -0
- package/dist/pipeline/compiler/task-graph-builder.d.ts +54 -0
- package/dist/pipeline/compiler/task-graph-builder.js +291 -0
- package/dist/pipeline/compiler/telemetry/cost-tracker.d.ts +90 -0
- package/dist/pipeline/compiler/telemetry/cost-tracker.js +146 -0
- package/dist/pipeline/compiler/telemetry/index.d.ts +14 -0
- package/dist/pipeline/compiler/telemetry/index.js +19 -0
- package/dist/pipeline/compiler/telemetry/redactor.d.ts +58 -0
- package/dist/pipeline/compiler/telemetry/redactor.js +222 -0
- package/dist/pipeline/compiler/telemetry/tool-classifier.d.ts +32 -0
- package/dist/pipeline/compiler/telemetry/tool-classifier.js +120 -0
- package/dist/pipeline/compiler/telemetry/trace-collector.d.ts +75 -0
- package/dist/pipeline/compiler/telemetry/trace-collector.js +297 -0
- package/dist/pipeline/compiler/telemetry/trace-store.d.ts +78 -0
- package/dist/pipeline/compiler/telemetry/trace-store.js +85 -0
- package/dist/pipeline/compiler/variable-resolver.d.ts +46 -0
- package/dist/pipeline/compiler/variable-resolver.js +115 -0
- package/dist/pipeline/coverage-audit.d.ts +15 -5
- package/dist/pipeline/coverage-audit.js +41 -22
- package/dist/pipeline/eval-constants.d.ts +16 -6
- package/dist/pipeline/eval-constants.js +25 -4
- package/dist/pipeline/eval-fingerprint.d.ts +2 -2
- package/dist/pipeline/eval-fingerprint.js +8 -9
- package/dist/pipeline/expand-tasks.d.ts +23 -14
- package/dist/pipeline/expand-tasks.js +37 -31
- package/dist/pipeline/gap-analysis.d.ts +1 -1
- package/dist/pipeline/gap-analysis.js +2 -2
- package/dist/pipeline/generate-configs.d.ts +22 -4
- package/dist/pipeline/generate-configs.js +53 -24
- package/dist/pipeline/grader-api.d.ts +3 -3
- package/dist/pipeline/grader-api.js +5 -12
- package/dist/pipeline/grader-compare-runner.js +20 -27
- package/dist/pipeline/grader-comparison.d.ts +4 -8
- package/dist/pipeline/grader-comparison.js +11 -17
- package/dist/pipeline/grader-consistency-runner.d.ts +2 -3
- package/dist/pipeline/grader-consistency-runner.js +18 -21
- package/dist/pipeline/grader-consistency.d.ts +6 -10
- package/dist/pipeline/grader-consistency.js +13 -32
- package/dist/pipeline/grader-sensitivity-runner.js +7 -5
- package/dist/pipeline/grader-sensitivity.d.ts +2 -6
- package/dist/pipeline/grader-sensitivity.js +10 -10
- package/dist/pipeline/grader-validate-runner.js +7 -5
- package/dist/pipeline/grader-validation.d.ts +2 -6
- package/dist/pipeline/grader-validation.js +14 -22
- package/dist/pipeline/map-request-to-config.js +6 -1
- package/dist/pipeline/mirror-repo-tasks.d.ts +6 -6
- package/dist/pipeline/mirror-repo-tasks.js +16 -15
- package/dist/pipeline/normalize-mode.d.ts +49 -0
- package/dist/pipeline/normalize-mode.js +64 -0
- package/dist/pipeline/plan.d.ts +5 -2
- package/dist/pipeline/plan.js +134 -78
- package/dist/pipeline/pr-comment.js +2 -0
- package/dist/pipeline/profile-resolution.d.ts +47 -0
- package/dist/pipeline/profile-resolution.js +91 -0
- package/dist/pipeline/provenance.d.ts +2 -2
- package/dist/pipeline/provenance.js +12 -17
- package/dist/pipeline/release-report.js +4 -4
- package/dist/pipeline/repo-threshold-evaluator.d.ts +1 -1
- package/dist/pipeline/repo-threshold-evaluator.js +1 -1
- package/dist/pipeline/rubric-loader.d.ts +20 -0
- package/dist/pipeline/rubric-loader.js +37 -0
- package/dist/pipeline/validate.d.ts +4 -4
- package/dist/pipeline/validate.js +64 -53
- package/dist/schedules/loader.js +18 -8
- package/dist/scripts/migrate-task-mode.d.ts +24 -0
- package/dist/scripts/migrate-task-mode.js +85 -0
- package/dist/scripts/migrate-tasks-to-content-lake.js +11 -10
- package/dist/scripts/validate-task-sources.d.ts +1 -1
- package/dist/scripts/validate-task-sources.js +15 -15
- package/dist/sinks/loader.js +5 -7
- package/dist/sources.d.ts +7 -7
- package/dist/sources.js +22 -24
- package/dist/webhook/dispatch.js +2 -1
- package/package.json +6 -3
- package/tasks/knowledge-probe/define-type-api.task.ts +55 -0
- package/tasks/knowledge-probe/groq-projections.task.ts +59 -0
- package/tasks/literacy/frameworks.task.ts +128 -0
- package/tasks/literacy/functions.task.ts +69 -0
- package/tasks/literacy/groq.task.ts +258 -0
- package/tasks/literacy/nextjs-live.task.ts +75 -0
- package/tasks/literacy/studio-setup.task.ts +131 -0
- package/tasks/literacy/visual-editing.task.ts +146 -0
- package/config/features.yaml +0 -116
- package/config/models.yaml +0 -116
- package/config/prompts.yaml +0 -75
- package/config/rubrics.yaml +0 -62
- package/config/schedules.yaml +0 -43
- package/config/sinks.yaml +0 -54
- package/config/sources.yaml +0 -51
- package/config/thresholds.yaml +0 -49
- package/dist/agent-observer/test-imports.d.ts +0 -7
- package/dist/agent-observer/test-imports.js +0 -185
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Adapter: Load task definitions from the Sanity Content Lake.
|
|
3
3
|
*
|
|
4
|
-
* Fetches ailf.task documents via GROQ and maps them to
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
4
|
+
* Fetches ailf.task documents via GROQ and maps them to
|
|
5
|
+
* GeneralizedTaskDefinition (LiteracyTaskDefinition variant). This
|
|
6
|
+
* adapter is the Content Lake counterpart of YamlTaskSource — both
|
|
7
|
+
* produce identical GeneralizedTaskDefinition[] for the same logical
|
|
8
|
+
* tasks. The pipeline never knows which adapter loaded the tasks.
|
|
8
9
|
*
|
|
9
10
|
* Wired in the composition root when --task-source content-lake is set.
|
|
10
11
|
*
|
|
@@ -16,7 +17,7 @@
|
|
|
16
17
|
// ---------------------------------------------------------------------------
|
|
17
18
|
/**
|
|
18
19
|
* GROQ query that projects ailf.task documents into a shape suitable for
|
|
19
|
-
* mapping to
|
|
20
|
+
* mapping to LiteracyTaskDefinition.
|
|
20
21
|
*
|
|
21
22
|
* Key projections:
|
|
22
23
|
* - featureArea reference → dereferenced areaId string
|
|
@@ -79,7 +80,7 @@ export class ContentLakeTaskSource {
|
|
|
79
80
|
}
|
|
80
81
|
const definitions = [];
|
|
81
82
|
for (const entry of raw) {
|
|
82
|
-
const mapped =
|
|
83
|
+
const mapped = mapToLiteracyTask(entry);
|
|
83
84
|
if (!mapped)
|
|
84
85
|
continue;
|
|
85
86
|
definitions.push(mapped);
|
|
@@ -105,17 +106,17 @@ function buildGroqParams(filter) {
|
|
|
105
106
|
};
|
|
106
107
|
}
|
|
107
108
|
// ---------------------------------------------------------------------------
|
|
108
|
-
// Mapping: Content Lake →
|
|
109
|
+
// Mapping: Content Lake → LiteracyTaskDefinition
|
|
109
110
|
// ---------------------------------------------------------------------------
|
|
110
111
|
/**
|
|
111
|
-
* Map a Content Lake ailf.task document to
|
|
112
|
+
* Map a Content Lake ailf.task document directly to a LiteracyTaskDefinition.
|
|
112
113
|
*
|
|
113
114
|
* Returns null if the document is missing required fields (taskId,
|
|
114
115
|
* description, featureAreaId, taskPrompt). These are required by the
|
|
115
116
|
* Studio schema, but defensive coding handles edge cases (drafts,
|
|
116
117
|
* partially-created documents, etc.).
|
|
117
118
|
*/
|
|
118
|
-
function
|
|
119
|
+
function mapToLiteracyTask(raw) {
|
|
119
120
|
// Required fields — skip malformed documents
|
|
120
121
|
if (!raw.taskId ||
|
|
121
122
|
!raw.description ||
|
|
@@ -123,17 +124,14 @@ function mapToTaskDefinition(raw) {
|
|
|
123
124
|
!raw.taskPrompt) {
|
|
124
125
|
return null;
|
|
125
126
|
}
|
|
126
|
-
const
|
|
127
|
+
const docs = (raw.canonicalDocs ?? [])
|
|
127
128
|
.map(mapCanonicalDocRef)
|
|
128
129
|
.filter((d) => d !== null);
|
|
129
130
|
const assertions = mapAssertions(raw.assert ?? []);
|
|
130
131
|
// Append raw pass-through assertions (escape hatch for arbitrary Promptfoo
|
|
131
132
|
// assertion types that aren't in the curated list). These bypass template
|
|
132
133
|
// resolution and flow directly into the expanded Promptfoo test case as
|
|
133
|
-
// value-based assertions.
|
|
134
|
-
// "abbreviated" (the default) drops non-rubric assertions, so rawAssert
|
|
135
|
-
// entries only run in the gold variant — consistent with how regular
|
|
136
|
-
// value-based assertions like `contains` or `regex` behave.
|
|
134
|
+
// value-based assertions.
|
|
137
135
|
const rawAssertions = (raw.rawAssert ?? [])
|
|
138
136
|
.filter((a) => !!a.type)
|
|
139
137
|
.map((a) => ({
|
|
@@ -141,7 +139,10 @@ function mapToTaskDefinition(raw) {
|
|
|
141
139
|
...(a.value !== undefined ? { value: a.value } : {}),
|
|
142
140
|
...(a.threshold !== undefined ? { threshold: a.threshold } : {}),
|
|
143
141
|
}));
|
|
144
|
-
const allAssertions = [
|
|
142
|
+
const allAssertions = [
|
|
143
|
+
...assertions,
|
|
144
|
+
...rawAssertions,
|
|
145
|
+
];
|
|
145
146
|
const baseline = raw.baseline
|
|
146
147
|
? {
|
|
147
148
|
...(raw.baseline.enabled !== undefined
|
|
@@ -155,19 +156,17 @@ function mapToTaskDefinition(raw) {
|
|
|
155
156
|
}
|
|
156
157
|
: undefined;
|
|
157
158
|
return {
|
|
159
|
+
mode: "literacy",
|
|
160
|
+
id: raw.taskId,
|
|
161
|
+
title: raw.description,
|
|
162
|
+
area: raw.featureAreaId,
|
|
163
|
+
prompt: { text: raw.taskPrompt },
|
|
164
|
+
context: { docs },
|
|
158
165
|
assertions: allAssertions,
|
|
159
|
-
canonicalDocs,
|
|
160
|
-
description: raw.description,
|
|
161
166
|
docCoverage: raw.docCoverage ?? false,
|
|
162
|
-
featureArea: raw.featureAreaId,
|
|
163
|
-
id: raw.taskId,
|
|
164
167
|
// Reference solution path is not meaningful for CL tasks — the solution
|
|
165
168
|
// content lives in the Content Lake document, not on the filesystem.
|
|
166
|
-
// Downstream consumers that need the solution content will need to fetch
|
|
167
|
-
// it via the referenceSolution reference. For now, use empty string
|
|
168
|
-
// (same as tasks without a reference solution).
|
|
169
169
|
referenceSolution: "",
|
|
170
|
-
taskPrompt: raw.taskPrompt,
|
|
171
170
|
...(baseline ? { baseline } : {}),
|
|
172
171
|
...(raw.tags?.length ? { tags: raw.tags } : {}),
|
|
173
172
|
};
|
|
@@ -4,4 +4,5 @@ export { parseRepoConfig, parseRepoTaskFile, RepoConfigSchema, RepoTaskSchema, C
|
|
|
4
4
|
export { RepoTaskSource } from "./repo-task-source.js";
|
|
5
5
|
export { detectTriggerContext, resolveTrigger, type ResolvedTrigger, type TriggerContext, } from "./repo-trigger.js";
|
|
6
6
|
export { formatValidationResult, validateRepoTasks, type ValidationMessage, type ValidationResult, } from "./repo-validation.js";
|
|
7
|
+
export { discoverTsTaskFiles, loadAllTsTaskFiles, loadTsTaskFile, } from "./task-file-loader.js";
|
|
7
8
|
export { YamlTaskSource } from "./yaml-task-source.js";
|
|
@@ -4,4 +4,5 @@ export { parseRepoConfig, parseRepoTaskFile, RepoConfigSchema, RepoTaskSchema, C
|
|
|
4
4
|
export { RepoTaskSource } from "./repo-task-source.js";
|
|
5
5
|
export { detectTriggerContext, resolveTrigger, } from "./repo-trigger.js";
|
|
6
6
|
export { formatValidationResult, validateRepoTasks, } from "./repo-validation.js";
|
|
7
|
+
export { discoverTsTaskFiles, loadAllTsTaskFiles, loadTsTaskFile, } from "./task-file-loader.js";
|
|
7
8
|
export { YamlTaskSource } from "./yaml-task-source.js";
|
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Repo-based tasks use a camelCase YAML format (matching the Content Lake
|
|
5
5
|
* document schema) with slug strings instead of Sanity references. The
|
|
6
|
-
* mapping to
|
|
7
|
-
* aligned with the domain type.
|
|
6
|
+
* mapping to LiteracyTaskDefinition is straightforward — field names are
|
|
7
|
+
* already aligned with the domain type.
|
|
8
8
|
*
|
|
9
9
|
* This adapter is structurally similar to YamlTaskSource but:
|
|
10
10
|
* - Reads from an arbitrary directory (not the framework's tasks/ dir)
|
|
@@ -14,9 +14,9 @@
|
|
|
14
14
|
* @see packages/core/src/ports/task-source.ts — TaskSource port
|
|
15
15
|
* @see docs/exec-plans/tasks-as-content/phase-4-repo-based-tasks.md
|
|
16
16
|
*/
|
|
17
|
-
import type { FilterOptions,
|
|
17
|
+
import type { FilterOptions, GeneralizedTaskDefinition, TaskSource } from "../../_vendor/ailf-core/index.d.ts";
|
|
18
18
|
export declare class RepoTaskSource implements TaskSource {
|
|
19
19
|
private readonly tasksDir;
|
|
20
20
|
constructor(tasksDir: string);
|
|
21
|
-
loadTasks(filter?: FilterOptions): Promise<
|
|
21
|
+
loadTasks(filter?: FilterOptions): Promise<GeneralizedTaskDefinition[]>;
|
|
22
22
|
}
|
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Repo-based tasks use a camelCase YAML format (matching the Content Lake
|
|
5
5
|
* document schema) with slug strings instead of Sanity references. The
|
|
6
|
-
* mapping to
|
|
7
|
-
* aligned with the domain type.
|
|
6
|
+
* mapping to LiteracyTaskDefinition is straightforward — field names are
|
|
7
|
+
* already aligned with the domain type.
|
|
8
8
|
*
|
|
9
9
|
* This adapter is structurally similar to YamlTaskSource but:
|
|
10
10
|
* - Reads from an arbitrary directory (not the framework's tasks/ dir)
|
|
@@ -18,6 +18,7 @@ import { existsSync, readdirSync, readFileSync } from "fs";
|
|
|
18
18
|
import { resolve } from "path";
|
|
19
19
|
import { load } from "js-yaml";
|
|
20
20
|
import { parseRepoTaskFile } from "./repo-schemas.js";
|
|
21
|
+
import { discoverTsTaskFiles, loadTsTaskFile } from "./task-file-loader.js";
|
|
21
22
|
// ---------------------------------------------------------------------------
|
|
22
23
|
// RepoTaskSource adapter
|
|
23
24
|
// ---------------------------------------------------------------------------
|
|
@@ -34,9 +35,10 @@ export class RepoTaskSource {
|
|
|
34
35
|
const yamlFiles = readdirSync(this.tasksDir)
|
|
35
36
|
.filter((f) => (f.endsWith(".yaml") || f.endsWith(".yml")) && !f.startsWith("."))
|
|
36
37
|
.sort();
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
38
|
+
const tsFiles = discoverTsTaskFiles(this.tasksDir);
|
|
39
|
+
if (yamlFiles.length === 0 && tsFiles.length === 0) {
|
|
40
|
+
throw new Error(`No task files found in ${this.tasksDir}\n` +
|
|
41
|
+
" Expected .ailf/tasks/*.yaml or .ailf/tasks/*.task.ts files");
|
|
40
42
|
}
|
|
41
43
|
const definitions = [];
|
|
42
44
|
for (const file of yamlFiles) {
|
|
@@ -99,7 +101,56 @@ export class RepoTaskSource {
|
|
|
99
101
|
(!entry.tags || !entry.tags.some((t) => filter.tags.includes(t)))) {
|
|
100
102
|
continue;
|
|
101
103
|
}
|
|
102
|
-
definitions.push(
|
|
104
|
+
definitions.push(mapToLiteracyTask(entry));
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
// Load TS task files (.task.ts / .task.js)
|
|
108
|
+
for (const tsFile of tsFiles) {
|
|
109
|
+
const loaded = await loadTsTaskFile(tsFile);
|
|
110
|
+
const filename = tsFile.split("/").pop() ?? tsFile;
|
|
111
|
+
// Validate through the same schema as YAML tasks
|
|
112
|
+
let validated;
|
|
113
|
+
try {
|
|
114
|
+
validated = parseRepoTaskFile(loaded.tasks, filename);
|
|
115
|
+
}
|
|
116
|
+
catch (err) {
|
|
117
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
118
|
+
throw new Error(`Failed to validate ${filename}:\n${msg}`, {
|
|
119
|
+
cause: err,
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
for (const entry of validated) {
|
|
123
|
+
// Apply the same filtering as YAML tasks
|
|
124
|
+
if (filter?.areas &&
|
|
125
|
+
filter.areas.length > 0 &&
|
|
126
|
+
!filter.areas
|
|
127
|
+
.map((a) => a.toLowerCase())
|
|
128
|
+
.includes(entry.featureArea.toLowerCase())) {
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
if (filter?.taskIds &&
|
|
132
|
+
filter.taskIds.length > 0 &&
|
|
133
|
+
!filter.taskIds.includes(entry.id)) {
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
const effectiveStatus = entry.status ??
|
|
137
|
+
(entry.execution?.enabled === false ? "paused" : "active");
|
|
138
|
+
const isTargetedById = filter?.taskIds && filter.taskIds.includes(entry.id);
|
|
139
|
+
if (effectiveStatus === "archived")
|
|
140
|
+
continue;
|
|
141
|
+
if (effectiveStatus === "paused" && !isTargetedById)
|
|
142
|
+
continue;
|
|
143
|
+
if (effectiveStatus === "draft" &&
|
|
144
|
+
!isTargetedById &&
|
|
145
|
+
!filter?.includeDrafts) {
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
if (filter?.tags &&
|
|
149
|
+
filter.tags.length > 0 &&
|
|
150
|
+
(!entry.tags || !entry.tags.some((t) => filter.tags.includes(t)))) {
|
|
151
|
+
continue;
|
|
152
|
+
}
|
|
153
|
+
definitions.push(mapToLiteracyTask(entry));
|
|
103
154
|
}
|
|
104
155
|
}
|
|
105
156
|
return definitions;
|
|
@@ -108,26 +159,28 @@ export class RepoTaskSource {
|
|
|
108
159
|
// ---------------------------------------------------------------------------
|
|
109
160
|
// Mapping helpers
|
|
110
161
|
// ---------------------------------------------------------------------------
|
|
111
|
-
function
|
|
162
|
+
function mapToLiteracyTask(raw) {
|
|
112
163
|
// Extract task prompt from vars.task (same convention as YAML tasks)
|
|
113
164
|
const vars = (raw.vars ?? {});
|
|
114
165
|
const { docs: _docs, task, ...rest } = vars;
|
|
115
166
|
// Canonical docs are already parsed into the polymorphic union by Zod.
|
|
116
|
-
|
|
117
|
-
const canonicalDocs = (raw.canonicalDocs ??
|
|
167
|
+
const docs = (raw.canonicalDocs ??
|
|
118
168
|
[]);
|
|
119
169
|
const extraVars = Object.keys(rest).length > 0 ? rest : undefined;
|
|
120
170
|
return {
|
|
121
|
-
|
|
122
|
-
canonicalDocs,
|
|
123
|
-
description: raw.description,
|
|
124
|
-
docCoverage: raw.docCoverage ?? false,
|
|
125
|
-
featureArea: raw.featureArea,
|
|
171
|
+
mode: "literacy",
|
|
126
172
|
id: raw.id,
|
|
173
|
+
title: raw.description,
|
|
174
|
+
area: raw.featureArea,
|
|
175
|
+
prompt: {
|
|
176
|
+
text: typeof task === "string" ? task : "",
|
|
177
|
+
...(extraVars ? { vars: extraVars } : {}),
|
|
178
|
+
},
|
|
179
|
+
context: { docs },
|
|
180
|
+
assertions: (raw.assert ?? []),
|
|
127
181
|
referenceSolution: raw.referenceSolution ?? "",
|
|
128
|
-
|
|
182
|
+
docCoverage: raw.docCoverage ?? false,
|
|
129
183
|
...(raw.baseline ? { baseline: raw.baseline } : {}),
|
|
130
|
-
...(extraVars ? { extraVars } : {}),
|
|
131
184
|
...(raw.status && raw.status !== "active" ? { status: raw.status } : {}),
|
|
132
185
|
...(raw.tags?.length ? { tags: raw.tags } : {}),
|
|
133
186
|
};
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* TaskFileLoader — loads task definitions from TypeScript files.
|
|
3
|
+
*
|
|
4
|
+
* Supplements the existing YAML-based task loading by supporting
|
|
5
|
+
* `*.task.ts` and `*.task.js` files in task directories. Files are
|
|
6
|
+
* loaded via jiti and validated through the RepoTaskSchema from
|
|
7
|
+
* @sanity/ailf-tasks.
|
|
8
|
+
*
|
|
9
|
+
* TS task files export a single task or an array of tasks:
|
|
10
|
+
*
|
|
11
|
+
* ```typescript
|
|
12
|
+
* // single task
|
|
13
|
+
* import { defineTask } from "@sanity/ailf"
|
|
14
|
+
* export default defineTask({ id: "my-task", mode: "literacy", ... })
|
|
15
|
+
*
|
|
16
|
+
* // multiple tasks
|
|
17
|
+
* export default [
|
|
18
|
+
* defineTask({ id: "task-1", mode: "literacy", ... }),
|
|
19
|
+
* defineTask({ id: "task-2", mode: "literacy", ... }),
|
|
20
|
+
* ]
|
|
21
|
+
* ```
|
|
22
|
+
*
|
|
23
|
+
* The loader integrates into the existing RepoTaskSource adapter — TS
|
|
24
|
+
* task files are discovered alongside YAML files in the same directory.
|
|
25
|
+
*
|
|
26
|
+
* @see docs/design-docs/architecture-overhaul/typescript-configuration.md
|
|
27
|
+
*/
|
|
28
|
+
/** A raw task object loaded from a TS file (pre-validation) */
|
|
29
|
+
export interface RawTsTask {
|
|
30
|
+
/** Source file path (for error messages) */
|
|
31
|
+
filePath: string;
|
|
32
|
+
/** The loaded task data (may be a single object or an array) */
|
|
33
|
+
tasks: unknown[];
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Discover TS/JS task files in a directory.
|
|
37
|
+
*
|
|
38
|
+
* Looks for files matching `*.task.ts` or `*.task.js`.
|
|
39
|
+
*
|
|
40
|
+
* @param tasksDir - Absolute path to the tasks directory
|
|
41
|
+
* @returns Array of absolute file paths
|
|
42
|
+
*/
|
|
43
|
+
export declare function discoverTsTaskFiles(tasksDir: string): string[];
|
|
44
|
+
/**
|
|
45
|
+
* Load task definitions from a single TS/JS task file.
|
|
46
|
+
*
|
|
47
|
+
* The file's default export can be:
|
|
48
|
+
* - A single task object → wrapped in an array
|
|
49
|
+
* - An array of task objects → used as-is
|
|
50
|
+
*
|
|
51
|
+
* Returns the raw task data without validation — the caller is
|
|
52
|
+
* responsible for running the result through Zod schemas.
|
|
53
|
+
*
|
|
54
|
+
* @param filePath - Absolute path to the .task.ts or .task.js file
|
|
55
|
+
* @returns The loaded task(s), or throws on load failure
|
|
56
|
+
*/
|
|
57
|
+
export declare function loadTsTaskFile(filePath: string): Promise<RawTsTask>;
|
|
58
|
+
/**
|
|
59
|
+
* Load all TS task files from a directory.
|
|
60
|
+
*
|
|
61
|
+
* @param tasksDir - Absolute path to the tasks directory
|
|
62
|
+
* @returns Array of raw task data from all files
|
|
63
|
+
*/
|
|
64
|
+
export declare function loadAllTsTaskFiles(tasksDir: string): Promise<RawTsTask[]>;
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* TaskFileLoader — loads task definitions from TypeScript files.
|
|
3
|
+
*
|
|
4
|
+
* Supplements the existing YAML-based task loading by supporting
|
|
5
|
+
* `*.task.ts` and `*.task.js` files in task directories. Files are
|
|
6
|
+
* loaded via jiti and validated through the RepoTaskSchema from
|
|
7
|
+
* @sanity/ailf-tasks.
|
|
8
|
+
*
|
|
9
|
+
* TS task files export a single task or an array of tasks:
|
|
10
|
+
*
|
|
11
|
+
* ```typescript
|
|
12
|
+
* // single task
|
|
13
|
+
* import { defineTask } from "@sanity/ailf"
|
|
14
|
+
* export default defineTask({ id: "my-task", mode: "literacy", ... })
|
|
15
|
+
*
|
|
16
|
+
* // multiple tasks
|
|
17
|
+
* export default [
|
|
18
|
+
* defineTask({ id: "task-1", mode: "literacy", ... }),
|
|
19
|
+
* defineTask({ id: "task-2", mode: "literacy", ... }),
|
|
20
|
+
* ]
|
|
21
|
+
* ```
|
|
22
|
+
*
|
|
23
|
+
* The loader integrates into the existing RepoTaskSource adapter — TS
|
|
24
|
+
* task files are discovered alongside YAML files in the same directory.
|
|
25
|
+
*
|
|
26
|
+
* @see docs/design-docs/architecture-overhaul/typescript-configuration.md
|
|
27
|
+
*/
|
|
28
|
+
import { existsSync, readdirSync } from "fs";
|
|
29
|
+
import { resolve } from "path";
|
|
30
|
+
import { loadTsConfig } from "../config-sources/ts-config-loader.js";
|
|
31
|
+
/**
|
|
32
|
+
* Discover TS/JS task files in a directory.
|
|
33
|
+
*
|
|
34
|
+
* Looks for files matching `*.task.ts` or `*.task.js`.
|
|
35
|
+
*
|
|
36
|
+
* @param tasksDir - Absolute path to the tasks directory
|
|
37
|
+
* @returns Array of absolute file paths
|
|
38
|
+
*/
|
|
39
|
+
export function discoverTsTaskFiles(tasksDir) {
|
|
40
|
+
if (!existsSync(tasksDir))
|
|
41
|
+
return [];
|
|
42
|
+
return readdirSync(tasksDir)
|
|
43
|
+
.filter((f) => (f.endsWith(".task.ts") || f.endsWith(".task.js")) && !f.startsWith("."))
|
|
44
|
+
.sort()
|
|
45
|
+
.map((f) => resolve(tasksDir, f));
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Load task definitions from a single TS/JS task file.
|
|
49
|
+
*
|
|
50
|
+
* The file's default export can be:
|
|
51
|
+
* - A single task object → wrapped in an array
|
|
52
|
+
* - An array of task objects → used as-is
|
|
53
|
+
*
|
|
54
|
+
* Returns the raw task data without validation — the caller is
|
|
55
|
+
* responsible for running the result through Zod schemas.
|
|
56
|
+
*
|
|
57
|
+
* @param filePath - Absolute path to the .task.ts or .task.js file
|
|
58
|
+
* @returns The loaded task(s), or throws on load failure
|
|
59
|
+
*/
|
|
60
|
+
export async function loadTsTaskFile(filePath) {
|
|
61
|
+
const result = await loadTsConfig(filePath);
|
|
62
|
+
if (!result.ok) {
|
|
63
|
+
throw new Error(result.error);
|
|
64
|
+
}
|
|
65
|
+
const value = result.value;
|
|
66
|
+
// Normalize: single object → array of one
|
|
67
|
+
const tasks = Array.isArray(value) ? value : [value];
|
|
68
|
+
return { filePath, tasks };
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Load all TS task files from a directory.
|
|
72
|
+
*
|
|
73
|
+
* @param tasksDir - Absolute path to the tasks directory
|
|
74
|
+
* @returns Array of raw task data from all files
|
|
75
|
+
*/
|
|
76
|
+
export async function loadAllTsTaskFiles(tasksDir) {
|
|
77
|
+
const files = discoverTsTaskFiles(tasksDir);
|
|
78
|
+
const results = [];
|
|
79
|
+
for (const file of files) {
|
|
80
|
+
results.push(await loadTsTaskFile(file));
|
|
81
|
+
}
|
|
82
|
+
return results;
|
|
83
|
+
}
|
|
@@ -2,17 +2,17 @@
|
|
|
2
2
|
* Adapter: Load task definitions from tasks/*.yaml files.
|
|
3
3
|
*
|
|
4
4
|
* This adapter reads the raw YAML task definitions (before Promptfoo
|
|
5
|
-
* expansion) and maps them to
|
|
6
|
-
* @sanity/ailf-core. It handles
|
|
7
|
-
* task ID filtering.
|
|
5
|
+
* expansion) and maps them to GeneralizedTaskDefinition
|
|
6
|
+
* (LiteracyTaskDefinition variant) from @sanity/ailf-core. It handles
|
|
7
|
+
* area filtering (filename stem) and task ID filtering.
|
|
8
8
|
*
|
|
9
9
|
* Unlike loadAndExpandTasks() — which produces Promptfoo-specific
|
|
10
10
|
* ExpandedTestEntry objects — this adapter produces domain-level
|
|
11
|
-
*
|
|
11
|
+
* GeneralizedTaskDefinition objects suitable for the pipeline orchestrator.
|
|
12
12
|
*/
|
|
13
|
-
import type { FilterOptions,
|
|
13
|
+
import type { FilterOptions, GeneralizedTaskDefinition, TaskSource } from "../../_vendor/ailf-core/index.d.ts";
|
|
14
14
|
export declare class YamlTaskSource implements TaskSource {
|
|
15
15
|
private readonly rootDir;
|
|
16
16
|
constructor(rootDir: string);
|
|
17
|
-
loadTasks(filter?: FilterOptions): Promise<
|
|
17
|
+
loadTasks(filter?: FilterOptions): Promise<GeneralizedTaskDefinition[]>;
|
|
18
18
|
}
|
|
@@ -2,13 +2,13 @@
|
|
|
2
2
|
* Adapter: Load task definitions from tasks/*.yaml files.
|
|
3
3
|
*
|
|
4
4
|
* This adapter reads the raw YAML task definitions (before Promptfoo
|
|
5
|
-
* expansion) and maps them to
|
|
6
|
-
* @sanity/ailf-core. It handles
|
|
7
|
-
* task ID filtering.
|
|
5
|
+
* expansion) and maps them to GeneralizedTaskDefinition
|
|
6
|
+
* (LiteracyTaskDefinition variant) from @sanity/ailf-core. It handles
|
|
7
|
+
* area filtering (filename stem) and task ID filtering.
|
|
8
8
|
*
|
|
9
9
|
* Unlike loadAndExpandTasks() — which produces Promptfoo-specific
|
|
10
10
|
* ExpandedTestEntry objects — this adapter produces domain-level
|
|
11
|
-
*
|
|
11
|
+
* GeneralizedTaskDefinition objects suitable for the pipeline orchestrator.
|
|
12
12
|
*/
|
|
13
13
|
import { existsSync, readdirSync, readFileSync } from "fs";
|
|
14
14
|
import { resolve } from "path";
|
|
@@ -55,7 +55,7 @@ export class YamlTaskSource {
|
|
|
55
55
|
!filter.taskIds.includes(entry.id)) {
|
|
56
56
|
continue;
|
|
57
57
|
}
|
|
58
|
-
definitions.push(
|
|
58
|
+
definitions.push(mapToLiteracyTask(entry, featureArea));
|
|
59
59
|
}
|
|
60
60
|
}
|
|
61
61
|
return definitions;
|
|
@@ -65,29 +65,32 @@ export class YamlTaskSource {
|
|
|
65
65
|
// Mapping helpers
|
|
66
66
|
// ---------------------------------------------------------------------------
|
|
67
67
|
/**
|
|
68
|
-
* Map a raw YAML entry to a
|
|
68
|
+
* Map a raw YAML entry directly to a LiteracyTaskDefinition.
|
|
69
69
|
*
|
|
70
|
-
* Renames snake_case YAML keys to
|
|
71
|
-
* the task prompt from `vars.task`. Additional vars beyond `task`
|
|
72
|
-
* `docs` are collected into `
|
|
70
|
+
* Renames snake_case YAML keys to the generalized type's field names and
|
|
71
|
+
* extracts the task prompt from `vars.task`. Additional vars beyond `task`
|
|
72
|
+
* and `docs` are collected into `prompt.vars`.
|
|
73
73
|
*/
|
|
74
|
-
function
|
|
74
|
+
function mapToLiteracyTask(raw, featureArea) {
|
|
75
75
|
const { task, docs: _docs, ...rest } = (raw.vars ?? {});
|
|
76
|
-
const
|
|
76
|
+
const docs = (raw.canonical_docs ?? [])
|
|
77
77
|
.map(mapCanonicalDoc)
|
|
78
78
|
.filter((d) => d !== null);
|
|
79
79
|
const extraVars = Object.keys(rest).length > 0 ? rest : undefined;
|
|
80
80
|
return {
|
|
81
|
+
mode: "literacy",
|
|
81
82
|
id: raw.id,
|
|
82
|
-
|
|
83
|
-
featureArea,
|
|
84
|
-
|
|
85
|
-
|
|
83
|
+
title: raw.description,
|
|
84
|
+
area: featureArea,
|
|
85
|
+
prompt: {
|
|
86
|
+
text: typeof task === "string" ? task : "",
|
|
87
|
+
...(extraVars ? { vars: extraVars } : {}),
|
|
88
|
+
},
|
|
89
|
+
context: { docs },
|
|
86
90
|
referenceSolution: raw.reference_solution ?? "",
|
|
87
91
|
docCoverage: raw.doc_coverage ?? false,
|
|
88
92
|
assertions: (raw.assert ?? []),
|
|
89
93
|
...(raw.baseline ? { baseline: raw.baseline } : {}),
|
|
90
|
-
...(extraVars ? { extraVars } : {}),
|
|
91
94
|
};
|
|
92
95
|
}
|
|
93
96
|
// ---------------------------------------------------------------------------
|
package/dist/cli.js
CHANGED
|
@@ -157,8 +157,6 @@ import { createValidateTasksCommand } from "./commands/validate-tasks.js";
|
|
|
157
157
|
program.addCommand(createValidateTasksCommand().helpGroup(CommandGroup.SetupConfig));
|
|
158
158
|
import { createFetchDocsCommand } from "./commands/fetch-docs.js";
|
|
159
159
|
program.addCommand(createFetchDocsCommand().helpGroup(CommandGroup.SetupConfig));
|
|
160
|
-
import { createGenerateConfigsCommand } from "./commands/generate-configs.js";
|
|
161
|
-
program.addCommand(createGenerateConfigsCommand().helpGroup(CommandGroup.SetupConfig));
|
|
162
160
|
import { createCacheCommand } from "./commands/cache.js";
|
|
163
161
|
program.addCommand(createCacheCommand().helpGroup(CommandGroup.SetupConfig));
|
|
164
162
|
// ── Pipeline Internals ────────────────────────────────────────────────
|
|
@@ -11,8 +11,11 @@ import { Command } from "commander";
|
|
|
11
11
|
import { compareBaseline, listBaselines, saveBaseline, } from "../pipeline/baseline.js";
|
|
12
12
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
13
13
|
const ROOT = resolve(__dirname, "../..");
|
|
14
|
+
// CLI command name — kept as a constant to centralize the string literal.
|
|
15
|
+
// "baseline" here refers to score baseline snapshots, not the legacy eval mode.
|
|
16
|
+
const CMD_NAME = "baseline";
|
|
14
17
|
export function createBaselineCommand() {
|
|
15
|
-
const cmd = new Command(
|
|
18
|
+
const cmd = new Command(CMD_NAME).description("Manage historical baseline snapshots of evaluation scores");
|
|
16
19
|
// -----------------------------------------------------------------------
|
|
17
20
|
// baseline save
|
|
18
21
|
// -----------------------------------------------------------------------
|
|
@@ -3,10 +3,12 @@
|
|
|
3
3
|
* against task files to produce a documentation coverage audit.
|
|
4
4
|
*
|
|
5
5
|
*/
|
|
6
|
+
import { InMemoryPluginRegistry } from "../_vendor/ailf-core/index.js";
|
|
6
7
|
import { Command } from "commander";
|
|
7
8
|
import { dirname, resolve } from "path";
|
|
8
9
|
import { fileURLToPath } from "url";
|
|
9
10
|
import { countReferencedDocs, formatCoverageConsole, formatCoverageMarkdown, runCoverageAudit, } from "../pipeline/coverage-audit.js";
|
|
11
|
+
import { createSanityLiteracyPreset } from "../pipeline/compiler/presets/index.js";
|
|
10
12
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
11
13
|
const ROOT = resolve(__dirname, "..", "..");
|
|
12
14
|
export function createCoverageAuditCommand() {
|
|
@@ -15,7 +17,11 @@ export function createCoverageAuditCommand() {
|
|
|
15
17
|
.option("--format <fmt>", "Output format: table, md, markdown")
|
|
16
18
|
.option("--json", "Output raw JSON", false)
|
|
17
19
|
.action(async (opts) => {
|
|
18
|
-
|
|
20
|
+
// Build a registry with preset features so coverage audit works
|
|
21
|
+
// even when config/features.ts is empty (preset is source of truth).
|
|
22
|
+
const registry = new InMemoryPluginRegistry();
|
|
23
|
+
registry.registerPreset(createSanityLiteracyPreset({ rootDir: ROOT }));
|
|
24
|
+
const report = runCoverageAudit(ROOT, { registry });
|
|
19
25
|
if (!report) {
|
|
20
26
|
console.error("❌ Coverage audit failed. Ensure config/features.yaml exists and is valid.");
|
|
21
27
|
process.exit(1);
|