@sanity/ailf 3.6.0 → 3.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config/airbyte/ai_literacy_framework.connector.yaml +1 -1
- package/config/thresholds.ts +3 -3
- package/dist/_vendor/ailf-core/examples/index.d.ts +2 -2
- package/dist/_vendor/ailf-core/examples/index.js +2 -2
- package/dist/_vendor/ailf-core/ports/context.d.ts +0 -4
- package/dist/_vendor/ailf-core/schemas/eval-config.d.ts +38 -12
- package/dist/_vendor/ailf-core/schemas/eval-config.js +102 -22
- package/dist/_vendor/ailf-core/schemas/pipeline-request.d.ts +4 -6
- package/dist/_vendor/ailf-core/schemas/pipeline-request.js +1 -3
- package/dist/_vendor/ailf-core/schemas/schedules.d.ts +2 -2
- package/dist/_vendor/ailf-shared/run-classification.d.ts +2 -2
- package/dist/_vendor/ailf-shared/run-classification.js +1 -1
- package/dist/_vendor/ailf-shared/run-context.d.ts +1 -1
- package/dist/adapters/api-client/build-request.d.ts +0 -2
- package/dist/adapters/api-client/build-request.js +2 -6
- package/dist/adapters/config-sources/cli-config-adapter.d.ts +1 -1
- package/dist/adapters/config-sources/file-config-adapter.d.ts +1 -1
- package/dist/adapters/config-sources/file-config-adapter.js +38 -12
- package/dist/adapters/task-sources/content-lake-task-source.js +17 -0
- package/dist/adapters/task-sources/index.d.ts +1 -1
- package/dist/adapters/task-sources/index.js +1 -1
- package/dist/adapters/task-sources/repo-schemas.d.ts +154 -0
- package/dist/adapters/task-sources/repo-schemas.js +137 -0
- package/dist/cli.d.ts +2 -2
- package/dist/cli.js +134 -38
- package/dist/commands/agent-report.js +1 -1
- package/dist/commands/calculate-scores.js +0 -2
- package/dist/commands/check-staleness.js +1 -1
- package/dist/commands/chronic-failures.js +4 -4
- package/dist/commands/coverage-audit.js +6 -7
- package/dist/commands/discovery-report.js +16 -4
- package/dist/commands/eval.d.ts +1 -1
- package/dist/commands/eval.js +1 -1
- package/dist/commands/explain-handler.d.ts +1 -1
- package/dist/commands/explain-handler.js +13 -44
- package/dist/commands/fetch-docs.js +0 -2
- package/dist/commands/generate-configs.js +0 -2
- package/dist/commands/grader/index.js +3 -3
- package/dist/commands/init.d.ts +2 -2
- package/dist/commands/init.js +10 -9
- package/dist/commands/interactive.d.ts +1 -1
- package/dist/commands/interactive.js +8 -8
- package/dist/commands/pipeline-action.d.ts +1 -3
- package/dist/commands/pipeline-action.js +174 -140
- package/dist/commands/pr-comment.js +1 -3
- package/dist/commands/publish.d.ts +1 -1
- package/dist/commands/publish.js +2 -4
- package/dist/commands/readiness-report.js +17 -8
- package/dist/commands/remote-pipeline.d.ts +1 -1
- package/dist/commands/remote-pipeline.js +1 -3
- package/dist/commands/run.d.ts +64 -0
- package/dist/commands/{pipeline.js → run.js} +19 -30
- package/dist/commands/shared/help.js +4 -4
- package/dist/commands/shared/options.d.ts +29 -3
- package/dist/commands/shared/options.js +37 -13
- package/dist/commands/validate-tasks.js +1 -1
- package/dist/commands/validate.d.ts +1 -1
- package/dist/commands/validate.js +2 -2
- package/dist/commands/weekly-digest.js +3 -3
- package/dist/config/thresholds.ts +3 -3
- package/dist/orchestration/build-app-context.js +0 -2
- package/dist/orchestration/build-step-sequence.js +1 -11
- package/dist/orchestration/steps/fetch-docs-step.js +1 -1
- package/dist/orchestration/steps/index.d.ts +0 -2
- package/dist/orchestration/steps/index.js +0 -2
- package/dist/orchestration/steps/run-eval-step.js +1 -1
- package/dist/pipeline/cache.d.ts +1 -1
- package/dist/pipeline/map-request-to-config.js +0 -2
- package/dist/pipeline/plan.d.ts +2 -4
- package/dist/pipeline/plan.js +4 -32
- package/dist/pipeline/run-context.d.ts +1 -1
- package/dist/pipeline/run-context.js +4 -4
- package/dist/pipeline/validate.d.ts +1 -1
- package/dist/pipeline/validate.js +1 -1
- package/package.json +7 -7
- package/dist/commands/pipeline.d.ts +0 -77
- package/dist/orchestration/steps/discovery-report-step.d.ts +0 -13
- package/dist/orchestration/steps/discovery-report-step.js +0 -62
- package/dist/orchestration/steps/readiness-step.d.ts +0 -13
- package/dist/orchestration/steps/readiness-step.js +0 -98
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* FileConfigAdapter — resolves pipeline config from a local config file.
|
|
3
3
|
*
|
|
4
|
-
* Enables `ailf
|
|
4
|
+
* Enables `ailf run --config <path>` to load all pipeline options
|
|
5
5
|
* from a file instead of CLI flags. Supports multiple formats in
|
|
6
6
|
* priority order:
|
|
7
7
|
*
|
|
@@ -91,9 +91,16 @@ function mapEvalConfigToResolvedConfig(config, rootDir) {
|
|
|
91
91
|
}
|
|
92
92
|
// Normalize legacy mode names (e.g., "full" → literacy + variant)
|
|
93
93
|
const normalized = normalizeMode(config.mode ?? "literacy");
|
|
94
|
+
// Output directory (W0077 Phase 6c) — resolve `output.dir` relative to the
|
|
95
|
+
// rootDir (the caller's workspace, set by the FileConfigAdapter caller).
|
|
96
|
+
// When unset, fall back to <rootDir>/.ailf/results/latest/ to mirror the
|
|
97
|
+
// CLI's default. This matches `resolveOutputDir` for the auto-load path.
|
|
98
|
+
const outputDir = config.output?.dir
|
|
99
|
+
? resolve(rootDir, config.output.dir)
|
|
100
|
+
: resolve(rootDir, ".ailf", "results", "latest");
|
|
94
101
|
return {
|
|
95
102
|
rootDir,
|
|
96
|
-
outputDir
|
|
103
|
+
outputDir,
|
|
97
104
|
mode: normalized.mode,
|
|
98
105
|
variant: normalized.variant,
|
|
99
106
|
noAutoScope: config.noAutoScope ?? false,
|
|
@@ -107,21 +114,40 @@ function mapEvalConfigToResolvedConfig(config, rootDir) {
|
|
|
107
114
|
compareEnabled: config.compare ?? false,
|
|
108
115
|
compareThreshold: config.compareThreshold,
|
|
109
116
|
compareBaseline: config.compareBaseline,
|
|
110
|
-
gapAnalysisEnabled: config.gapAnalysis ?? true,
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
117
|
+
gapAnalysisEnabled: config.execution?.gapAnalysis ?? true,
|
|
118
|
+
// W0077 Phase 4 — `publish` is now a policy object. Map the auto value
|
|
119
|
+
// directly to a boolean for the file-config path; the runtime
|
|
120
|
+
// smart-default logic in pipeline-action.ts isn't relevant here because
|
|
121
|
+
// the user has explicitly handed us a config file.
|
|
122
|
+
publishEnabled: config.publish?.auto === "never"
|
|
123
|
+
? false
|
|
124
|
+
: config.publish?.auto !== undefined,
|
|
125
|
+
publishTag: config.publish?.tag,
|
|
115
126
|
noCache: config.noCache ?? false,
|
|
116
127
|
noRemoteCache: config.noRemoteCache ?? false,
|
|
117
|
-
graderReplications: config.graderReplications,
|
|
128
|
+
graderReplications: config.execution?.graderReplications,
|
|
118
129
|
urls: config.urls,
|
|
119
|
-
headers: config.headers,
|
|
120
|
-
allowedOrigins: config.allowedOrigins,
|
|
130
|
+
headers: config.agentic?.headers,
|
|
131
|
+
allowedOrigins: config.agentic?.allowedOrigins,
|
|
121
132
|
searchMode: config.searchMode ?? "open",
|
|
122
|
-
concurrency: config.concurrency,
|
|
133
|
+
concurrency: config.execution?.concurrency,
|
|
123
134
|
remote: false,
|
|
124
|
-
apiUrl: "https://ailf-api.sanity.build",
|
|
135
|
+
apiUrl: config.execution?.apiUrl ?? "https://ailf-api.sanity.build",
|
|
136
|
+
// W0077 Phase 6g — artifact writer settings. `enabled: false` flips
|
|
137
|
+
// `artifactsDisabled` so composition-root selects the NoOp writer.
|
|
138
|
+
artifactsDisabled: config.artifacts?.enabled === false ? true : undefined,
|
|
139
|
+
artifactsDir: config.artifacts?.dir
|
|
140
|
+
? resolve(rootDir, config.artifacts.dir)
|
|
141
|
+
: undefined,
|
|
142
|
+
artifactsExclude: config.artifacts?.exclude,
|
|
143
|
+
// W0077 Phase 6h — task-source selection. Default is content-lake
|
|
144
|
+
// (signaled by `taskSourceType` undefined); when `repo`, the
|
|
145
|
+
// composition-root resolves `repoTasksPath` (defaulting to
|
|
146
|
+
// `<rootDir>/.ailf/tasks/` when unset).
|
|
147
|
+
taskSourceType: config.taskSource?.type === "repo" ? "repo" : undefined,
|
|
148
|
+
repoTasksPath: config.taskSource?.repoTasksPath
|
|
149
|
+
? resolve(rootDir, config.taskSource.repoTasksPath)
|
|
150
|
+
: undefined,
|
|
125
151
|
presets: config.presets,
|
|
126
152
|
};
|
|
127
153
|
}
|
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
* @see packages/core/src/ports/task-source.ts — TaskSource port
|
|
16
16
|
* @see docs/decisions/D0038-content-lake-authorable-task-modes.md
|
|
17
17
|
*/
|
|
18
|
+
import { ContentLakeAuthorableTaskSchema } from "./repo-schemas.js";
|
|
18
19
|
// ---------------------------------------------------------------------------
|
|
19
20
|
// GROQ query — fetches ailf.task documents with resolved references
|
|
20
21
|
// ---------------------------------------------------------------------------
|
|
@@ -98,6 +99,22 @@ export class ContentLakeTaskSource {
|
|
|
98
99
|
const mapped = mapToAuthorableTask(entry);
|
|
99
100
|
if (!mapped)
|
|
100
101
|
continue;
|
|
102
|
+
// Runtime gate (W0073): every mapped task must satisfy the domain
|
|
103
|
+
// schema before it flows into the pipeline. Throws loudly on drift so
|
|
104
|
+
// a Studio document that strays from GeneralizedTaskDefinition
|
|
105
|
+
// surfaces at load time rather than corrupting downstream state.
|
|
106
|
+
const parsed = ContentLakeAuthorableTaskSchema.safeParse(mapped);
|
|
107
|
+
if (!parsed.success) {
|
|
108
|
+
const issues = parsed.error.issues
|
|
109
|
+
.slice(0, 5)
|
|
110
|
+
.map((i) => ` [${i.path.join(".")}]: ${i.message}`)
|
|
111
|
+
.join("\n");
|
|
112
|
+
const more = parsed.error.issues.length > 5
|
|
113
|
+
? `\n …and ${parsed.error.issues.length - 5} more issue(s)`
|
|
114
|
+
: "";
|
|
115
|
+
throw new Error(`ContentLakeTaskSource: ailf.task "${mapped.id}" failed domain ` +
|
|
116
|
+
`schema validation:\n${issues}${more}`);
|
|
117
|
+
}
|
|
101
118
|
definitions.push(mapped);
|
|
102
119
|
}
|
|
103
120
|
if (definitions.length === 0 && !filter) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export { CompositeTaskSource } from "./composite-task-source.js";
|
|
2
2
|
export { ContentLakeTaskSource } from "./content-lake-task-source.js";
|
|
3
|
-
export { CanonicalTaskFileSchema, CanonicalTaskSchema, CURATED_ASSERTION_TYPES, detectLegacyFieldNames, parseCanonicalTaskFile, parseRepoConfig, RepoConfigSchema, RUBRIC_TEMPLATE_NAMES, type CanonicalTask, type CuratedAssertionType, type RepoConfig, type RubricTemplateName, } from "./repo-schemas.js";
|
|
3
|
+
export { CanonicalTaskFileSchema, CanonicalTaskSchema, ContentLakeAuthorableTaskSchema, CURATED_ASSERTION_TYPES, detectLegacyFieldNames, parseCanonicalTaskFile, parseRepoConfig, RepoConfigSchema, RUBRIC_TEMPLATE_NAMES, type CanonicalTask, type ContentLakeAuthorableTaskParsed, type CuratedAssertionType, type RepoConfig, type RubricTemplateName, } from "./repo-schemas.js";
|
|
4
4
|
export { RepoTaskSource } from "./repo-task-source.js";
|
|
5
5
|
export { detectTriggerContext, resolveTrigger, type ResolvedTrigger, type TriggerContext, } from "./repo-trigger.js";
|
|
6
6
|
export { formatValidationResult, validateCanonicalTasks, type ValidationMessage, type ValidationResult, } from "./repo-validation.js";
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export { CompositeTaskSource } from "./composite-task-source.js";
|
|
2
2
|
export { ContentLakeTaskSource } from "./content-lake-task-source.js";
|
|
3
|
-
export { CanonicalTaskFileSchema, CanonicalTaskSchema, CURATED_ASSERTION_TYPES, detectLegacyFieldNames, parseCanonicalTaskFile, parseRepoConfig, RepoConfigSchema, RUBRIC_TEMPLATE_NAMES, } from "./repo-schemas.js";
|
|
3
|
+
export { CanonicalTaskFileSchema, CanonicalTaskSchema, ContentLakeAuthorableTaskSchema, CURATED_ASSERTION_TYPES, detectLegacyFieldNames, parseCanonicalTaskFile, parseRepoConfig, RepoConfigSchema, RUBRIC_TEMPLATE_NAMES, } from "./repo-schemas.js";
|
|
4
4
|
export { RepoTaskSource } from "./repo-task-source.js";
|
|
5
5
|
export { detectTriggerContext, resolveTrigger, } from "./repo-trigger.js";
|
|
6
6
|
export { formatValidationResult, validateCanonicalTasks, } from "./repo-validation.js";
|
|
@@ -664,6 +664,122 @@ export declare const CanonicalTaskSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
664
664
|
}, z.core.$strip>>;
|
|
665
665
|
}, z.core.$strict>], "mode">;
|
|
666
666
|
export type CanonicalTask = z.infer<typeof CanonicalTaskSchema>;
|
|
667
|
+
export declare const ContentLakeAuthorableTaskSchema: z.ZodObject<{
|
|
668
|
+
id: z.ZodString;
|
|
669
|
+
title: z.ZodString;
|
|
670
|
+
description: z.ZodOptional<z.ZodString>;
|
|
671
|
+
area: z.ZodOptional<z.ZodString>;
|
|
672
|
+
difficulty: z.ZodOptional<z.ZodEnum<{
|
|
673
|
+
basic: "basic";
|
|
674
|
+
intermediate: "intermediate";
|
|
675
|
+
advanced: "advanced";
|
|
676
|
+
}>>;
|
|
677
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
678
|
+
status: z.ZodDefault<z.ZodOptional<z.ZodEnum<{
|
|
679
|
+
active: "active";
|
|
680
|
+
draft: "draft";
|
|
681
|
+
paused: "paused";
|
|
682
|
+
archived: "archived";
|
|
683
|
+
}>>>;
|
|
684
|
+
assertions: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
|
|
685
|
+
type: z.ZodLiteral<"llm-rubric">;
|
|
686
|
+
template: z.ZodEnum<{
|
|
687
|
+
"task-completion": "task-completion";
|
|
688
|
+
"code-correctness": "code-correctness";
|
|
689
|
+
"doc-coverage": "doc-coverage";
|
|
690
|
+
"mcp-input-validation": "mcp-input-validation";
|
|
691
|
+
"mcp-output-correctness": "mcp-output-correctness";
|
|
692
|
+
"mcp-error-handling": "mcp-error-handling";
|
|
693
|
+
"mcp-security": "mcp-security";
|
|
694
|
+
"factual-correctness": "factual-correctness";
|
|
695
|
+
completeness: "completeness";
|
|
696
|
+
currency: "currency";
|
|
697
|
+
"process-quality": "process-quality";
|
|
698
|
+
"agent-output": "agent-output";
|
|
699
|
+
"agent-tool-usage": "agent-tool-usage";
|
|
700
|
+
}>;
|
|
701
|
+
criteria: z.ZodArray<z.ZodString>;
|
|
702
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
703
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
704
|
+
type: z.ZodEnum<{
|
|
705
|
+
"llm-rubric": "llm-rubric";
|
|
706
|
+
contains: "contains";
|
|
707
|
+
"contains-any": "contains-any";
|
|
708
|
+
"contains-all": "contains-all";
|
|
709
|
+
"not-contains": "not-contains";
|
|
710
|
+
icontains: "icontains";
|
|
711
|
+
"icontains-any": "icontains-any";
|
|
712
|
+
regex: "regex";
|
|
713
|
+
javascript: "javascript";
|
|
714
|
+
similar: "similar";
|
|
715
|
+
cost: "cost";
|
|
716
|
+
latency: "latency";
|
|
717
|
+
"file-exists": "file-exists";
|
|
718
|
+
"file-contains": "file-contains";
|
|
719
|
+
"command-succeeds": "command-succeeds";
|
|
720
|
+
"diff-matches": "diff-matches";
|
|
721
|
+
}>;
|
|
722
|
+
value: z.ZodOptional<z.ZodUnknown>;
|
|
723
|
+
threshold: z.ZodOptional<z.ZodNumber>;
|
|
724
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
725
|
+
}, z.core.$loose>]>>>;
|
|
726
|
+
rubric: z.ZodOptional<z.ZodUnion<readonly [z.ZodObject<{
|
|
727
|
+
ref: z.ZodString;
|
|
728
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
729
|
+
inline: z.ZodString;
|
|
730
|
+
dimensions: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
731
|
+
key: z.ZodString;
|
|
732
|
+
weight: z.ZodNumber;
|
|
733
|
+
}, z.core.$strip>>>;
|
|
734
|
+
}, z.core.$strip>]>>;
|
|
735
|
+
providers: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
736
|
+
id: z.ZodString;
|
|
737
|
+
config: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
738
|
+
}, z.core.$strip>>>;
|
|
739
|
+
options: z.ZodOptional<z.ZodObject<{
|
|
740
|
+
timeout: z.ZodOptional<z.ZodNumber>;
|
|
741
|
+
cache: z.ZodOptional<z.ZodBoolean>;
|
|
742
|
+
transformOutput: z.ZodOptional<z.ZodString>;
|
|
743
|
+
promptfooOverrides: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
744
|
+
}, z.core.$strip>>;
|
|
745
|
+
prompt: z.ZodOptional<z.ZodObject<{
|
|
746
|
+
template: z.ZodOptional<z.ZodString>;
|
|
747
|
+
text: z.ZodOptional<z.ZodString>;
|
|
748
|
+
systemMessage: z.ZodOptional<z.ZodString>;
|
|
749
|
+
vars: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
750
|
+
}, z.core.$strip>>;
|
|
751
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
752
|
+
mode: z.ZodLiteral<"literacy">;
|
|
753
|
+
context: z.ZodOptional<z.ZodObject<{
|
|
754
|
+
docs: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
|
|
755
|
+
id: z.ZodString;
|
|
756
|
+
reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
757
|
+
slug: z.ZodOptional<z.ZodString>;
|
|
758
|
+
path: z.ZodOptional<z.ZodString>;
|
|
759
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
760
|
+
slug: z.ZodString;
|
|
761
|
+
reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
762
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
763
|
+
path: z.ZodString;
|
|
764
|
+
reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
765
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
766
|
+
perspective: z.ZodString;
|
|
767
|
+
reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
768
|
+
}, z.core.$strip>]>>>;
|
|
769
|
+
fixtures: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
770
|
+
}, z.core.$strip>>;
|
|
771
|
+
referenceSolution: z.ZodOptional<z.ZodString>;
|
|
772
|
+
docCoverage: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
|
|
773
|
+
baseline: z.ZodOptional<z.ZodObject<{
|
|
774
|
+
enabled: z.ZodOptional<z.ZodBoolean>;
|
|
775
|
+
rubric: z.ZodOptional<z.ZodEnum<{
|
|
776
|
+
full: "full";
|
|
777
|
+
abbreviated: "abbreviated";
|
|
778
|
+
none: "none";
|
|
779
|
+
}>>;
|
|
780
|
+
}, z.core.$strip>>;
|
|
781
|
+
}, z.core.$strict>;
|
|
782
|
+
export type ContentLakeAuthorableTaskParsed = z.infer<typeof ContentLakeAuthorableTaskSchema>;
|
|
667
783
|
/**
|
|
668
784
|
* Schema for an array of canonical tasks — what a single .ailf/tasks/*.yaml
|
|
669
785
|
* file contains. Each file must define at least one task.
|
|
@@ -1318,11 +1434,49 @@ export declare const RepoConfigSchema: z.ZodObject<{
|
|
|
1318
1434
|
projectId: z.ZodOptional<z.ZodString>;
|
|
1319
1435
|
dataset: z.ZodOptional<z.ZodString>;
|
|
1320
1436
|
baseUrl: z.ZodOptional<z.ZodString>;
|
|
1437
|
+
studioOrigin: z.ZodOptional<z.ZodString>;
|
|
1321
1438
|
}, z.core.$strip>>;
|
|
1322
1439
|
reportStore: z.ZodOptional<z.ZodObject<{
|
|
1323
1440
|
projectId: z.ZodString;
|
|
1324
1441
|
dataset: z.ZodString;
|
|
1325
1442
|
}, z.core.$strip>>;
|
|
1443
|
+
publish: z.ZodOptional<z.ZodObject<{
|
|
1444
|
+
auto: z.ZodOptional<z.ZodEnum<{
|
|
1445
|
+
never: "never";
|
|
1446
|
+
always: "always";
|
|
1447
|
+
"full-runs": "full-runs";
|
|
1448
|
+
}>>;
|
|
1449
|
+
tag: z.ZodOptional<z.ZodString>;
|
|
1450
|
+
}, z.core.$strip>>;
|
|
1451
|
+
execution: z.ZodOptional<z.ZodObject<{
|
|
1452
|
+
concurrency: z.ZodOptional<z.ZodNumber>;
|
|
1453
|
+
graderReplications: z.ZodOptional<z.ZodNumber>;
|
|
1454
|
+
gapAnalysis: z.ZodOptional<z.ZodBoolean>;
|
|
1455
|
+
apiUrl: z.ZodOptional<z.ZodString>;
|
|
1456
|
+
}, z.core.$strip>>;
|
|
1457
|
+
output: z.ZodOptional<z.ZodObject<{
|
|
1458
|
+
dir: z.ZodOptional<z.ZodString>;
|
|
1459
|
+
}, z.core.$strip>>;
|
|
1460
|
+
owner: z.ZodOptional<z.ZodObject<{
|
|
1461
|
+
team: z.ZodOptional<z.ZodString>;
|
|
1462
|
+
individual: z.ZodOptional<z.ZodString>;
|
|
1463
|
+
}, z.core.$strip>>;
|
|
1464
|
+
agentic: z.ZodOptional<z.ZodObject<{
|
|
1465
|
+
headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
1466
|
+
allowedOrigins: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
1467
|
+
}, z.core.$strip>>;
|
|
1468
|
+
artifacts: z.ZodOptional<z.ZodObject<{
|
|
1469
|
+
enabled: z.ZodOptional<z.ZodBoolean>;
|
|
1470
|
+
dir: z.ZodOptional<z.ZodString>;
|
|
1471
|
+
exclude: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
1472
|
+
}, z.core.$strip>>;
|
|
1473
|
+
taskSource: z.ZodOptional<z.ZodObject<{
|
|
1474
|
+
type: z.ZodOptional<z.ZodEnum<{
|
|
1475
|
+
"content-lake": "content-lake";
|
|
1476
|
+
repo: "repo";
|
|
1477
|
+
}>>;
|
|
1478
|
+
repoTasksPath: z.ZodOptional<z.ZodString>;
|
|
1479
|
+
}, z.core.$strip>>;
|
|
1326
1480
|
triggers: z.ZodOptional<z.ZodObject<{
|
|
1327
1481
|
pr: z.ZodOptional<z.ZodObject<{
|
|
1328
1482
|
mode: z.ZodDefault<z.ZodEnum<{
|
|
@@ -320,6 +320,16 @@ export const CanonicalTaskSchema = z.discriminatedUnion("mode", [
|
|
|
320
320
|
KnowledgeProbeTaskSchema,
|
|
321
321
|
CustomTaskSchema,
|
|
322
322
|
]);
|
|
323
|
+
// ---------------------------------------------------------------------------
|
|
324
|
+
// Content Lake authorable slice (D0038 / W0073)
|
|
325
|
+
//
|
|
326
|
+
// Runtime validator for Content Lake-authored tasks. Mirrors the
|
|
327
|
+
// `ContentLakeAuthorableTask` type from @sanity/ailf-core: today exactly
|
|
328
|
+
// the literacy variant. Expanding `ContentLakeAuthorableMode` requires
|
|
329
|
+
// extending this schema in the same pass (use `z.discriminatedUnion("mode", [...])`
|
|
330
|
+
// once the authorable set has more than one member).
|
|
331
|
+
// ---------------------------------------------------------------------------
|
|
332
|
+
export const ContentLakeAuthorableTaskSchema = LiteracyTaskSchema;
|
|
323
333
|
/**
|
|
324
334
|
* Schema for an array of canonical tasks — what a single .ailf/tasks/*.yaml
|
|
325
335
|
* file contains. Each file must define at least one task.
|
|
@@ -417,12 +427,17 @@ const ScheduleTriggerSchema = TriggerConfigSchema.extend({
|
|
|
417
427
|
/**
|
|
418
428
|
* Documentation source configuration.
|
|
419
429
|
* Defines which Sanity project holds the documentation being evaluated.
|
|
430
|
+
*
|
|
431
|
+
* `studioOrigin` (W0077 Phase 6d) replaces the retired
|
|
432
|
+
* `--sanity-studio-origin` CLI flag on `ailf run`. The `SANITY_STUDIO_ORIGIN`
|
|
433
|
+
* env var still wins over this value at resolution time.
|
|
420
434
|
*/
|
|
421
435
|
const SourceConfigSchema = z
|
|
422
436
|
.object({
|
|
423
437
|
projectId: z.string().min(1).optional(),
|
|
424
438
|
dataset: z.string().min(1).optional(),
|
|
425
439
|
baseUrl: z.string().url().optional(),
|
|
440
|
+
studioOrigin: z.string().url().optional(),
|
|
426
441
|
})
|
|
427
442
|
.optional();
|
|
428
443
|
/**
|
|
@@ -435,6 +450,121 @@ const ReportStoreConfigSchema = z
|
|
|
435
450
|
dataset: z.string().min(1),
|
|
436
451
|
})
|
|
437
452
|
.optional();
|
|
453
|
+
/**
|
|
454
|
+
* Publish policy. Controls when `ailf run` writes a report to the Content
|
|
455
|
+
* Lake without an explicit `--publish` / `--no-publish` flag.
|
|
456
|
+
*
|
|
457
|
+
* - `auto: "always"` — publish any run with a configured report store
|
|
458
|
+
* - `auto: "full-runs"` — publish non-debug runs (default)
|
|
459
|
+
* - `auto: "never"` — never auto-publish; users must pass --publish
|
|
460
|
+
*
|
|
461
|
+
* `tag` is a default value for `--publish-tag` when not passed at the CLI.
|
|
462
|
+
*
|
|
463
|
+
* @see docs/design-docs/pipeline-command-surface.md §5.3
|
|
464
|
+
*/
|
|
465
|
+
const PublishConfigSchema = z
|
|
466
|
+
.object({
|
|
467
|
+
auto: z.enum(["always", "full-runs", "never"]).optional(),
|
|
468
|
+
tag: z.string().optional(),
|
|
469
|
+
})
|
|
470
|
+
.optional();
|
|
471
|
+
/**
|
|
472
|
+
* Execution-tier configuration. Per-environment values that the four
|
|
473
|
+
* retired CLI flags used to set: concurrency, grader replications, gap
|
|
474
|
+
* analysis toggle, and the AILF API URL.
|
|
475
|
+
*
|
|
476
|
+
* @see docs/design-docs/pipeline-command-surface.md §5.7 (W0077 Phase 6b)
|
|
477
|
+
*/
|
|
478
|
+
const ExecutionConfigSchema = z
|
|
479
|
+
.object({
|
|
480
|
+
concurrency: z.number().int().positive().optional(),
|
|
481
|
+
graderReplications: z.number().int().positive().optional(),
|
|
482
|
+
gapAnalysis: z.boolean().optional(),
|
|
483
|
+
apiUrl: z.string().url().optional(),
|
|
484
|
+
})
|
|
485
|
+
.optional();
|
|
486
|
+
/**
|
|
487
|
+
* Task-source configuration (W0077 Phase 6h). Replaces the retired
|
|
488
|
+
* `--task-source` and `--repo-tasks-path` CLI flags on `ailf run`.
|
|
489
|
+
*
|
|
490
|
+
* - `type` — `content-lake` (default) or `repo`. When `repo`,
|
|
491
|
+
* tasks load from `repoTasksPath` (or
|
|
492
|
+
* `<cwd>/.ailf/tasks/` if unset).
|
|
493
|
+
* - `repoTasksPath` — optional explicit path. Resolved relative to the
|
|
494
|
+
* caller's cwd. Required to exist on disk.
|
|
495
|
+
*
|
|
496
|
+
* No env-var fallback today; cascade is config-file → built-in default.
|
|
497
|
+
*/
|
|
498
|
+
const TaskSourceConfigSchema = z
|
|
499
|
+
.object({
|
|
500
|
+
type: z.enum(["content-lake", "repo"]).optional(),
|
|
501
|
+
repoTasksPath: z.string().min(1).optional(),
|
|
502
|
+
})
|
|
503
|
+
.optional();
|
|
504
|
+
/**
|
|
505
|
+
* Artifact-writer configuration (W0077 Phase 6g). Replaces the retired
|
|
506
|
+
* `--no-artifacts`, `--artifacts-dir`, and `--artifacts-exclude` CLI flags
|
|
507
|
+
* on `ailf run`. The `AILF_ARTIFACTS_DIR` env var still wins over
|
|
508
|
+
* `artifacts.dir` at resolution time. `artifacts.enabled` defaults to
|
|
509
|
+
* `true` (writers attached); set `false` to disable all writers (mirrors
|
|
510
|
+
* the legacy `--no-artifacts` semantics).
|
|
511
|
+
*
|
|
512
|
+
* Other commands (`ailf runs export`, etc.) keep their `--artifacts-dir`
|
|
513
|
+
* flag — that's a "read from this directory" override, distinct from the
|
|
514
|
+
* pipeline's write-side `artifacts.dir`.
|
|
515
|
+
*/
|
|
516
|
+
const ArtifactsConfigSchema = z
|
|
517
|
+
.object({
|
|
518
|
+
enabled: z.boolean().optional(),
|
|
519
|
+
dir: z.string().min(1).optional(),
|
|
520
|
+
exclude: z.array(z.string().min(1)).optional(),
|
|
521
|
+
})
|
|
522
|
+
.optional();
|
|
523
|
+
/**
|
|
524
|
+
* Owner attribution (W0077 Phase 6f). Replaces the retired `--owner-team`
|
|
525
|
+
* and `--owner-individual` CLI flags. Both feed the D0037 caller envelope
|
|
526
|
+
* that surfaces in remote-mode runs. Env vars `AILF_OWNER_TEAM` and
|
|
527
|
+
* `AILF_OWNER_INDIVIDUAL` still win over these values at resolution time.
|
|
528
|
+
*/
|
|
529
|
+
const OwnerConfigSchema = z
|
|
530
|
+
.object({
|
|
531
|
+
team: z.string().min(1).optional(),
|
|
532
|
+
individual: z.string().min(1).optional(),
|
|
533
|
+
})
|
|
534
|
+
.optional();
|
|
535
|
+
/**
|
|
536
|
+
* Agentic-mode configuration (W0077 Phase 6f). Replaces the retired
|
|
537
|
+
* `--header` and `--allowed-origin` CLI flags. `headers` is a key/value
|
|
538
|
+
* object (mirrors `DOC_HEADERS` env-var JSON shape); `allowedOrigins` is a
|
|
539
|
+
* list of origin globs. The `DOC_HEADERS` and `DOC_ALLOWED_ORIGIN(S)` env
|
|
540
|
+
* vars still apply downstream as additive merges.
|
|
541
|
+
*/
|
|
542
|
+
const AgenticConfigSchema = z
|
|
543
|
+
.object({
|
|
544
|
+
headers: z.record(z.string(), z.string()).optional(),
|
|
545
|
+
allowedOrigins: z.array(z.string().min(1)).optional(),
|
|
546
|
+
})
|
|
547
|
+
.optional();
|
|
548
|
+
/**
|
|
549
|
+
* Output-directory configuration. Replaces the retired `--output-dir`
|
|
550
|
+
* CLI flag on `ailf run`. Resolution order:
|
|
551
|
+
*
|
|
552
|
+
* .ailf/config.yaml `output.dir` > built-in default
|
|
553
|
+
*
|
|
554
|
+
* Path is resolved relative to the caller's cwd. The built-in default is
|
|
555
|
+
* `<cwd>/.ailf/results/latest/` (see resolve-output-dir.ts). Other commands
|
|
556
|
+
* (`ailf publish`, `ailf pr-comment`, etc.) keep their `--output-dir`
|
|
557
|
+
* flag — that's a "read from this directory" override, distinct from the
|
|
558
|
+
* pipeline's write-side `output.dir`.
|
|
559
|
+
*
|
|
560
|
+
* @see docs/design-docs/pipeline-command-surface.md §5.7 (W0077 Phase 6c)
|
|
561
|
+
* @see docs/design-docs/output-dir-routing.md
|
|
562
|
+
*/
|
|
563
|
+
const OutputConfigSchema = z
|
|
564
|
+
.object({
|
|
565
|
+
dir: z.string().min(1).optional(),
|
|
566
|
+
})
|
|
567
|
+
.optional();
|
|
438
568
|
/**
|
|
439
569
|
* Zod schema for .ailf/config.yaml — controls documentation source,
|
|
440
570
|
* report destination, and trigger behavior for evaluations from an
|
|
@@ -443,6 +573,13 @@ const ReportStoreConfigSchema = z
|
|
|
443
573
|
export const RepoConfigSchema = z.object({
|
|
444
574
|
source: SourceConfigSchema,
|
|
445
575
|
reportStore: ReportStoreConfigSchema,
|
|
576
|
+
publish: PublishConfigSchema,
|
|
577
|
+
execution: ExecutionConfigSchema,
|
|
578
|
+
output: OutputConfigSchema,
|
|
579
|
+
owner: OwnerConfigSchema,
|
|
580
|
+
agentic: AgenticConfigSchema,
|
|
581
|
+
artifacts: ArtifactsConfigSchema,
|
|
582
|
+
taskSource: TaskSourceConfigSchema,
|
|
446
583
|
triggers: z
|
|
447
584
|
.object({
|
|
448
585
|
pr: TriggerConfigSchema.optional(),
|
package/dist/cli.d.ts
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* appends Quick Start examples.
|
|
12
12
|
*
|
|
13
13
|
* Usage:
|
|
14
|
-
* ailf
|
|
14
|
+
* ailf run [flags] # full evaluation run
|
|
15
15
|
* ailf compare [flags] # compare evaluation runs
|
|
16
16
|
* ailf baseline <cmd> [flags] # baseline management
|
|
17
17
|
* ailf validate [flags] # config validation
|
|
@@ -24,6 +24,6 @@
|
|
|
24
24
|
* --dotenv <path> # override default .env path
|
|
25
25
|
*
|
|
26
26
|
* Dev mode (without building):
|
|
27
|
-
* tsx src/cli.ts
|
|
27
|
+
* tsx src/cli.ts run --debug
|
|
28
28
|
*/
|
|
29
29
|
export {};
|