@sanity/ailf 3.7.0 → 3.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config/airbyte/ai_literacy_framework.connector.yaml +1 -1
- package/config/thresholds.ts +3 -3
- package/dist/_vendor/ailf-core/examples/index.d.ts +2 -2
- package/dist/_vendor/ailf-core/examples/index.js +2 -2
- package/dist/_vendor/ailf-core/ports/context.d.ts +0 -4
- package/dist/_vendor/ailf-core/schemas/eval-config.d.ts +38 -12
- package/dist/_vendor/ailf-core/schemas/eval-config.js +102 -22
- package/dist/_vendor/ailf-core/schemas/pipeline-request.d.ts +4 -6
- package/dist/_vendor/ailf-core/schemas/pipeline-request.js +1 -3
- package/dist/_vendor/ailf-core/schemas/schedules.d.ts +2 -2
- package/dist/_vendor/ailf-shared/run-classification.d.ts +2 -2
- package/dist/_vendor/ailf-shared/run-classification.js +1 -1
- package/dist/_vendor/ailf-shared/run-context.d.ts +1 -1
- package/dist/adapters/api-client/build-request.d.ts +0 -2
- package/dist/adapters/api-client/build-request.js +2 -6
- package/dist/adapters/config-sources/cli-config-adapter.d.ts +1 -1
- package/dist/adapters/config-sources/file-config-adapter.d.ts +1 -1
- package/dist/adapters/config-sources/file-config-adapter.js +42 -17
- package/dist/adapters/task-sources/repo-schemas.d.ts +41 -3
- package/dist/adapters/task-sources/repo-schemas.js +127 -0
- package/dist/cli-program.d.ts +39 -0
- package/dist/cli-program.js +137 -0
- package/dist/cli.d.ts +8 -2
- package/dist/cli.js +128 -142
- package/dist/commands/agent-report.js +1 -1
- package/dist/commands/calculate-scores.js +0 -2
- package/dist/commands/check-staleness.js +1 -1
- package/dist/commands/chronic-failures.js +4 -4
- package/dist/commands/coverage-audit.js +6 -7
- package/dist/commands/discovery-report.js +16 -4
- package/dist/commands/eval.d.ts +1 -1
- package/dist/commands/eval.js +1 -1
- package/dist/commands/explain-handler.d.ts +1 -1
- package/dist/commands/explain-handler.js +13 -44
- package/dist/commands/fetch-docs.js +0 -2
- package/dist/commands/generate-configs.js +0 -2
- package/dist/commands/grader/index.js +3 -3
- package/dist/commands/init.d.ts +2 -2
- package/dist/commands/init.js +10 -9
- package/dist/commands/interactive.d.ts +1 -1
- package/dist/commands/interactive.js +8 -8
- package/dist/commands/pipeline-action.d.ts +1 -3
- package/dist/commands/pipeline-action.js +174 -140
- package/dist/commands/pr-comment.js +1 -3
- package/dist/commands/publish.d.ts +1 -1
- package/dist/commands/publish.js +2 -4
- package/dist/commands/readiness-report.js +17 -8
- package/dist/commands/remote-pipeline.d.ts +1 -1
- package/dist/commands/remote-pipeline.js +1 -3
- package/dist/commands/run.d.ts +64 -0
- package/dist/commands/{pipeline.js → run.js} +19 -30
- package/dist/commands/shared/help.js +4 -4
- package/dist/commands/shared/options.d.ts +29 -3
- package/dist/commands/shared/options.js +37 -13
- package/dist/commands/validate-tasks.js +1 -1
- package/dist/commands/validate.d.ts +1 -1
- package/dist/commands/validate.js +2 -2
- package/dist/commands/weekly-digest.js +3 -3
- package/dist/config/thresholds.ts +3 -3
- package/dist/orchestration/build-app-context.js +0 -2
- package/dist/orchestration/build-step-sequence.js +1 -11
- package/dist/orchestration/steps/fetch-docs-step.js +1 -1
- package/dist/orchestration/steps/index.d.ts +0 -2
- package/dist/orchestration/steps/index.js +0 -2
- package/dist/orchestration/steps/run-eval-step.js +1 -1
- package/dist/pipeline/cache.d.ts +1 -1
- package/dist/pipeline/map-request-to-config.js +0 -2
- package/dist/pipeline/mirror-repo-tasks.d.ts +1 -1
- package/dist/pipeline/plan.d.ts +2 -4
- package/dist/pipeline/plan.js +4 -32
- package/dist/pipeline/run-context.d.ts +1 -1
- package/dist/pipeline/run-context.js +4 -4
- package/dist/pipeline/validate.d.ts +1 -1
- package/dist/pipeline/validate.js +1 -1
- package/package.json +11 -9
- package/dist/commands/pipeline.d.ts +0 -77
- package/dist/orchestration/steps/discovery-report-step.d.ts +0 -13
- package/dist/orchestration/steps/discovery-report-step.js +0 -62
- package/dist/orchestration/steps/readiness-step.d.ts +0 -13
- package/dist/orchestration/steps/readiness-step.js +0 -98
- package/dist/pipeline/compiler/__tests__/agent-harness-handler.test.d.ts +0 -10
- package/dist/pipeline/compiler/__tests__/agent-harness-handler.test.js +0 -366
- package/dist/pipeline/compiler/__tests__/assertion-mapper.test.d.ts +0 -9
- package/dist/pipeline/compiler/__tests__/assertion-mapper.test.js +0 -145
- package/dist/pipeline/compiler/__tests__/knowledge-probe-handler.test.d.ts +0 -10
- package/dist/pipeline/compiler/__tests__/knowledge-probe-handler.test.js +0 -314
- package/dist/pipeline/compiler/__tests__/literacy-handler.test.d.ts +0 -10
- package/dist/pipeline/compiler/__tests__/literacy-handler.test.js +0 -486
- package/dist/pipeline/compiler/__tests__/mcp-server-handler.test.d.ts +0 -10
- package/dist/pipeline/compiler/__tests__/mcp-server-handler.test.js +0 -425
- package/dist/pipeline/compiler/__tests__/promptfoo-compiler.test.d.ts +0 -9
- package/dist/pipeline/compiler/__tests__/promptfoo-compiler.test.js +0 -332
- package/dist/pipeline/compiler/__tests__/sandbox-and-fixtures.test.d.ts +0 -12
- package/dist/pipeline/compiler/__tests__/sandbox-and-fixtures.test.js +0 -210
- package/dist/pipeline/compiler/__tests__/scoring-and-presets.test.d.ts +0 -7
- package/dist/pipeline/compiler/__tests__/scoring-and-presets.test.js +0 -404
- package/dist/pipeline/compiler/__tests__/scoring-bridge.test.d.ts +0 -10
- package/dist/pipeline/compiler/__tests__/scoring-bridge.test.js +0 -184
- package/dist/pipeline/compiler/__tests__/task-graph-builder.test.d.ts +0 -8
- package/dist/pipeline/compiler/__tests__/task-graph-builder.test.js +0 -301
- package/dist/pipeline/compiler/__tests__/telemetry.test.d.ts +0 -9
- package/dist/pipeline/compiler/__tests__/telemetry.test.js +0 -503
- package/dist/pipeline/compiler/__tests__/tool-loop-openai.test.d.ts +0 -10
- package/dist/pipeline/compiler/__tests__/tool-loop-openai.test.js +0 -509
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
*
|
|
5
5
|
* Reads a score-summary.json (defaulting to results/latest/score-summary.json),
|
|
6
6
|
* builds provenance, writes the report to Sanity, and fans out to configured
|
|
7
|
-
* sinks — exactly the same as the publish step in `ailf
|
|
7
|
+
* sinks — exactly the same as the publish step in `ailf run`, but
|
|
8
8
|
* standalone.
|
|
9
9
|
*
|
|
10
10
|
* Uses createAppContext() (composition root) for all infrastructure access.
|
package/dist/commands/publish.js
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
*
|
|
5
5
|
* Reads a score-summary.json (defaulting to results/latest/score-summary.json),
|
|
6
6
|
* builds provenance, writes the report to Sanity, and fans out to configured
|
|
7
|
-
* sinks — exactly the same as the publish step in `ailf
|
|
7
|
+
* sinks — exactly the same as the publish step in `ailf run`, but
|
|
8
8
|
* standalone.
|
|
9
9
|
*
|
|
10
10
|
* Uses createAppContext() (composition root) for all infrastructure access.
|
|
@@ -88,7 +88,6 @@ async function runPublishCommand(summaryPath, outputDir, opts) {
|
|
|
88
88
|
// Wire up infrastructure via composition root
|
|
89
89
|
const ctx = createAppContext({
|
|
90
90
|
compareEnabled: false,
|
|
91
|
-
discoveryReportEnabled: false,
|
|
92
91
|
gapAnalysisEnabled: false,
|
|
93
92
|
mode: "literacy",
|
|
94
93
|
noAutoScope: false,
|
|
@@ -97,7 +96,6 @@ async function runPublishCommand(summaryPath, outputDir, opts) {
|
|
|
97
96
|
outputDir,
|
|
98
97
|
publishEnabled: true,
|
|
99
98
|
publishTag: opts.tag,
|
|
100
|
-
readinessEnabled: false,
|
|
101
99
|
rootDir: ROOT,
|
|
102
100
|
searchMode: "open",
|
|
103
101
|
skipEval: true,
|
|
@@ -117,7 +115,7 @@ async function runPublishCommand(summaryPath, outputDir, opts) {
|
|
|
117
115
|
if (!existsSync(resolvedPath)) {
|
|
118
116
|
console.error(` ✖ File not found: ${resolvedPath}`);
|
|
119
117
|
console.error();
|
|
120
|
-
console.error(" Hint: Run `ailf
|
|
118
|
+
console.error(" Hint: Run `ailf run` first to generate results,");
|
|
121
119
|
console.error(" or provide a path to an existing score-summary.json.");
|
|
122
120
|
process.exit(1);
|
|
123
121
|
}
|
|
@@ -15,23 +15,32 @@ import { formatReadinessMarkdown, generateReadinessReport, } from "../pipeline/r
|
|
|
15
15
|
import { ThresholdConfigSchema, } from "../pipeline/schemas.js";
|
|
16
16
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
17
17
|
const ROOT = resolve(__dirname, "..", "..");
|
|
18
|
-
const
|
|
19
|
-
const GAP_ANALYSIS_PATH = join(ROOT, "results", "latest", "gap-analysis.json");
|
|
18
|
+
const DEFAULT_RESULTS_DIR = join(ROOT, "results", "latest");
|
|
20
19
|
// thresholds loaded via loadConfigFile below
|
|
21
20
|
const BASELINES_DIR = join(ROOT, "results", "baselines");
|
|
21
|
+
/** Resolve `--from-run` to an absolute results directory. */
|
|
22
|
+
function resolveFromRun(value) {
|
|
23
|
+
if (value === "latest")
|
|
24
|
+
return DEFAULT_RESULTS_DIR;
|
|
25
|
+
return resolve(value);
|
|
26
|
+
}
|
|
22
27
|
export function createReadinessReportCommand() {
|
|
23
|
-
return new Command("readiness
|
|
28
|
+
return new Command("readiness")
|
|
24
29
|
.description("Generate launch readiness report for a feature area")
|
|
25
30
|
.requiredOption("-a, --area <area>", "Feature area to evaluate (required)")
|
|
31
|
+
.option("--from-run <path>", "Results directory to read from (`latest` or a path to a results directory containing score-summary.json)", "latest")
|
|
26
32
|
.option("-H, --history", "Include historical progress from baselines", false)
|
|
27
33
|
.option("-o, --output <path>", "Write markdown to file instead of stdout")
|
|
28
34
|
.action(async (opts) => {
|
|
35
|
+
const resultsDir = resolveFromRun(opts.fromRun);
|
|
36
|
+
const scoreSummaryPath = join(resultsDir, "score-summary.json");
|
|
37
|
+
const gapAnalysisPath = join(resultsDir, "gap-analysis.json");
|
|
29
38
|
// Load score summary
|
|
30
|
-
if (!existsSync(
|
|
31
|
-
console.error(`❌ Score summary not found at ${
|
|
39
|
+
if (!existsSync(scoreSummaryPath)) {
|
|
40
|
+
console.error(`❌ Score summary not found at ${scoreSummaryPath}. Run \`ailf run\` first or pass --from-run <path>.`);
|
|
32
41
|
process.exit(1);
|
|
33
42
|
}
|
|
34
|
-
const scoreSummary = JSON.parse(readFileSync(
|
|
43
|
+
const scoreSummary = JSON.parse(readFileSync(scoreSummaryPath, "utf-8"));
|
|
35
44
|
// Load threshold config
|
|
36
45
|
let parsedThresholds;
|
|
37
46
|
try {
|
|
@@ -57,8 +66,8 @@ export function createReadinessReportCommand() {
|
|
|
57
66
|
const thresholdConfig = thresholdResult.data;
|
|
58
67
|
// Load gap analysis (optional)
|
|
59
68
|
let gapAnalysis;
|
|
60
|
-
if (existsSync(
|
|
61
|
-
gapAnalysis = JSON.parse(readFileSync(
|
|
69
|
+
if (existsSync(gapAnalysisPath)) {
|
|
70
|
+
gapAnalysis = JSON.parse(readFileSync(gapAnalysisPath, "utf-8"));
|
|
62
71
|
}
|
|
63
72
|
const history = [];
|
|
64
73
|
if (opts.history && existsSync(BASELINES_DIR)) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* remote-pipeline.ts — Remote execution flow for `ailf
|
|
2
|
+
* remote-pipeline.ts — Remote execution flow for `ailf run --remote`.
|
|
3
3
|
*
|
|
4
4
|
* Reads local `.ailf/tasks/` YAML, validates locally via Zod schemas,
|
|
5
5
|
* submits to the AILF API, polls for completion, and writes the same
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* remote-pipeline.ts — Remote execution flow for `ailf
|
|
2
|
+
* remote-pipeline.ts — Remote execution flow for `ailf run --remote`.
|
|
3
3
|
*
|
|
4
4
|
* Reads local `.ailf/tasks/` YAML, validates locally via Zod schemas,
|
|
5
5
|
* submits to the AILF API, polls for completion, and writes the same
|
|
@@ -135,8 +135,6 @@ function toConfigSlice(opts) {
|
|
|
135
135
|
perspectiveOverride: opts.perspectiveOverride,
|
|
136
136
|
graderReplications: opts.graderReplications,
|
|
137
137
|
gapAnalysisEnabled: opts.gapAnalysisEnabled,
|
|
138
|
-
readinessEnabled: opts.readinessEnabled,
|
|
139
|
-
discoveryReportEnabled: opts.discoveryReportEnabled,
|
|
140
138
|
noRemoteCache: opts.noRemoteCache,
|
|
141
139
|
// D0037 / W0069 caller envelope overrides — flags override env vars
|
|
142
140
|
// inside buildCallerEnvelope(), which also merges AILF_* defaults.
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* run command — the main evaluation entry point.
|
|
3
|
+
*
|
|
4
|
+
* Renamed from `ailf pipeline` to `ailf run` per W0077 Phase 1
|
|
5
|
+
* (see docs/design-docs/pipeline-command-surface.md §3). The command drives
|
|
6
|
+
* the evaluation pipeline; the orchestrator internals retain the "pipeline"
|
|
7
|
+
* name because they describe a multi-step process, not the CLI verb.
|
|
8
|
+
*
|
|
9
|
+
* Defines all 44+ CLI flags via Commander, resolves them into a typed
|
|
10
|
+
* options object, bridges to process.env for downstream modules, and
|
|
11
|
+
* delegates to runPipeline().
|
|
12
|
+
*
|
|
13
|
+
* @see docs/guides/cli-guide.md for per-flag behavior notes, or
|
|
14
|
+
* docs/references/cli-reference.md for the auto-generated flag matrix.
|
|
15
|
+
*/
|
|
16
|
+
import { Command } from "commander";
|
|
17
|
+
/**
|
|
18
|
+
* Raw CLI options as parsed by Commander.
|
|
19
|
+
* Field names follow Commander's camelCase convention for kebab-case flags.
|
|
20
|
+
*/
|
|
21
|
+
export interface PipelineCliOptions {
|
|
22
|
+
area?: string;
|
|
23
|
+
autoScope: boolean;
|
|
24
|
+
/** `--before-source <name>` — swap the doc source to a "before" state for impact evaluation. */
|
|
25
|
+
beforeSource?: string;
|
|
26
|
+
cache: boolean;
|
|
27
|
+
changedDocs?: string;
|
|
28
|
+
/**
|
|
29
|
+
* `--compare [baseline]` — Commander optional argument.
|
|
30
|
+
* undefined → flag not passed
|
|
31
|
+
* true → bare `--compare` (compare against latest baseline)
|
|
32
|
+
* string → `--compare path/to/baseline.json` (pin to a file)
|
|
33
|
+
*/
|
|
34
|
+
compare?: boolean | string;
|
|
35
|
+
config?: string;
|
|
36
|
+
debug: boolean;
|
|
37
|
+
filterFirstN?: number;
|
|
38
|
+
filterPattern?: string;
|
|
39
|
+
filterSample?: number;
|
|
40
|
+
dryRun: boolean;
|
|
41
|
+
eval: boolean;
|
|
42
|
+
fetch: boolean;
|
|
43
|
+
mode: string;
|
|
44
|
+
variant?: string;
|
|
45
|
+
output?: string;
|
|
46
|
+
promptfooUrl?: string;
|
|
47
|
+
publish?: boolean;
|
|
48
|
+
publishTag?: string;
|
|
49
|
+
remoteCache?: boolean;
|
|
50
|
+
sanityDocument: string[];
|
|
51
|
+
sanityPerspective?: string;
|
|
52
|
+
search?: string;
|
|
53
|
+
source?: string;
|
|
54
|
+
remote: boolean;
|
|
55
|
+
task?: string;
|
|
56
|
+
tag: string[];
|
|
57
|
+
threshold?: number;
|
|
58
|
+
url: string[];
|
|
59
|
+
artifactsWrite: boolean;
|
|
60
|
+
classification?: string;
|
|
61
|
+
purpose?: string;
|
|
62
|
+
label: string[];
|
|
63
|
+
}
|
|
64
|
+
export declare function createRunCommand(): Command;
|
|
@@ -1,23 +1,29 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* run command — the main evaluation entry point.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
4
|
+
* Renamed from `ailf pipeline` to `ailf run` per W0077 Phase 1
|
|
5
|
+
* (see docs/design-docs/pipeline-command-surface.md §3). The command drives
|
|
6
|
+
* the evaluation pipeline; the orchestrator internals retain the "pipeline"
|
|
7
|
+
* name because they describe a multi-step process, not the CLI verb.
|
|
8
|
+
*
|
|
9
|
+
* Defines all 44+ CLI flags via Commander, resolves them into a typed
|
|
5
10
|
* options object, bridges to process.env for downstream modules, and
|
|
6
11
|
* delegates to runPipeline().
|
|
7
12
|
*
|
|
8
|
-
* @see docs/cli.md for
|
|
13
|
+
* @see docs/guides/cli-guide.md for per-flag behavior notes, or
|
|
14
|
+
* docs/references/cli-reference.md for the auto-generated flag matrix.
|
|
9
15
|
*/
|
|
10
16
|
import { Command } from "commander";
|
|
11
|
-
import { addAgenticOptions, addDebugOptions,
|
|
12
|
-
export function
|
|
13
|
-
const cmd = new Command("
|
|
17
|
+
import { addAgenticOptions, addDebugOptions, addSanityScopeOptions, } from "./shared/options.js";
|
|
18
|
+
export function createRunCommand() {
|
|
19
|
+
const cmd = new Command("run")
|
|
14
20
|
.description("Run the full evaluation pipeline")
|
|
15
21
|
.option("-m, --mode <mode>", "Evaluation mode: literacy (default), mcp-server, agent-harness, knowledge-probe, custom. Legacy aliases (baseline, agentic, observed, full) are accepted and normalized to literacy + variant.", "literacy")
|
|
16
22
|
.option("--variant <variant>", "Literacy variant: full (default — standard + agentic), baseline (standard only), agentic (agentic only), observed. Only applies to --mode literacy.")
|
|
17
23
|
.option("-s, --source <name>", "Documentation source name (from sources.yaml)")
|
|
18
24
|
.option("-n, --dry-run", "Validate configuration only, no execution", false)
|
|
19
|
-
.option("--
|
|
20
|
-
.option("--
|
|
25
|
+
.option("--no-fetch", "Reuse cached documentation contexts")
|
|
26
|
+
.option("--no-eval", "Recalculate from existing eval results")
|
|
21
27
|
.option("--no-cache", "Bypass all pipeline-level caching")
|
|
22
28
|
.option("--no-remote-cache", "Disable Content Lake cache lookup (local cache still active)")
|
|
23
29
|
.option("--no-auto-scope", "Disable release-aware auto-scoping (evaluate all tasks even when a perspective is set)")
|
|
@@ -31,39 +37,22 @@ export function createPipelineCommand() {
|
|
|
31
37
|
.filter(Boolean),
|
|
32
38
|
], [])
|
|
33
39
|
.option("--changed-docs <slugs>", "Auto-scope to tasks affected by these document slugs")
|
|
34
|
-
.option("-
|
|
35
|
-
.option("
|
|
36
|
-
.option("--before <source>", "Before-state for impact evaluation")
|
|
37
|
-
.option("-c, --compare", "Compare scores against latest baseline", false)
|
|
38
|
-
.option("--compare-baseline <path>", "Specific baseline file to compare")
|
|
40
|
+
.option("--before-source <name>", "Swap the doc source to a `before` state and run a paired evaluation for impact analysis. Pairs with `ailf baseline` and `--compare`. Distinct from `--compare <baseline>`, which compares scores against a saved snapshot.")
|
|
41
|
+
.option("-c, --compare [baseline]", "Compare scores against the latest baseline. Pass a path to pin a specific baseline file (e.g. --compare results/baselines/2026-04-22.json).")
|
|
39
42
|
.option("--threshold <n>", "Noise threshold for comparison (default: 2)", parseFloat)
|
|
40
|
-
.option("--no-gap-analysis", "Skip failure mode + impact analysis")
|
|
41
|
-
.option("--readiness", "Generate launch readiness checklist", false)
|
|
42
|
-
.option("--discovery-report", "Generate agent discoverability report", false)
|
|
43
43
|
.option("-p, --publish", "Write report to Sanity + fan out to sinks (auto-enabled for full runs when report store is configured)")
|
|
44
44
|
.option("--no-publish", "Suppress auto-publishing")
|
|
45
45
|
.option("--publish-tag <tag>", "Label for published report")
|
|
46
|
-
.option("--report-dataset <name>", "Sanity dataset for report store")
|
|
47
|
-
.option("--report-project <id>", "Sanity project ID for report store")
|
|
48
46
|
.option("--config <path>", "Load pipeline config from a TS/JS/YAML/JSON file (overrides most CLI flags)")
|
|
49
47
|
.option("-o, --output <path>", "Write PR comment markdown to file")
|
|
50
|
-
.option("--output-dir <path>", "Base directory for pipeline output artifacts (default: inferred from execution context)")
|
|
51
48
|
.option("--promptfoo-url <url>", "Promptfoo share URL for report")
|
|
52
|
-
.option("--task-source <type>", "Task definition source: content-lake (default — Sanity Content Lake), repo (repo tasks only, no Content Lake merge)", "content-lake")
|
|
53
|
-
.option("--repo-tasks-path <path>", "Path to repo-based task definitions. Defaults to ./.ailf/tasks/ when --task-source=repo.")
|
|
54
49
|
.option("--remote", "Submit evaluation to the AILF API instead of running locally", false)
|
|
55
|
-
.option("--
|
|
56
|
-
.option("--no-artifacts", "Disable all artifact writers (D0033). Overrides --artifacts-dir.")
|
|
57
|
-
.option("--artifacts-dir <path>", "Root directory for local artifact output (D0033; default: .ailf/results/captures/)")
|
|
58
|
-
.option("--artifacts-dry-run", "Run artifact writers in dry-run mode — log intended writes, touch no storage", false)
|
|
59
|
-
.option("--artifacts-exclude <types>", "Comma-separated artifact types to skip (e.g. traces,graderPrompts)")
|
|
50
|
+
.option("--no-artifacts-write", "Run artifact writers in dry-run mode — log intended writes, touch no storage")
|
|
60
51
|
// D0037 caller envelope (W0069) — threads through --remote so the
|
|
61
52
|
// server-side pipeline attributes provenance to the caller, not the
|
|
62
53
|
// API gateway runner. All env-var equivalents are honored too;
|
|
63
54
|
// explicit flags win over env vars.
|
|
64
|
-
.option("--classification <value>", "Run classification for provenance: official |
|
|
65
|
-
.option("--owner-team <slug>", "Team slug this run is attributable to. Overrides AILF_OWNER_TEAM.")
|
|
66
|
-
.option("--owner-individual <slug>", "Individual (GH actor / user ID) this run is attributable to. Overrides AILF_OWNER_INDIVIDUAL.")
|
|
55
|
+
.option("--classification <value>", "Run classification for provenance: official | adhoc | experimental | test | external. Overrides AILF_CLASSIFICATION. See D0037.")
|
|
67
56
|
.option("--purpose <text>", 'Free-text "why I ran this" attached to provenance. Overrides AILF_PURPOSE.')
|
|
68
57
|
.option("--label <value>", "Free-form searchable label (repeatable). Appends to any AILF_LABELS env value.", (val, prev) => [
|
|
69
58
|
...prev,
|
|
@@ -78,7 +67,7 @@ export function createPipelineCommand() {
|
|
|
78
67
|
});
|
|
79
68
|
// Add shared option groups
|
|
80
69
|
addDebugOptions(cmd);
|
|
81
|
-
|
|
70
|
+
addSanityScopeOptions(cmd);
|
|
82
71
|
addAgenticOptions(cmd);
|
|
83
72
|
return cmd;
|
|
84
73
|
}
|
|
@@ -67,14 +67,14 @@ function hasColorSupport() {
|
|
|
67
67
|
// ---------------------------------------------------------------------------
|
|
68
68
|
const afterHelpText = `
|
|
69
69
|
Quick Start:
|
|
70
|
-
$ ailf
|
|
71
|
-
$ ailf
|
|
72
|
-
$ ailf
|
|
70
|
+
$ ailf run --debug Run a quick evaluation (first 2 tests)
|
|
71
|
+
$ ailf run --area groq Evaluate a specific feature area
|
|
72
|
+
$ ailf run --explain Preview the execution plan
|
|
73
73
|
$ ailf init Set up AILF in a new project
|
|
74
74
|
|
|
75
75
|
Documentation:
|
|
76
76
|
Repository https://github.com/sanity-io/ai-literacy-framework
|
|
77
|
-
CLI Guide https://github.com/sanity-io/ai-literacy-framework/blob/main/docs/cli.md
|
|
77
|
+
CLI Guide https://github.com/sanity-io/ai-literacy-framework/blob/main/docs/guides/cli-guide.md
|
|
78
78
|
Getting Started https://github.com/sanity-io/ai-literacy-framework/blob/main/docs/getting-started.md
|
|
79
79
|
|
|
80
80
|
Run ailf <command> --help for detailed usage of any command.`;
|
|
@@ -7,11 +7,22 @@
|
|
|
7
7
|
*/
|
|
8
8
|
import type { Command } from "commander";
|
|
9
9
|
/**
|
|
10
|
-
* Add agentic options
|
|
10
|
+
* Add agentic options on `ailf run`: `--url` and `--search`. Both are
|
|
11
|
+
* per-run overrides.
|
|
12
|
+
*
|
|
13
|
+
* **`--header` and `--allowed-origin` retired in W0077 Phase 6f** —
|
|
14
|
+
* configure them in `.ailf/config.yaml`'s `agentic` block instead. The
|
|
15
|
+
* `DOC_HEADERS` and `DOC_ALLOWED_ORIGIN(S)` env vars still apply.
|
|
11
16
|
*/
|
|
12
17
|
export declare function addAgenticOptions(cmd: Command): Command;
|
|
13
18
|
/**
|
|
14
|
-
* Add debug
|
|
19
|
+
* Add debug + filter options.
|
|
20
|
+
*
|
|
21
|
+
* `-d, --debug` is a shortcut that runs only the first 2 tests for fast
|
|
22
|
+
* feedback. `--filter-first-n`, `--filter-pattern`, and `--filter-sample`
|
|
23
|
+
* narrow the test set explicitly and mirror the same-named flags on
|
|
24
|
+
* `ailf eval` so the pipeline surface matches the underlying Promptfoo
|
|
25
|
+
* passthrough.
|
|
15
26
|
*/
|
|
16
27
|
export declare function addDebugOptions(cmd: Command): Command;
|
|
17
28
|
/**
|
|
@@ -27,9 +38,24 @@ export declare function addOutputOptions(cmd: Command): Command;
|
|
|
27
38
|
*/
|
|
28
39
|
export declare function addOutputDirOption(cmd: Command): Command;
|
|
29
40
|
/**
|
|
30
|
-
* Add Sanity
|
|
41
|
+
* Add the full Sanity-source CLI surface — dataset, project, perspective,
|
|
42
|
+
* studio origin, document. Used by `ailf fetch-docs`, which is invoked
|
|
43
|
+
* directly with explicit Sanity targeting per run.
|
|
44
|
+
*
|
|
45
|
+
* **Not used by `ailf run`** post-W0077 Phase 6d. The dataset, project, and
|
|
46
|
+
* studio-origin trio moved to `.ailf/config.yaml`'s `source` block (with
|
|
47
|
+
* `SANITY_DATASET` / `SANITY_PROJECT_ID` / `SANITY_STUDIO_ORIGIN` env-var
|
|
48
|
+
* fallbacks). `ailf run` uses `addSanityScopeOptions` instead, which keeps
|
|
49
|
+
* only the per-run flags (`--sanity-perspective`, `--sanity-document`).
|
|
31
50
|
*/
|
|
32
51
|
export declare function addSanitySourceOptions(cmd: Command): Command;
|
|
52
|
+
/**
|
|
53
|
+
* Add the per-run Sanity-scope CLI surface — `--sanity-perspective` and
|
|
54
|
+
* `--sanity-document`. Used by `ailf run`. The per-environment trio
|
|
55
|
+
* (dataset, project, studio origin) lives in `.ailf/config.yaml`'s
|
|
56
|
+
* `source` block instead (W0077 Phase 6d).
|
|
57
|
+
*/
|
|
58
|
+
export declare function addSanityScopeOptions(cmd: Command): Command;
|
|
33
59
|
/**
|
|
34
60
|
* Collect repeatable string options into an array.
|
|
35
61
|
* Used as a Commander argParser for options like --url, --header, --allowed-origin.
|
|
@@ -6,27 +6,33 @@
|
|
|
6
6
|
* (e.g., debug options, Sanity source options, output options).
|
|
7
7
|
*/
|
|
8
8
|
/**
|
|
9
|
-
* Add agentic options
|
|
9
|
+
* Add agentic options on `ailf run`: `--url` and `--search`. Both are
|
|
10
|
+
* per-run overrides.
|
|
11
|
+
*
|
|
12
|
+
* **`--header` and `--allowed-origin` retired in W0077 Phase 6f** —
|
|
13
|
+
* configure them in `.ailf/config.yaml`'s `agentic` block instead. The
|
|
14
|
+
* `DOC_HEADERS` and `DOC_ALLOWED_ORIGIN(S)` env vars still apply.
|
|
10
15
|
*/
|
|
11
16
|
export function addAgenticOptions(cmd) {
|
|
12
17
|
return cmd
|
|
13
18
|
.option("--url <url>", "Documentation URL (repeatable)", collect, [])
|
|
14
|
-
.option("--urls <url>", "Alias for --url (repeatable)", collect, [])
|
|
15
|
-
.option("--header <header>", 'Custom HTTP header "Key: Value" (repeatable)', collect, [])
|
|
16
|
-
.option("--headers <header>", "Alias for --header (repeatable)", collect, [])
|
|
17
|
-
.option("--allowed-origin <origin>", "Agent origin sandbox (repeatable, supports globs)", collect, [])
|
|
18
|
-
.option("--allowed-origins <origin>", "Alias for --allowed-origin (repeatable)", collect, [])
|
|
19
19
|
.option("-S, --search <mode>", "Web search mode: open, origin-only, off");
|
|
20
20
|
}
|
|
21
21
|
/**
|
|
22
|
-
* Add debug
|
|
22
|
+
* Add debug + filter options.
|
|
23
|
+
*
|
|
24
|
+
* `-d, --debug` is a shortcut that runs only the first 2 tests for fast
|
|
25
|
+
* feedback. `--filter-first-n`, `--filter-pattern`, and `--filter-sample`
|
|
26
|
+
* narrow the test set explicitly and mirror the same-named flags on
|
|
27
|
+
* `ailf eval` so the pipeline surface matches the underlying Promptfoo
|
|
28
|
+
* passthrough.
|
|
23
29
|
*/
|
|
24
30
|
export function addDebugOptions(cmd) {
|
|
25
31
|
return cmd
|
|
26
32
|
.option("-d, --debug", "Run subset of tests for fast feedback", false)
|
|
27
|
-
.option("--
|
|
28
|
-
.option("--
|
|
29
|
-
.option("--
|
|
33
|
+
.option("--filter-first-n <n>", "Run only first N tests", parseInt)
|
|
34
|
+
.option("--filter-pattern <regex>", "Filter tests by description regex")
|
|
35
|
+
.option("--filter-sample <n>", "Random sample of N tests", parseInt);
|
|
30
36
|
}
|
|
31
37
|
/**
|
|
32
38
|
* Add output options: --output, --format
|
|
@@ -47,7 +53,15 @@ export function addOutputDirOption(cmd) {
|
|
|
47
53
|
return cmd.option("--output-dir <path>", "Base directory for output artifacts (default: .ailf/results/latest/)");
|
|
48
54
|
}
|
|
49
55
|
/**
|
|
50
|
-
* Add Sanity
|
|
56
|
+
* Add the full Sanity-source CLI surface — dataset, project, perspective,
|
|
57
|
+
* studio origin, document. Used by `ailf fetch-docs`, which is invoked
|
|
58
|
+
* directly with explicit Sanity targeting per run.
|
|
59
|
+
*
|
|
60
|
+
* **Not used by `ailf run`** post-W0077 Phase 6d. The dataset, project, and
|
|
61
|
+
* studio-origin trio moved to `.ailf/config.yaml`'s `source` block (with
|
|
62
|
+
* `SANITY_DATASET` / `SANITY_PROJECT_ID` / `SANITY_STUDIO_ORIGIN` env-var
|
|
63
|
+
* fallbacks). `ailf run` uses `addSanityScopeOptions` instead, which keeps
|
|
64
|
+
* only the per-run flags (`--sanity-perspective`, `--sanity-document`).
|
|
51
65
|
*/
|
|
52
66
|
export function addSanitySourceOptions(cmd) {
|
|
53
67
|
return cmd
|
|
@@ -55,8 +69,18 @@ export function addSanitySourceOptions(cmd) {
|
|
|
55
69
|
.option("--sanity-project <id>", "Override Sanity project ID")
|
|
56
70
|
.option("--sanity-perspective <id>", "Sanity release perspective ID")
|
|
57
71
|
.option("--sanity-studio-origin <url>", "Sanity Studio base URL")
|
|
58
|
-
.option("--sanity-document <id>", "Evaluate specific Sanity document(s) (repeatable)", collect, [])
|
|
59
|
-
|
|
72
|
+
.option("--sanity-document <id>", "Evaluate specific Sanity document(s) (repeatable)", collect, []);
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Add the per-run Sanity-scope CLI surface — `--sanity-perspective` and
|
|
76
|
+
* `--sanity-document`. Used by `ailf run`. The per-environment trio
|
|
77
|
+
* (dataset, project, studio origin) lives in `.ailf/config.yaml`'s
|
|
78
|
+
* `source` block instead (W0077 Phase 6d).
|
|
79
|
+
*/
|
|
80
|
+
export function addSanityScopeOptions(cmd) {
|
|
81
|
+
return cmd
|
|
82
|
+
.option("--sanity-perspective <id>", "Sanity release perspective ID")
|
|
83
|
+
.option("--sanity-document <id>", "Evaluate specific Sanity document(s) (repeatable)", collect, []);
|
|
60
84
|
}
|
|
61
85
|
/**
|
|
62
86
|
* Collect repeatable string options into an array.
|
|
@@ -21,7 +21,7 @@ import { detectLegacyFieldNames, parseCanonicalTaskFile, } from "../adapters/tas
|
|
|
21
21
|
import { validateCanonicalTasks, formatValidationResult, } from "../adapters/task-sources/repo-validation.js";
|
|
22
22
|
import { discoverTsTaskFiles, loadTsTaskFile, } from "../adapters/task-sources/task-file-loader.js";
|
|
23
23
|
export function createValidateTasksCommand() {
|
|
24
|
-
return new Command("
|
|
24
|
+
return new Command("tasks")
|
|
25
25
|
.description("Validate task files (YAML and TypeScript) in .ailf/tasks/ against the canonical schema")
|
|
26
26
|
.argument("[path]", "Path to tasks directory (default: .ailf/tasks/)", ".ailf/tasks")
|
|
27
27
|
.option("--strict", "Treat warnings as errors", false)
|
|
@@ -10,8 +10,8 @@ import { dirname, resolve } from "path";
|
|
|
10
10
|
import { fileURLToPath } from "url";
|
|
11
11
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
12
12
|
const ROOT = resolve(__dirname, "..", "..");
|
|
13
|
-
export function
|
|
14
|
-
return new Command("
|
|
13
|
+
export function createValidateConfigCommand() {
|
|
14
|
+
return new Command("config")
|
|
15
15
|
.description("Validate pipeline configuration")
|
|
16
16
|
.option("--strict", "Treat warnings as errors", false)
|
|
17
17
|
.option("--contexts", "Check that context files exist", false)
|
|
@@ -8,11 +8,11 @@
|
|
|
8
8
|
*/
|
|
9
9
|
import { Command } from "commander";
|
|
10
10
|
export function createWeeklyDigestCommand() {
|
|
11
|
-
return new Command("
|
|
11
|
+
return new Command("digest")
|
|
12
12
|
.description("Generate and deliver a weekly evaluation digest")
|
|
13
13
|
.option("-n, --dry-run", "Print to stdout only, do not send to Slack", false)
|
|
14
14
|
.option("--lookback <days>", "Lookback window in days (default: from config or 7)", parseInt)
|
|
15
|
-
.option("--
|
|
15
|
+
.option("-f, --format <fmt>", "Output format: console or json", "console")
|
|
16
16
|
.action(async (opts) => {
|
|
17
17
|
const { generateDigest } = await import("../schedules/digest.js");
|
|
18
18
|
const { getDigestConfig } = await import("../schedules/loader.js");
|
|
@@ -45,7 +45,7 @@ export function createWeeklyDigestCommand() {
|
|
|
45
45
|
console.log(` Regressed: ${digest.regressed.join(", ") || "none"}`);
|
|
46
46
|
console.log(` Stable: ${digest.stable.join(", ") || "none"}`);
|
|
47
47
|
console.log();
|
|
48
|
-
if (opts.json) {
|
|
48
|
+
if (opts.format === "json") {
|
|
49
49
|
console.log(JSON.stringify(digest, null, 2));
|
|
50
50
|
return;
|
|
51
51
|
}
|
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
* thresholds.ts — Quality thresholds for readiness gates and regression alerts.
|
|
3
3
|
*
|
|
4
4
|
* Used by:
|
|
5
|
-
* - `npx @sanity/ailf
|
|
6
|
-
* - `npx @sanity/ailf
|
|
7
|
-
* - `npx @sanity/ailf
|
|
5
|
+
* - `npx @sanity/ailf report readiness` (launch readiness checklist)
|
|
6
|
+
* - `npx @sanity/ailf run --publish` (severity-aware sink routing)
|
|
7
|
+
* - `npx @sanity/ailf run --compare` (regression alerting)
|
|
8
8
|
*
|
|
9
9
|
* @see docs/archive/exec-plans/scenario-matrix-implementation/phase-5-readiness-thresholds.md
|
|
10
10
|
*/
|
|
@@ -44,8 +44,6 @@ export function mapToResolvedConfig(opts, rootDir) {
|
|
|
44
44
|
compareThreshold: opts.compareThreshold,
|
|
45
45
|
compareBaseline: opts.compareBaseline,
|
|
46
46
|
gapAnalysisEnabled: opts.gapAnalysisEnabled,
|
|
47
|
-
readinessEnabled: opts.readinessEnabled,
|
|
48
|
-
discoveryReportEnabled: opts.discoveryReportEnabled,
|
|
49
47
|
publishEnabled: opts.publishEnabled,
|
|
50
48
|
publishTag: opts.publishTag,
|
|
51
49
|
noCache: opts.noCache,
|
|
@@ -9,14 +9,12 @@ import { LiteracyVariant } from "../pipeline/normalize-mode.js";
|
|
|
9
9
|
import { CallbackStep } from "./steps/callback-step.js";
|
|
10
10
|
import { CalculateScoresStep } from "./steps/calculate-scores-step.js";
|
|
11
11
|
import { CompareStep } from "./steps/compare-step.js";
|
|
12
|
-
import { DiscoveryReportStep } from "./steps/discovery-report-step.js";
|
|
13
12
|
import { FetchDocsStep } from "./steps/fetch-docs-step.js";
|
|
14
13
|
import { FinalizeRunStep } from "./steps/finalize-run-step.js";
|
|
15
14
|
import { GapAnalysisStep } from "./steps/gap-analysis-step.js";
|
|
16
15
|
import { GenerateConfigsStep } from "./steps/generate-configs-step.js";
|
|
17
16
|
import { GraderConsistencyStep } from "./steps/grader-consistency-step.js";
|
|
18
17
|
import { PublishReportStep } from "./steps/publish-report-step.js";
|
|
19
|
-
import { ReadinessStep } from "./steps/readiness-step.js";
|
|
20
18
|
import { ReportStep } from "./steps/report-step.js";
|
|
21
19
|
import { RunEvalStep } from "./steps/run-eval-step.js";
|
|
22
20
|
import { MirrorRepoTasksStep } from "./steps/mirror-repo-tasks-step.js";
|
|
@@ -93,15 +91,7 @@ export function buildStepSequence(ctx, pipelineStart = Date.now()) {
|
|
|
93
91
|
if (config.compareEnabled) {
|
|
94
92
|
steps.push(new CompareStep());
|
|
95
93
|
}
|
|
96
|
-
// Step
|
|
97
|
-
if (config.readinessEnabled) {
|
|
98
|
-
steps.push(new ReadinessStep());
|
|
99
|
-
}
|
|
100
|
-
// Step 6c: Discovery report (optional)
|
|
101
|
-
if (config.discoveryReportEnabled) {
|
|
102
|
-
steps.push(new DiscoveryReportStep());
|
|
103
|
-
}
|
|
104
|
-
// Step 7: Callback delivery (optional, API-triggered evaluations)
|
|
94
|
+
// Step 6: Callback delivery (optional, API-triggered evaluations)
|
|
105
95
|
if (config.callback?.url) {
|
|
106
96
|
steps.push(new CallbackStep(config.callback, config.jobId));
|
|
107
97
|
}
|
|
@@ -27,7 +27,7 @@ export class FetchDocsStep {
|
|
|
27
27
|
}
|
|
28
28
|
async execute(ctx, state) {
|
|
29
29
|
if (ctx.config.skipFetch) {
|
|
30
|
-
return { status: "skipped", reason: "--
|
|
30
|
+
return { status: "skipped", reason: "--no-fetch" };
|
|
31
31
|
}
|
|
32
32
|
const start = Date.now();
|
|
33
33
|
// Load tasks — use the same source as GenerateConfigsStep to avoid
|
|
@@ -6,14 +6,12 @@
|
|
|
6
6
|
*/
|
|
7
7
|
export { CalculateScoresStep } from "./calculate-scores-step.js";
|
|
8
8
|
export { CompareStep } from "./compare-step.js";
|
|
9
|
-
export { DiscoveryReportStep } from "./discovery-report-step.js";
|
|
10
9
|
export { FetchDocsStep } from "./fetch-docs-step.js";
|
|
11
10
|
export { GapAnalysisStep } from "./gap-analysis-step.js";
|
|
12
11
|
export { GenerateConfigsStep } from "./generate-configs-step.js";
|
|
13
12
|
export { MirrorRepoTasksStep } from "./mirror-repo-tasks-step.js";
|
|
14
13
|
export { GraderConsistencyStep } from "./grader-consistency-step.js";
|
|
15
14
|
export { PublishReportStep } from "./publish-report-step.js";
|
|
16
|
-
export { ReadinessStep } from "./readiness-step.js";
|
|
17
15
|
export { ReportStep } from "./report-step.js";
|
|
18
16
|
export { RunEvalStep } from "./run-eval-step.js";
|
|
19
17
|
export { ValidateStep } from "./validate-step.js";
|
|
@@ -6,14 +6,12 @@
|
|
|
6
6
|
*/
|
|
7
7
|
export { CalculateScoresStep } from "./calculate-scores-step.js";
|
|
8
8
|
export { CompareStep } from "./compare-step.js";
|
|
9
|
-
export { DiscoveryReportStep } from "./discovery-report-step.js";
|
|
10
9
|
export { FetchDocsStep } from "./fetch-docs-step.js";
|
|
11
10
|
export { GapAnalysisStep } from "./gap-analysis-step.js";
|
|
12
11
|
export { GenerateConfigsStep } from "./generate-configs-step.js";
|
|
13
12
|
export { MirrorRepoTasksStep } from "./mirror-repo-tasks-step.js";
|
|
14
13
|
export { GraderConsistencyStep } from "./grader-consistency-step.js";
|
|
15
14
|
export { PublishReportStep } from "./publish-report-step.js";
|
|
16
|
-
export { ReadinessStep } from "./readiness-step.js";
|
|
17
15
|
export { ReportStep } from "./report-step.js";
|
|
18
16
|
export { RunEvalStep } from "./run-eval-step.js";
|
|
19
17
|
export { ValidateStep } from "./validate-step.js";
|
|
@@ -25,7 +25,7 @@ export class RunEvalStep {
|
|
|
25
25
|
}
|
|
26
26
|
async execute(ctx, state) {
|
|
27
27
|
if (ctx.config.skipEval) {
|
|
28
|
-
return { status: "skipped", reason: "--
|
|
28
|
+
return { status: "skipped", reason: "--no-eval" };
|
|
29
29
|
}
|
|
30
30
|
const start = Date.now();
|
|
31
31
|
const { rootDir, debug, concurrency, noCache } = ctx.config;
|
package/dist/pipeline/cache.d.ts
CHANGED
|
@@ -48,7 +48,7 @@ export interface CacheStats {
|
|
|
48
48
|
hits: number;
|
|
49
49
|
/** Steps where cache was missed (executed normally) */
|
|
50
50
|
misses: number;
|
|
51
|
-
/** Steps that were skipped for other reasons (--
|
|
51
|
+
/** Steps that were skipped for other reasons (--no-fetch, etc.) */
|
|
52
52
|
skipped: number;
|
|
53
53
|
/** Per-step detail */
|
|
54
54
|
steps: Record<string, "disabled" | "hit" | "miss" | "skipped">;
|
|
@@ -47,8 +47,6 @@ export function mapRequestToConfig(request, rootDir) {
|
|
|
47
47
|
compareThreshold: request.compareThreshold,
|
|
48
48
|
compareBaseline: request.compareBaseline,
|
|
49
49
|
gapAnalysisEnabled: request.gapAnalysis ?? true,
|
|
50
|
-
readinessEnabled: request.readiness ?? false,
|
|
51
|
-
discoveryReportEnabled: request.discoveryReport ?? false,
|
|
52
50
|
publishEnabled: request.publish ?? publishDefault,
|
|
53
51
|
publishTag: request.publishTag,
|
|
54
52
|
noAutoScope: request.noAutoScope ?? false,
|
|
@@ -107,7 +107,7 @@ export declare function buildMirrorDocument(task: LiteracyTaskDefinition, opts:
|
|
|
107
107
|
slugToDocId: Map<string, string>;
|
|
108
108
|
}): {
|
|
109
109
|
baseline?: {
|
|
110
|
-
rubric?: "
|
|
110
|
+
rubric?: "abbreviated" | "full" | "none" | undefined;
|
|
111
111
|
enabled?: boolean | undefined;
|
|
112
112
|
} | undefined;
|
|
113
113
|
_id: string;
|