@sanity/ailf 4.0.7 → 4.1.0

package/dist/_vendor/ailf-core/schemas/report.js

@@ -0,0 +1,235 @@
+/**
+ * @sanity/ailf-core — Report schema (W0191)
+ *
+ * Runtime Zod gate for `ailf.report` documents at the ReportStore
+ * read/write boundary. Mirrors the W0073 pattern that
+ * `ContentLakeAuthorableTaskSchema` applies to `ailf.task` — turn silent
+ * shape drift into a loud parse failure at the Content Lake boundary.
+ *
+ * **Scope A** (this file): top-level shape is `passthrough()` so that
+ * adding a new top-level Report field on the TS side does not silently
+ * fail validation before the schema author updates this mirror.
+ * `provenance` (and nested `provenance.lineage`) are `strict()` because
+ * they are fully mirrored — unknown keys there signal real drift.
+ * `summary`, `comparison`, and `artifactManifest` are modeled as
+ * non-empty objects with passthrough; their deep contents
+ * (`ScoreSummary`, `FeatureScore`, the W0051 slim shapes, etc.) are
+ * intentionally out of scope and may grow into a Scope B follow-up.
+ *
+ * @see packages/eval/src/adapters/task-sources/content-lake-task-source.ts
+ *      — sibling W0073 gate
+ * @see packages/core/src/types/index.ts — `Report`, `ReportProvenance`,
+ *      `ReportLineage`
+ * @see packages/shared/src/run-context.ts — `RunContext`
+ * @see docs/work-items/W0191-report-store-schema-gate.json
+ */
+import { z } from "zod";
+// ---------------------------------------------------------------------------
+// RunContext building blocks (mirrors packages/shared/src/run-context.ts)
+// ---------------------------------------------------------------------------
+const RunOwnerSchema = z
+    .object({
+    team: z.string().min(1),
+    individual: z.string().optional(),
+})
+    .strict();
+const RunExecutorUserSchema = z
+    .object({
+    type: z.literal("user"),
+    name: z.string().optional(),
+    // `email` carries PII and is gated by AILF_CAPTURE_EMAIL on capture; it
+    // may legitimately appear here on stored documents that captured it.
+    email: z.string().optional(),
+    surface: z.enum(["cli", "studio", "api"]),
+    githubActor: z.string().optional(),
+})
+    .strict();
+const RunExecutorSystemSchema = z
+    .object({
+    type: z.literal("system"),
+    name: z.string().min(1),
+    workflow: z.string().optional(),
+    runId: z.string().optional(),
+})
+    .strict();
+const RunExecutorSchema = z.discriminatedUnion("type", [
+    RunExecutorUserSchema,
+    RunExecutorSystemSchema,
+]);
+const RunHostSchema = z
+    .object({
+    platform: z.string().min(1),
+    arch: z.string().min(1),
+    ci: z.string().optional(),
+})
+    .strict();
+const RunToolSchema = z
+    .object({
+    ailfVersion: z.string().min(1),
+    nodeVersion: z.string().min(1),
+})
+    .strict();
+const RunTriggerSchema = z.discriminatedUnion("type", [
+    z
+        .object({
+        type: z.literal("ci"),
+        runId: z.string().min(1),
+        workflow: z.string().min(1),
+    })
+        .strict(),
+    z
+        .object({
+        type: z.literal("cross-repo"),
+        callerRef: z.string().optional(),
+        callerRepo: z.string().min(1),
+    })
+        .strict(),
+    z.object({ type: z.literal("manual") }).strict(),
+    z
+        .object({
+        type: z.literal("scheduled"),
+        schedule: z.string().min(1),
+    })
+        .strict(),
+    z
+        .object({
+        type: z.literal("webhook"),
+        documentId: z.string().optional(),
+        source: z.string().min(1),
+    })
+        .strict(),
+]);
+const RunGitSchema = z
+    .object({
+    branch: z.string().min(1),
+    prNumber: z.number().optional(),
+    repo: z.string().min(1),
+    sha: z.string().min(1),
+})
+    .strict();
+const RunSourceSchema = z
+    .object({
+    baseUrl: z.string().min(1),
+    dataset: z.string().optional(),
+    name: z.string().min(1),
+    perspective: z.string().optional(),
+    projectId: z.string().optional(),
+})
+    .strict();
+const RunModelEntrySchema = z
+    .object({
+    id: z.string().min(1),
+    label: z.string().min(1),
+})
+    .strict();
+// ---------------------------------------------------------------------------
+// ReportLineage (mirrors `ReportLineage` in core types)
+//
+// The TS type narrows `RunContext.lineage` (which is `RunLineage` with
+// `parentJobId`) down to `{ rerunOf?, comparedAgainst? }`. In practice
+// `publish-report-step.ts` spreads existing lineage, so a stored
+// `provenance.lineage` may legitimately carry `parentJobId` inherited
+// from the run side. We model the runtime contract — strict, but
+// `parentJobId` allowed — so that legitimate data does not trip the gate.
+// ---------------------------------------------------------------------------
+export const ReportLineageSchema = z
+    .object({
+    rerunOf: z.string().optional(),
+    comparedAgainst: z.string().optional(),
+    parentJobId: z.string().optional(),
+})
+    .strict();
+// ---------------------------------------------------------------------------
+// ReportAutoScope (mirrors `ReportAutoScope` in core types)
+// ---------------------------------------------------------------------------
+const ReportAutoScopeSchema = z
+    .object({
+    enabled: z.boolean(),
+    affectedTaskIds: z.array(z.string()),
+    skippedTaskIds: z.array(z.string()),
+    perspective: z.string().min(1),
+    impactSummary: z
+        .object({
+        added: z.number(),
+        modified: z.number(),
+        removed: z.number(),
+    })
+        .strict(),
+})
+    .strict();
+// ---------------------------------------------------------------------------
+// PromptfooUrlEntry (mirrors `PromptfooUrlEntry` in core types)
+// ---------------------------------------------------------------------------
+const PromptfooUrlEntrySchema = z
+    .object({
+    mode: z.string().min(1),
+    url: z.string().min(1),
+})
+    .strict();
+// ---------------------------------------------------------------------------
+// ReportProvenance — full mirror of `ReportProvenance extends RunContext`
+// ---------------------------------------------------------------------------
+export const ReportProvenanceSchema = z
+    .object({
+    // RunContext fields
+    areas: z.array(z.string()),
+    classification: z.enum([
+        "official",
+        "adhoc",
+        "experimental",
+        "test",
+        "external",
+    ]),
+    evalFingerprint: z.string().optional(),
+    executor: RunExecutorSchema,
+    git: RunGitSchema.optional(),
+    graderModel: z.string().min(1),
+    host: RunHostSchema.optional(),
+    labels: z.array(z.string()).optional(),
+    lineage: ReportLineageSchema.optional(),
+    mode: z.string().min(1),
+    models: z.array(RunModelEntrySchema),
+    owner: RunOwnerSchema,
+    purpose: z.string().optional(),
+    source: RunSourceSchema,
+    taskIds: z.array(z.string()).optional(),
+    tool: RunToolSchema.optional(),
+    trigger: RunTriggerSchema,
+    // ReportProvenance additions
+    autoScope: ReportAutoScopeSchema.optional(),
+    contextHash: z.string().optional(),
+    promptfooUrl: z.string().optional(),
+    promptfooUrls: z.array(PromptfooUrlEntrySchema).optional(),
+    runId: z.string().min(1),
+    targetDocuments: z.array(z.string()).optional(),
+})
+    .strict();
+// ---------------------------------------------------------------------------
+// ReportSchema — top-level Report shape
+//
+// Top-level uses `.passthrough()`: adding a new top-level Report field on
+// the TS side should not break stored documents at the read boundary
+// before the schema author updates this mirror. `summary`, `comparison`,
+// and `artifactManifest` are also passthrough — their deep shapes
+// (ScoreSummary, FeatureScore, the W0051 slim types) are out of Scope A.
+// ---------------------------------------------------------------------------
+const RecordPassthroughSchema = z.record(z.string(), z.unknown());
+export const ReportSchema = z
+    .object({
+    id: z.string().min(1),
+    completedAt: z.iso.datetime({ offset: true }),
+    durationMs: z.number().nonnegative(),
+    summary: RecordPassthroughSchema.refine((s) => Object.keys(s).length > 0, {
+        message: "summary must be a non-empty object",
+    }),
+    provenance: ReportProvenanceSchema,
+    // The eval write path persists `comparison: null` when the slim copy
+    // is empty, so the schema accepts null at the wire boundary.
+    comparison: RecordPassthroughSchema.nullable().optional(),
+    artifactManifest: RecordPassthroughSchema.optional(),
+    // The eval write path stores `tag: report.tag ?? null` and
+    // `title: report.title ?? null`, so the schema accepts null on both.
+    tag: z.string().nullable().optional(),
+    title: z.string().nullable().optional(),
+})
+    .passthrough();

package/dist/_vendor/ailf-core/services/index.d.ts

@@ -12,3 +12,4 @@ export { formatComparisonMarkdown, formatComparisonTable, } from "./comparison-f
 export { aggregateAreas, aggregateDimensions, computeEnsembleScore, computeTaskScore, normalizeScore, type AggregationStrategy, type AreaScore, type AssertionScore, type DimensionScore, type EnsembleGradingConfig, type GraderTransitionConfig, type TaskScore, type TaskScoreOptions, } from "./scoring-engine.js";
 export { extractModelName, extractProvider, mergeConfig, modelMatchesMode, resolveModelVariants, } from "./config-helpers.js";
 export { buildSlimReportSummary } from "./slim-report-summary.js";
+export { reportToMarkdown, type RenderableReport, } from "./report-to-markdown.js";

package/dist/_vendor/ailf-core/services/index.js

@@ -12,3 +12,4 @@ export { formatComparisonMarkdown, formatComparisonTable, } from "./comparison-f
 export { aggregateAreas, aggregateDimensions, computeEnsembleScore, computeTaskScore, normalizeScore, } from "./scoring-engine.js";
 export { extractModelName, extractProvider, mergeConfig, modelMatchesMode, resolveModelVariants, } from "./config-helpers.js";
 export { buildSlimReportSummary } from "./slim-report-summary.js";
+export { reportToMarkdown, } from "./report-to-markdown.js";

package/dist/_vendor/ailf-core/services/report-to-markdown.d.ts

@@ -0,0 +1,38 @@
+/**
+ * report-to-markdown.ts — Canonical PR-comment markdown renderer (W0150).
+ *
+ * Single source of truth for rendering an AILF report as PR-comment
+ * markdown. Used by:
+ *   - the API gateway (`/v1/reports/:id/markdown`)
+ *   - the eval pipeline (`ailf pr-comment` CLI / `pipeline/pr-comment.ts`)
+ *
+ * Operates on a structurally lenient `RenderableReport` shape so callers
+ * can pass either the persisted slim Report (Sanity doc) or an in-memory
+ * envelope built from `score-summary.json` + `comparison-report.json`.
+ *
+ * Canonical formatting decisions (W0150):
+ *   - Header: level-1 (`# {emoji} AI Literacy Score Report`).
+ *   - Footer: markdown link form `[view detailed results](url)`. The link
+ *     URL comes from `provenance.promptfooUrls[0].url`.
+ *   - Source verification block (sourceVerification + sourceIsolation) is
+ *     rendered when present on the summary — preserves info from agentic
+ *     / sandboxed local-mode runs without breaking remote-mode reports
+ *     that don't carry those fields.
+ */
+export interface RenderableReport {
+    /** Sanity report id; absent for local-only runs. */
+    id?: string;
+    /** ISO timestamp; falls back to `summary.timestamp` when absent. */
+    completedAt?: string;
+    /** Run duration in ms; absent for local-only runs. */
+    durationMs?: number;
+    /** Run tag/label. */
+    tag?: string;
+    /** Slim or full summary. Structurally lenient. */
+    summary: unknown;
+    /** Comparison report (delta against a baseline). */
+    comparison?: unknown;
+    /** Provenance — surfaces promptfooUrls for the footer link. */
+    provenance?: unknown;
+}
+export declare function reportToMarkdown(report: RenderableReport): string;