@sanity/ailf 7.2.2 → 7.3.0

package/config/airbyte/ai_literacy_framework.connector.yaml

@@ -104,6 +104,11 @@ definitions:
                 "host_platform": provenance.host.platform,
                 "host_arch": provenance.host.arch,
                 "host_ci": provenance.host.ci,
+                "validity_status": validity.status,
+                "validity_method": validity.method,
+                "validity_ruleset_version": validity.rulesetVersion,
+                "validity_assessed_at": validity.assessedAt,
+                "validity_reasons": validity.reasons,
                 _createdAt
               }
         record_selector:
@@ -724,6 +729,39 @@ schemas:
           - string
           - "null"
         description: CI provider when running under one (e.g., github-actions).
+      # ----------------------------------------------------------------
+      # D0059 — report validity (data-health axis, orthogonal to intent)
+      # ----------------------------------------------------------------
+      validity_status:
+        type:
+          - string
+          - "null"
+        description:
+          "Data-health verdict (D0059): ok | degraded | incomplete | suspect.
+          NULL for reports predating the validity stamp (treated as trusted)."
+      validity_method:
+        type:
+          - string
+          - "null"
+        description: '"auto" | "manual" — how the validity verdict was reached.'
+      validity_ruleset_version:
+        type:
+          - string
+          - "null"
+        description: Detector ruleset version, so re-assessments are comparable.
+      validity_assessed_at:
+        type:
+          - string
+          - "null"
+        description:
+          ISO 8601 UTC timestamp when the validity verdict was produced.
+      validity_reasons:
+        type:
+          - array
+          - "null"
+        items:
+          type: string
+        description: Which detector rules fired — the audit trail behind status.
       _createdAt:
         type:
           - string

package/config/bigquery/README.md

@@ -22,13 +22,13 @@ BigQuery views (this directory)
 
 ## Files
 
-| File                             | Purpose                                                                                                             |
-| -------------------------------- | ------------------------------------------------------------------------------------------------------------------- |
-| `views/area_scores.sql`          | Flattens nested `model_scores` array into one row per area per model per report                                     |
-| `views/reports.sql`              | Clean passthrough view with correct types and column ordering                                                       |
-| `views/official_runs.sql`        | Canonical trend series (D0037): `classification='official' AND trigger_type='scheduled' AND owner_team='core-docs'` |
-| `views/official_area_scores.sql` | `area_scores` joined to `official_runs` — inherits the official-run predicate for area-level dashboards             |
-| `views/team_runs_template.sql`   | Recipe/template for instantiating per-team filtered views                                                           |
+| File                             | Purpose                                                                                                                                                      |
+| -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `views/area_scores.sql`          | Flattens nested `model_scores` array into one row per area per model per report                                                                              |
+| `views/reports.sql`              | Clean passthrough view with correct types and column ordering; also materializes `validity_*` columns and the `include_in_default_trends` trust gate (D0059) |
+| `views/official_runs.sql`        | Canonical trend series (D0037): `classification='official' AND trigger_type='scheduled' AND owner_team='core-docs'`                                          |
+| `views/official_area_scores.sql` | `area_scores` joined to `official_runs` — inherits the official-run predicate for area-level dashboards                                                      |
+| `views/team_runs_template.sql`   | Recipe/template for instantiating per-team filtered views                                                                                                    |
 
 ## Setup
 
@@ -88,6 +88,38 @@ bq --project_id=data-platform-302218 --location=EU query --use_legacy_sql=false
 > run `bq query` from this repo regularly, consider setting the default with
 > `gcloud config set project data-platform-302218`.
 
+## Querying default (trustworthy) reports
+
+`ailf.reports` exposes every synced report, including development runs and
+structurally incomplete ones. BI dashboards and ad-hoc analysis should gate on
+the **`include_in_default_trends`** boolean, which materializes the single
+trustworthiness predicate (D0059) shared with the dashboard and Studio — so all
+three surfaces show the same default population:
+
+```sql
+SELECT *
+FROM   `data-platform-302218.ailf.reports`
+WHERE  include_in_default_trends = TRUE;
+```
+
+`include_in_default_trends` is the SQL form of the `includeInDefaultTrends`
+predicate (`INCLUDE_IN_DEFAULT_TRENDS_SQL` in `@sanity/ailf-shared`):
+
+```text
+(validity_status IS NULL OR validity_status = 'ok')
+  AND (classification IS NULL OR classification NOT IN ('test', 'experimental'))
+```
+
+To audit the _excluded_ (untrusted) reports — the inverse population, useful for
+the Tier-2 review pass — flip the predicate to `= FALSE`. The `validity_status`
+/ `validity_reasons` columns explain why each was excluded.
+
+> The `official_runs` view (above) is a **stricter** canonical-series gate
+> (`classification='official' AND trigger_type='scheduled' AND owner_team='core-docs'`);
+> `include_in_default_trends` is the **broader** "trustworthy enough to show by
+> default" gate. Use `official_runs` for the tracked trend line,
+> `include_in_default_trends` for general default views.
+
 ## Naming conventions
 
 - **`ailf_raw.*`** — raw Airbyte-loaded tables (nested JSON, Airbyte metadata

package/config/bigquery/views/reports.sql

@@ -64,6 +64,12 @@ SELECT
   host_platform,
   host_arch,
   host_ci,
+  validity_status,
+  validity_method,
+  validity_ruleset_version,
+  TIMESTAMP(validity_assessed_at)                 AS validity_assessed_at,
+  validity_reasons,
+  (validity_status IS NULL OR validity_status = 'ok') AND (classification IS NULL OR classification NOT IN ('test', 'experimental')) AS include_in_default_trends,
   TIMESTAMP(_createdAt)                           AS synced_at
 FROM
   `data-platform-302218.ailf_raw.reports`;

package/dist/_vendor/ailf-core/ports/artifact-writer.d.ts

@@ -76,6 +76,28 @@ export interface ArtifactWriter {
      */
     writePerEntry(type: ArtifactType, runId: RunId, entries: readonly ArtifactEntry[]): Promise<ArtifactRef | null>;
 }
+/**
+ * Optional capability: check whether an object physically exists at a
+ * store-relative `path`. Kept **separate** from `ArtifactWriter` so writers
+ * opt in by structural typing rather than every implementor being forced to
+ * provide it — consumers feature-detect it (see the writer-chain walkers in
+ * the eval package).
+ *
+ * Used by the cache-hit restore path (D0040 / D0057) to drop artifact refs
+ * a prior report over-claimed: a degraded source run can list `rawResults`
+ * entries whose GCS objects were never written, and `remapToCacheHitRefs`
+ * would otherwise copy those phantom entries into the new run's manifest.
+ *
+ * Contract:
+ *   - resolves `true`  — the object exists.
+ *   - resolves `false` — the object is *definitively* absent.
+ *   - **throws** — existence could not be determined (auth / network /
+ *     transient error). Callers must treat a throw as "unknown" and
+ *     fail open (keep the ref) rather than dropping a real artifact.
+ */
+export interface ArtifactObjectChecker {
+    objectExists(path: string): Promise<boolean>;
+}
 /**
  * Thrown by writers that can't satisfy a method — e.g. an
  * `ApiGatewayArtifactWriter` cannot implement `appendNdjson` until the batch

package/dist/_vendor/ailf-core/ports/index.d.ts

@@ -4,7 +4,7 @@
  * Ports define the contracts between the domain kernel and the outside world.
  * Adapters (in packages/eval) implement these interfaces.
  */
-export type { ArtifactEntry, ArtifactWriter } from "./artifact-writer.js";
+export type { ArtifactEntry, ArtifactObjectChecker, ArtifactWriter, } from "./artifact-writer.js";
 export { NoOpArtifactWriter } from "./artifact-writer.js";
 export type { CacheEntryMetadata, CacheKey, CacheLookupResult, CacheRecordInput, CacheStore, } from "./cache-store.js";
 export type { ConfigSource } from "./config-source.js";

package/dist/_vendor/ailf-core/schemas/report.d.ts

@@ -139,6 +139,21 @@ export declare const ReportProvenanceSchema: z.ZodObject<{
     runId: z.ZodString;
     targetDocuments: z.ZodOptional<z.ZodArray<z.ZodString>>;
 }, z.core.$strict>;
+export declare const ReportValiditySchema: z.ZodObject<{
+    status: z.ZodEnum<{
+        ok: "ok";
+        degraded: "degraded";
+        incomplete: "incomplete";
+        suspect: "suspect";
+    }>;
+    reasons: z.ZodArray<z.ZodString>;
+    method: z.ZodEnum<{
+        auto: "auto";
+        manual: "manual";
+    }>;
+    rulesetVersion: z.ZodString;
+    assessedAt: z.ZodISODateTime;
+}, z.core.$strict>;
 export declare const ReportSchema: z.ZodObject<{
     id: z.ZodString;
     completedAt: z.ZodISODateTime;
@@ -269,6 +284,21 @@ export declare const ReportSchema: z.ZodObject<{
         }>>;
         detail: z.ZodString;
     }, z.core.$strict>>;
+    validity: z.ZodOptional<z.ZodObject<{
+        status: z.ZodEnum<{
+            ok: "ok";
+            degraded: "degraded";
+            incomplete: "incomplete";
+            suspect: "suspect";
+        }>;
+        reasons: z.ZodArray<z.ZodString>;
+        method: z.ZodEnum<{
+            auto: "auto";
+            manual: "manual";
+        }>;
+        rulesetVersion: z.ZodString;
+        assessedAt: z.ZodISODateTime;
+    }, z.core.$strict>>;
 }, z.core.$loose>;
 export type ReportSchemaInput = z.input<typeof ReportSchema>;
 export type ReportSchemaOutput = z.infer<typeof ReportSchema>;

package/dist/_vendor/ailf-core/schemas/report.js

@@ -25,7 +25,7 @@
  */
 import { z } from "zod";
 import { LITERACY_VARIANTS } from "../../ailf-shared/index.js";
-import { DEGRADED_ENRICHMENT_FIELDS } from "../types/index.js";
+import { DEGRADED_ENRICHMENT_FIELDS, REPORT_VALIDITY_STATUSES, } from "../types/index.js";
 // ---------------------------------------------------------------------------
 // RunContext building blocks (mirrors packages/shared/src/run-context.ts)
 // ---------------------------------------------------------------------------
@@ -217,6 +217,21 @@ export const ReportProvenanceSchema = z
 // (ScoreSummary, FeatureScore, the W0051 slim types) are out of Scope A.
 // ---------------------------------------------------------------------------
 const RecordPassthroughSchema = z.record(z.string(), z.unknown());
+// ---------------------------------------------------------------------------
+// ReportValidity — top-level data-health axis (D0059), orthogonal to
+// `provenance.classification`. `satisfies z.ZodType<ReportValidity>` (D0045):
+// the domain type is authored independently in ../types/report-validity.ts,
+// so schema/type drift is a build error. Strict — unknown keys signal drift.
+// ---------------------------------------------------------------------------
+export const ReportValiditySchema = z
+    .object({
+    status: z.enum(REPORT_VALIDITY_STATUSES),
+    reasons: z.array(z.string()),
+    method: z.enum(["auto", "manual"]),
+    rulesetVersion: z.string().min(1),
+    assessedAt: z.iso.datetime({ offset: true }),
+})
+    .strict();
 export const ReportSchema = z
     .object({
     id: z.string().min(1),
@@ -236,7 +251,8 @@ export const ReportSchema = z
     title: z.string().nullable().optional(),
     // Degraded marker (mirrors `ReportDegradation`): present only when a full
     // eval scored tests but enrichment did not complete. Strict — unknown
-    // keys here signal real drift.
+    // keys here signal real drift. Superseded by `validity` (D0059); retained
+    // for back-compat reads until the backfill migrates it.
     degraded: z
         .object({
         reason: z.literal("enrichment-missing"),
@@ -247,5 +263,8 @@ export const ReportSchema = z
     })
         .strict()
         .optional(),
+    // Data-health axis (D0059). Additive + nullable: pre-stamp reads have no
+    // `validity` and are treated as trustworthy until backfilled.
+    validity: ReportValiditySchema.optional(),
 })
     .passthrough();

package/dist/_vendor/ailf-core/services/diagnosis/cards/top-recommendations.js

@@ -59,6 +59,20 @@ export const generateTopRecommendations = async (report, ctx) => {
     }
     // Build allow-list from the runtime report
     const allowList = buildDocSlugAllowList(report);
+    // Short-circuit BEFORE the LLM call when the allow-list is empty — the shape
+    // a degraded run produces (enrichment never wrote `documentManifest`, and
+    // per-score `documents[]` carry no slugs). With an empty allow-list the
+    // system prompt instructs the model to omit every recommendation, so it
+    // returns `suggestions: []` and trips the `.min(1)` schema, surfacing as a
+    // degraded card with a raw ZodError. Degrade gracefully to "missing"
+    // instead — same structural seam the sibling attribution cards use.
+    if (allowList.size === 0) {
+        return {
+            status: "missing",
+            cardType: "top-recommendations",
+            reason: "no doc-slug allow-list for this run (enrichment incomplete)",
+        };
+    }
     // Per-call schema: additive docSlug allow-list refine (AI-SPEC §3 Pitfall 1)
     const PerCallSchema = z.object({
         summary: z.string().min(1).max(500),

package/dist/_vendor/ailf-core/services/index.d.ts

@@ -13,6 +13,7 @@ export { aggregateAreas, aggregateDimensions, computeEnsembleScore, computeTaskS
 export { extractModelName, extractProvider, mergeConfig, modelMatchesMode, resolveModelVariants, } from "./config-helpers.js";
 export { buildSlimReportSummary } from "./slim-report-summary.js";
 export { reportToMarkdown, type RenderableReport, } from "./report-to-markdown.js";
+export { assessReportValidity, REPORT_VALIDITY_RULESET_VERSION, type AssessReportValidityOptions, type ReportValidityAssessment, type ReportValidityInput, type ReportValidityReviewFlag, type ReportValidityReviewReason, } from "./report-validity-detector.js";
 export { createDiagnosisRunner, diagnosisVersion, type CardGenerator, type CardRegistry, type DiagnosisRunner, type DiagnosisRunnerDeps, type DiagnosisRunnerRunArgs, type GeneratorContext, } from "./diagnosis-runner.js";
 export { cardRegistry, type CardDefinition } from "./diagnosis/registry.js";
 export { createLLMClient, type LLMClientAdapters, type LLMClientFactoryConfig, type LLMClientKeys, } from "./llm-client-factory.js";

package/dist/_vendor/ailf-core/services/index.js

@@ -14,6 +14,10 @@ export { extractModelName, extractProvider, mergeConfig, modelMatchesMode, resol
 export { buildSlimReportSummary } from "./slim-report-summary.js";
 export { reportToMarkdown, } from "./report-to-markdown.js";
 // ---------------------------------------------------------------------------
+// Report-validity detector (D0059) — pure, versioned trustworthiness engine
+// ---------------------------------------------------------------------------
+export { assessReportValidity, REPORT_VALIDITY_RULESET_VERSION, } from "./report-validity-detector.js";
+// ---------------------------------------------------------------------------
 // Actionability ladder Phase 1 + Phase 5 — diagnosis runner + card registry
 // ---------------------------------------------------------------------------
 export { createDiagnosisRunner, diagnosisVersion, } from "./diagnosis-runner.js";

package/dist/_vendor/ailf-core/services/report-validity-detector.d.ts

@@ -0,0 +1,116 @@
+/**
+ * @sanity/ailf-core — Report-validity detector (D0059)
+ *
+ * The detection brain for the report trustworthiness model. A pure, versioned,
+ * I/O-free rules engine that computes both trustworthiness axes for a report —
+ * intent (`classification`, D0037) and data health (`validity`) — with a
+ * confidence tier so it never mislabels ambiguous data.
+ *
+ * Reused unchanged by the eval write path (`W-stamp-validity-write-path`,
+ * stamps every new report) and the one-shot backfill
+ * (`W-backfill-report-validity`, labels the historical store).
+ *
+ * **Confidence tiers (D0059 §2):**
+ *
+ *   - **Tier 1 — auto.** High-confidence, unambiguous rules. Intent:
+ *     `scheduled + github-actions → official`; `testing-ground` team /
+ *     `env-override` source / generated-id executor → `test`/`experimental`;
+ *     de-drift `ad-hoc → adhoc`. Validity: `degraded → "degraded"`.
+ *   - **Tier 2 — review list.** Anything resting on shape-sensitive content
+ *     signals (empty `testResults`, `testCount` mismatch) is *emitted to a
+ *     review list rather than guessed* — 34%/66% of the historical store hits
+ *     these and is dominated by report-shape evolution (GCS-externalized
+ *     per-test data, per-model granularity), NOT defects. The engine never
+ *     auto-flags them invalid; a human confirms in batch and the verdict is
+ *     written `method:"manual"`.
+ *
+ * A `method:"manual"` verdict is authoritative — re-running the detector never
+ * overwrites it.
+ *
+ * @see docs/decisions/D0059-report-validity-axis-and-trustworthiness-gate.md
+ * @see docs/design-docs/report-trustworthiness-model.md
+ */
+import { type RunClassification, type RunExecutor, type RunOwner, type RunTrigger } from "../../ailf-shared/index.d.ts";
+import type { ReportDegradation, ReportValidity } from "../types/index.js";
+/**
+ * Detector ruleset version. Bump on any rule or threshold recalibration so a
+ * later re-assessment is a comparable, auditable re-run (D0059 §2). Stamped
+ * onto every `validity` verdict the engine produces.
+ */
+export declare const REPORT_VALIDITY_RULESET_VERSION = "1.0.0";
+/**
+ * The minimal report projection the detector reads.
+ *
+ * Authored as a raw-tolerant *consumer view* (not `Pick<Report, …>`) on
+ * purpose: the engine runs at the raw-document boundary during the backfill,
+ * where legacy drift like `ad-hoc` still exists and `provenance.classification`
+ * is therefore typed `string`, not `RunClassification`. A fully-typed `Report`
+ * is structurally assignable to this shape, so the write path can pass one
+ * directly. Only the seven fields the rules actually consult appear here.
+ */
+export interface ReportValidityInput {
+    provenance: {
+        /** Raw classification — may carry legacy drift (`ad-hoc`); normalized here. */
+        classification?: string;
+        trigger: RunTrigger;
+        executor: RunExecutor;
+        owner: RunOwner;
+        source: {
+            name: string;
+        };
+    };
+    summary: {
+        /** Per-feature scores; only `testCount` is read (summed across features). */
+        scores?: ReadonlyArray<{
+            testCount?: number;
+        }>;
+        /** Per-test×model results; only the length is read. */
+        testResults?: ReadonlyArray<unknown>;
+        /** Low-scoring grader judgments; only the length is read. */
+        lowScoringJudgments?: ReadonlyArray<unknown>;
+    };
+    /** Legacy degradation marker (D0059 subsumes it into `validity.status`). */
+    degraded?: ReportDegradation;
+    /** Existing verdict — a `method:"manual"` value is preserved on re-run. */
+    validity?: ReportValidity;
+}
+/**
+ * Why a report was routed to Tier-2 human review instead of being
+ * auto-decided. Each corresponds to a shape-sensitive signal the engine
+ * deliberately refuses to interpret automatically (D0059 §caveat).
+ */
+export type ReportValidityReviewReason = "empty-test-results" | "test-count-mismatch";
+/** A single Tier-2 review-list entry. */
+export interface ReportValidityReviewFlag {
+    reason: ReportValidityReviewReason;
+    /** Human-readable explanation for the batch-review UI / backfill report. */
+    detail: string;
+}
+/**
+ * The detector's verdict for one report.
+ *
+ * `classification` is **optional**: present only when the engine has a
+ * positive opinion (a Tier-1 intent rule fired, or de-drift changed the
+ * value). It is `undefined` when the existing classification is already
+ * canonical and no rule applies — so callers patch idempotently and a re-run
+ * never clobbers an already-correct or human-corrected value.
+ */
+export interface ReportValidityAssessment {
+    classification?: RunClassification;
+    validity: ReportValidity;
+    reviewFlags: ReportValidityReviewFlag[];
+}
+export interface AssessReportValidityOptions {
+    /**
+     * ISO 8601 UTC timestamp stamped onto the verdict's `assessedAt`. Injected
+     * by the caller so the engine stays pure and deterministic (no clock read).
+     */
+    assessedAt: string;
+}
+/**
+ * Assess a report's trustworthiness — its intent (`classification`) and data
+ * health (`validity`) — plus any Tier-2 signals that need human review.
+ *
+ * Pure and deterministic: same input + `assessedAt` → identical output.
+ */
+export declare function assessReportValidity(report: ReportValidityInput, options: AssessReportValidityOptions): ReportValidityAssessment;

package/dist/_vendor/ailf-core/services/report-validity-detector.js

@@ -0,0 +1,128 @@
+/**
+ * @sanity/ailf-core — Report-validity detector (D0059)
+ *
+ * The detection brain for the report trustworthiness model. A pure, versioned,
+ * I/O-free rules engine that computes both trustworthiness axes for a report —
+ * intent (`classification`, D0037) and data health (`validity`) — with a
+ * confidence tier so it never mislabels ambiguous data.
+ *
+ * Reused unchanged by the eval write path (`W-stamp-validity-write-path`,
+ * stamps every new report) and the one-shot backfill
+ * (`W-backfill-report-validity`, labels the historical store).
+ *
+ * **Confidence tiers (D0059 §2):**
+ *
+ *   - **Tier 1 — auto.** High-confidence, unambiguous rules. Intent:
+ *     `scheduled + github-actions → official`; `testing-ground` team /
+ *     `env-override` source / generated-id executor → `test`/`experimental`;
+ *     de-drift `ad-hoc → adhoc`. Validity: `degraded → "degraded"`.
+ *   - **Tier 2 — review list.** Anything resting on shape-sensitive content
+ *     signals (empty `testResults`, `testCount` mismatch) is *emitted to a
+ *     review list rather than guessed* — 34%/66% of the historical store hits
+ *     these and is dominated by report-shape evolution (GCS-externalized
+ *     per-test data, per-model granularity), NOT defects. The engine never
+ *     auto-flags them invalid; a human confirms in batch and the verdict is
+ *     written `method:"manual"`.
+ *
+ * A `method:"manual"` verdict is authoritative — re-running the detector never
+ * overwrites it.
+ *
+ * @see docs/decisions/D0059-report-validity-axis-and-trustworthiness-gate.md
+ * @see docs/design-docs/report-trustworthiness-model.md
+ */
+import { isKnownExecutorIdentity, looksLikeGeneratedExecutorId, normalizeOwnerTeam, normalizeRunClassification, } from "../../ailf-shared/index.js";
+/**
+ * Detector ruleset version. Bump on any rule or threshold recalibration so a
+ * later re-assessment is a comparable, auditable re-run (D0059 §2). Stamped
+ * onto every `validity` verdict the engine produces.
+ */
+export const REPORT_VALIDITY_RULESET_VERSION = "1.0.0";
+/**
+ * Tier-1 intent detection. Returns a `RunClassification` only when the engine
+ * has a positive opinion; `undefined` means "leave the existing value".
+ */
+function detectIntent(provenance) {
+    const { classification, executor, owner, source, trigger } = provenance;
+    // Highest-confidence positive: a scheduled run executed by GitHub Actions is
+    // the canonical official series.
+    if (trigger.type === "scheduled" &&
+        executor.type === "system" &&
+        executor.name === "github-actions") {
+        return "official";
+    }
+    // Explicit non-canonical contexts.
+    if (normalizeOwnerTeam(owner.team) === "testing-ground")
+        return "test";
+    if (source.name === "env-override")
+        return "experimental";
+    if (executor.type === "user" &&
+        !isKnownExecutorIdentity(executor.name) &&
+        looksLikeGeneratedExecutorId(executor.name)) {
+        return "experimental";
+    }
+    // No rule fired — de-drift the existing value, returning it only when it
+    // actually changes.
+    const normalized = normalizeRunClassification(classification);
+    return normalized === classification ? undefined : normalized;
+}
+/**
+ * Tier-2 review flags. Surfaces shape-sensitive signals for human batch-review
+ * — it never decides validity itself. Skipped entirely for already-`degraded`
+ * reports, where missing/empty test data is already explained.
+ */
+function collectReviewFlags(summary) {
+    const flags = [];
+    const testResultCount = summary.testResults?.length ?? 0;
+    const summedTestCount = (summary.scores ?? []).reduce((total, score) => total + (score.testCount ?? 0), 0);
+    if (summedTestCount > 0 && testResultCount === 0) {
+        flags.push({
+            reason: "empty-test-results",
+            detail: `${summedTestCount} scored tests but testResults is empty — may be GCS-externalized (W0329) rather than a defect; needs human review.`,
+        });
+    }
+    else if (testResultCount > 0 &&
+        summedTestCount > 0 &&
+        testResultCount !== summedTestCount) {
+        flags.push({
+            reason: "test-count-mismatch",
+            detail: `testResults length (${testResultCount}) ≠ summed testCount (${summedTestCount}) — per-model/sampling granularity difference, not necessarily a defect.`,
+        });
+    }
+    return flags;
+}
+/**
+ * Assess a report's trustworthiness — its intent (`classification`) and data
+ * health (`validity`) — plus any Tier-2 signals that need human review.
+ *
+ * Pure and deterministic: same input + `assessedAt` → identical output.
+ */
+export function assessReportValidity(report, options) {
+    const classification = detectIntent(report.provenance);
+    // A human verdict is authoritative — never overwrite a manual data-health
+    // assessment on a re-run. Intent normalization is still surfaced: it is
+    // idempotent and orthogonal to the data-health review.
+    if (report.validity?.method === "manual") {
+        return { classification, validity: report.validity, reviewFlags: [] };
+    }
+    // Tier-1 validity: the only auto-assigned non-`ok` status is `degraded`
+    // (subsumes the legacy flag). `incomplete`/`suspect` rest on the shape
+    // caveat and require human review — they are written `method:"manual"`.
+    const reasons = [];
+    let status = "ok";
+    if (report.degraded) {
+        status = "degraded";
+        reasons.push(`degraded:${report.degraded.reason}`);
+    }
+    const validity = {
+        status,
+        reasons,
+        method: "auto",
+        rulesetVersion: REPORT_VALIDITY_RULESET_VERSION,
+        assessedAt: options.assessedAt,
+    };
+    return {
+        classification,
+        validity,
+        reviewFlags: status === "degraded" ? [] : collectReviewFlags(report.summary),
+    };
+}

package/dist/_vendor/ailf-core/types/index.d.ts

@@ -14,6 +14,7 @@ import type { ArtifactType } from "../artifact-registry.js";
 import type { SymbolPreflightReport } from "./symbol-preflight-report.js";
 import type { AssociationValues, RunId } from "./branded-ids.js";
 import type { GraderJudgment } from "./grader-judgment.js";
+import type { ReportValidity } from "./report-validity.js";
 export type { ActualScoreEntry, ComponentResult, TestResult, UrlMetadata, } from "./scoring-input.js";
 export type { DocumentRef, RunContext, RunTrigger } from "../../ailf-shared/index.d.ts";
 export type { StoredBaseline, StoredReport, StoredRun, StoredTaskResult, StoredTrace, SchemaVersioned, } from "./storage-schema.js";
@@ -43,6 +44,7 @@ export type { CriterionSubJudgment, DocCitation, DocCitationRole, GraderEmittedJ
 export type { LegacyGraderJudgment } from "./legacy-grader-judgment.js";
 export type { BaseChannel, ChannelScope, EmailChannel, EventType, KnownEventType, KnownMemberRole, MemberRole, NotificationChannel, NotificationChannelType, SlackChannel, Team, TeamId, TeamMember, TeamRef, TeamSlug, TeamStatus, WebhookChannel, } from "./team.js";
 export type { AilfUser, AilfUserPreferences, TeamReference } from "./user.js";
+export { isReportValidityStatus, REPORT_VALIDITY_STATUSES, type ReportValidity, type ReportValidityMethod, type ReportValidityStatus, } from "./report-validity.js";
 type DocumentRef = _DocumentRef;
 /** Aggregated retrieval metrics for a feature area */
 export interface AreaRetrievalMetrics {
@@ -1503,6 +1505,12 @@ export type DegradedEnrichmentField = (typeof DEGRADED_ENRICHMENT_FIELDS)[number
  * because `grader-judgments.json` was missing). Present so the dashboard and
  * Studio can show "enrichment failed" rather than a misleading empty
  * "no tests" state on a report that still has a score.
+ *
+ * @deprecated Superseded by {@link ReportValidity} (D0059). The data-health
+ * axis subsumes this flag: a degraded report is `validity.status:"degraded"`.
+ * Retained for back-compat reads; the backfill migrates `degraded:true →
+ * validity.status:"degraded"` (`W-backfill-report-validity`). Do not set it on
+ * new reports — stamp `validity` instead.
  */
 export interface ReportDegradation {
     /** Why the report is degraded. Single-variant union, widen as needed. */
@@ -1517,8 +1525,19 @@ export interface Report {
     /**
      * Set when the report is published in a degraded state — a full eval
      * scored tests but enrichment did not complete. Absent on healthy reports.
+     *
+     * @deprecated Superseded by {@link Report.validity} (D0059). New write paths
+     * stamp `validity` (with `status:"degraded"` in this case) instead.
      */
     degraded?: ReportDegradation;
+    /**
+     * Post-hoc data-health assessment (D0059) — orthogonal to
+     * `provenance.classification` (run intent). Top-level because it judges the
+     * report's *data*, not the run. Additive + nullable: absent on pre-stamp
+     * reports, which are treated as trustworthy until backfilled. Subsumes the
+     * legacy {@link Report.degraded} flag.
+     */
+    validity?: ReportValidity;
     /**
      * Snapshot of the run manifest's `artifacts` slice at publish time (D0032).
      * The source of truth lives in `gs://…/runs/{runId}/manifest.json`; this

package/dist/_vendor/ailf-core/types/index.js

@@ -19,6 +19,7 @@ export { evalModeType } from "./eval-mode-config.js";
 export { DEFAULT_PREFLIGHT_CODE_CORRECTNESS_WEIGHT, } from "./preflight-scoring.js";
 export { CONVENTIONAL_DERIVATIONS, isConfidence } from "./confidence.js";
 export { err, fixtureId, generateJudgmentId, generateRunId, judgmentId, ok, providerId, resultId, runId, suiteId, taskId, traceId, } from "./branded-ids.js";
+export { isReportValidityStatus, REPORT_VALIDITY_STATUSES, } from "./report-validity.js";
 /** Set of canonical legacy modes — exported for report-formatter use. */
 export const LEGACY_FAILURE_MODES = [
     "api-error",