@sanity/ailf 6.0.0 → 6.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/config/airbyte/ai_literacy_framework.connector.yaml +276 -0
  2. package/config/bigquery/views/synthesis_parse_failure_rate_7d.sql +42 -0
  3. package/dist/_vendor/ailf-core/ports/context.d.ts +12 -0
  4. package/dist/_vendor/ailf-core/schemas/eval-config.d.ts +7 -0
  5. package/dist/_vendor/ailf-core/schemas/eval-config.js +8 -0
  6. package/dist/_vendor/ailf-core/services/diagnosis/cards/__tests__/failure-mode-summary.test.js +59 -0
  7. package/dist/_vendor/ailf-core/services/diagnosis/cards/doc-attribution-spotlight.js +5 -1
  8. package/dist/_vendor/ailf-core/services/diagnosis/cards/failure-mode-summary.js +47 -3
  9. package/dist/_vendor/ailf-core/services/diagnosis/cards/index.d.ts +10 -0
  10. package/dist/_vendor/ailf-core/services/diagnosis/cards/index.js +13 -0
  11. package/dist/_vendor/ailf-core/services/diagnosis/cards/low-confidence-attribution.js +17 -1
  12. package/dist/_vendor/ailf-core/services/diagnosis/cards/no-issues.js +1 -1
  13. package/dist/_vendor/ailf-core/services/diagnosis/cards/regression-vs-baseline.js +5 -1
  14. package/dist/_vendor/ailf-core/services/diagnosis/cards/top-recommendations.js +5 -1
  15. package/dist/_vendor/ailf-core/services/diagnosis/cards/weakest-area.js +5 -1
  16. package/dist/_vendor/ailf-core/services/diagnosis/prompt-builders.js +15 -2
  17. package/dist/_vendor/ailf-core/services/diagnosis/prompts/weakest-area.system.d.ts +5 -3
  18. package/dist/_vendor/ailf-core/services/diagnosis/prompts/weakest-area.system.js +19 -31
  19. package/dist/_vendor/ailf-core/services/index.d.ts +1 -1
  20. package/dist/_vendor/ailf-core/services/index.js +1 -1
  21. package/dist/_vendor/ailf-core/types/diagnosis.d.ts +3 -0
  22. package/dist/_vendor/ailf-core/types/index.d.ts +7 -0
  23. package/dist/_vendor/ailf-core/types/repo-config.d.ts +16 -0
  24. package/dist/_vendor/ailf-core/types/synthesis-telemetry.d.ts +101 -0
  25. package/dist/_vendor/ailf-core/types/synthesis-telemetry.js +18 -0
  26. package/dist/adapters/config-sources/file-config-adapter.js +8 -6
  27. package/dist/adapters/llm/index.d.ts +1 -1
  28. package/dist/adapters/llm/index.js +1 -1
  29. package/dist/adapters/llm/openai-llm-client.js +7 -2
  30. package/dist/adapters/llm/retry.d.ts +18 -0
  31. package/dist/adapters/llm/retry.js +21 -0
  32. package/dist/adapters/synthesis/synthesis-telemetry-schema.d.ts +49 -0
  33. package/dist/adapters/synthesis/synthesis-telemetry-schema.js +55 -0
  34. package/dist/adapters/task-sources/content-lake-task-source.js +10 -5
  35. package/dist/adapters/task-sources/repo-schemas.d.ts +7 -0
  36. package/dist/adapters/task-sources/repo-schemas.js +10 -0
  37. package/dist/commands/interpret.d.ts +21 -1
  38. package/dist/commands/interpret.js +13 -4
  39. package/dist/commands/pipeline-action.d.ts +44 -0
  40. package/dist/commands/pipeline-action.js +193 -1
  41. package/dist/commands/run.d.ts +2 -0
  42. package/dist/commands/run.js +2 -0
  43. package/dist/orchestration/pipeline-orchestrator.js +3 -0
  44. package/dist/report-store.d.ts +26 -0
  45. package/dist/report-store.js +63 -0
  46. package/package.json +1 -1
@@ -646,6 +646,15 @@ const OwnerConfigSchema = z
646
646
  individual: z.string().min(1).optional(),
647
647
  })
648
648
  .optional();
649
+ /**
650
+ * Post-run diagnosis summary policy (Phase 6 / DIAG-06).
651
+ * Sits in the W0077 Phase-6a auto-load pathway.
652
+ */
653
+ const SummaryConfigSchema = z
654
+ .object({
655
+ onRun: z.enum(["auto", "always", "never"]).optional(),
656
+ })
657
+ .optional();
649
658
  /**
650
659
  * Agentic-mode configuration (W0077 Phase 6f). Replaces the retired
651
660
  * `--header` and `--allowed-origin` CLI flags. `headers` is a key/value
@@ -694,6 +703,7 @@ export const RepoConfigSchema = z.object({
694
703
  owner: OwnerConfigSchema,
695
704
  agentic: AgenticConfigSchema,
696
705
  artifacts: ArtifactsConfigSchema,
706
+ summary: SummaryConfigSchema,
697
707
  taskSource: TaskSourceConfigSchema,
698
708
  triggers: z
699
709
  .object({
@@ -16,7 +16,7 @@
16
16
  * @see .planning/phases/05-diagnosis-engine-cli-llm-cards/05-AI-SPEC.md §6
17
17
  */
18
18
  import { Command } from "commander";
19
- import type { DiagnosisRunner, VersionedInputs } from "../_vendor/ailf-core/index.d.ts";
19
+ import { type DiagnosisCard, type DiagnosisRunner, type VersionedInputs } from "../_vendor/ailf-core/index.d.ts";
20
20
  interface MinimalReportStore {
21
21
  read(id: string): Promise<unknown | null>;
22
22
  latest(): Promise<unknown | null>;
@@ -39,6 +39,26 @@ export interface InterpretCommandOptions {
39
39
  */
40
40
  readonly versionsFromReport?: (report: unknown) => VersionedInputs;
41
41
  }
42
+ /**
43
+ * Visual status markers — locked visual contract per plan Test 7:
44
+ * ready: "✓", degraded: "⚠", missing: "—"
45
+ *
46
+ * Exported so Plan 06-04's post-run hook imports the SAME object and
47
+ * D6-04's "single formatter, single visual contract" is physically
48
+ * enforced — no copy/paste drift possible.
49
+ */
50
+ export declare const STATUS_ICONS: Record<DiagnosisCard["status"], string>;
51
+ /**
52
+ * Format a single card as a one-line summary string.
53
+ *
54
+ * Format: `<icon> <cardType>: <summary>`
55
+ * Per AI-SPEC §6: distinct icons for ready / degraded / missing.
56
+ *
57
+ * Exported so Plan 06-04's post-run hook imports the SAME function and
58
+ * D6-04's "single formatter, single visual contract" is physically
59
+ * enforced — no copy/paste drift possible.
60
+ */
61
+ export declare function formatCardSummaryLine(card: DiagnosisCard): string;
42
62
  /**
43
63
  * Create the `ailf interpret <reportId>` Commander command.
44
64
  *
@@ -18,6 +18,7 @@
18
18
  import { dirname, resolve } from "path";
19
19
  import { fileURLToPath } from "url";
20
20
  import { Command } from "commander";
21
+ import { CARD_REGISTRY_VERSION, diagnosisVersion, } from "../_vendor/ailf-core/index.js";
21
22
  import { addOutputDirOption } from "./shared/options.js";
22
23
  import { resolveOutputDir } from "./shared/resolve-output-dir.js";
23
24
  // ---------------------------------------------------------------------------
@@ -31,8 +32,12 @@ const ROOT = resolve(__dirname, "..", "..");
31
32
  /**
32
33
  * Visual status markers — locked visual contract per plan Test 7:
33
34
  * ready: "✓", degraded: "⚠", missing: "—"
35
+ *
36
+ * Exported so Plan 06-04's post-run hook imports the SAME object and
37
+ * D6-04's "single formatter, single visual contract" is physically
38
+ * enforced — no copy/paste drift possible.
34
39
  */
35
- const STATUS_ICONS = {
40
+ export const STATUS_ICONS = {
36
41
  ready: "✓",
37
42
  degraded: "⚠",
38
43
  missing: "—",
@@ -52,8 +57,12 @@ function getCardSummaryText(card) {
52
57
  *
53
58
  * Format: `<icon> <cardType>: <summary>`
54
59
  * Per AI-SPEC §6: distinct icons for ready / degraded / missing.
60
+ *
61
+ * Exported so Plan 06-04's post-run hook imports the SAME function and
62
+ * D6-04's "single formatter, single visual contract" is physically
63
+ * enforced — no copy/paste drift possible.
55
64
  */
56
- function formatCardSummaryLine(card) {
65
+ export function formatCardSummaryLine(card) {
57
66
  const icon = STATUS_ICONS[card.status];
58
67
  const text = getCardSummaryText(card);
59
68
  return `${icon} ${card.cardType}: ${text}`;
@@ -82,10 +91,10 @@ function defaultVersionsFromReport(report) {
82
91
  : "unknown",
83
92
  diagnosisVersion: typeof versions?.diagnosisVersion === "string"
84
93
  ? versions.diagnosisVersion
85
- : "0.1.0",
94
+ : diagnosisVersion,
86
95
  cardVersion: typeof versions?.cardVersion === "string"
87
96
  ? versions.cardVersion
88
- : "0.1.0",
97
+ : CARD_REGISTRY_VERSION,
89
98
  };
90
99
  }
91
100
  // ---------------------------------------------------------------------------
@@ -12,6 +12,7 @@
12
12
  */
13
13
  import { type ImpactSummary } from "../pipeline/reverse-mapping.js";
14
14
  import type { DebugOptions, EvalMode } from "../pipeline/types.js";
15
+ import { type Diagnosis, type ReportStorePort, type SynthesisCostTelemetry } from "../_vendor/ailf-core/index.d.ts";
15
16
  import type { PipelineCliOptions } from "./run.js";
16
17
  export interface ResolvedOptions {
17
18
  allowedOriginArgs: string[];
@@ -63,6 +64,12 @@ export interface ResolvedOptions {
63
64
  studioOriginOverride?: string;
64
65
  remote: boolean;
65
66
  repoTasksPath?: string;
67
+ /** Phase 6 / DIAG-06: post-run diagnosis summary policy. Precedence
68
+ * resolution (CLI flag > env > config > auto) lives in
69
+ * shouldRunPostSummary() — this field carries only the config-file
70
+ * signal so the helper has a single typed input.
71
+ */
72
+ summaryOnRun?: "auto" | "always" | "never";
66
73
  taskOption?: string;
67
74
  tagOption?: string[];
68
75
  taskSourceType?: "content-lake" | "repo";
@@ -88,6 +95,43 @@ export interface ResolvedOptions {
88
95
  * Exported so the plan builder can call it independently.
89
96
  */
90
97
  export declare function computeResolvedOptions(opts: PipelineCliOptions): ResolvedOptions;
98
+ /**
99
+ * Determine whether the post-run diagnosis summary hook should fire.
100
+ *
101
+ * 4-level precedence chain (D6-20):
102
+ * Level 1 — CLI flag (absolute): if `cliOpts.summary` is boolean, use it.
103
+ * Level 2 — AILF_INTERPRET_ON_RUN env var (absolute): strict "1"/"0" parse;
104
+ * anything else falls through (T-06-11 spoofing mitigation).
105
+ * Level 3 — config `summary.onRun` (absolute): "always" → true; "never" → false;
106
+ * "auto" or absent falls through to level 4.
107
+ * Level 4 — default auto: TTY && !CI (SC1 default-off in CI).
108
+ */
109
+ export declare function shouldRunPostSummary(cliOpts: PipelineCliOptions, resolvedOnRun: "auto" | "always" | "never" | undefined): boolean;
110
+ export declare function buildSynthesisTelemetry(diagnosis: Diagnosis): SynthesisCostTelemetry;
111
+ /**
112
+ * Run post-pipeline hooks after the pipeline completes.
113
+ *
114
+ * Fires after orchestratePipeline() + writePipelineResult() (D6-02).
115
+ * Hook failure prints to stderr but does NOT change exit code (D6-03).
116
+ * CI default-off: fires only when shouldRunPostSummary returns true (D6-20).
117
+ *
118
+ * @param ctx - App context (composition root wiring)
119
+ * @param result - Pipeline result (includes reportId when published)
120
+ * @param args - Hook options (cliOpts, summaryOnRun from config, optional runnerFactory for tests)
121
+ */
122
+ export declare function runPostPipelineHooks(ctx: {
123
+ reportStore?: ReportStorePort;
124
+ }, result: {
125
+ success: boolean;
126
+ reportId?: string;
127
+ }, args: {
128
+ cliOpts: PipelineCliOptions;
129
+ summaryOnRun?: "auto" | "always" | "never";
130
+ /** Override runner factory for tests — avoids vi.mock of composition root */
131
+ runnerFactory?: (ctx: unknown) => {
132
+ run(opts: unknown): Promise<Diagnosis>;
133
+ };
134
+ }): Promise<void>;
91
135
  /**
92
136
  * Execute the evaluation pipeline.
93
137
  *
@@ -20,9 +20,13 @@ import { buildAppContext, parseArtifactUploadEnv, } from "../orchestration/build
20
20
  import { buildStepSequence } from "../orchestration/build-step-sequence.js";
21
21
  import { orchestratePipeline } from "../orchestration/pipeline-orchestrator.js";
22
22
  import { load } from "js-yaml";
23
- import { PLACEHOLDER_OWNER_TEAM } from "../_vendor/ailf-core/index.js";
23
+ import { PLACEHOLDER_OWNER_TEAM, } from "../_vendor/ailf-core/index.js";
24
24
  import { parseRepoConfig, } from "../adapters/task-sources/repo-schemas.js";
25
25
  import { getCallerCwd, resolveOutputDir } from "./shared/resolve-output-dir.js";
26
+ // Phase 6 / DIAG-06 — single formatter, single visual contract (D6-04).
27
+ // Import statically so bundlers and type-checkers can verify the export
28
+ // exists at build time rather than deferring to runtime dynamic import.
29
+ import { formatCardSummaryLine } from "./interpret.js";
26
30
  const __dirname = dirname(fileURLToPath(import.meta.url));
27
31
  const ROOT = resolve(__dirname, "..", "..");
28
32
  // ---------------------------------------------------------------------------
@@ -250,6 +254,10 @@ export function computeResolvedOptions(opts) {
250
254
  const graderReplications = repoConfig?.execution?.graderReplications;
251
255
  const borderlineReplications = repoConfig?.execution?.borderlineReplications;
252
256
  const gapAnalysisEnabled = repoConfig?.execution?.gapAnalysis ?? true;
257
+ // Phase 6 / DIAG-06 — post-run diagnosis summary policy from .ailf/config.yaml.
258
+ // Precedence resolution (CLI flag > env > config > auto) lives in
259
+ // shouldRunPostSummary(); this only carries the config-file signal.
260
+ const summaryOnRun = repoConfig?.summary?.onRun;
253
261
  // Grader context policy. Cascade: env var > .ailf/config.yaml > unset
254
262
  // (defaults to rubric-only at the EvalConfig boundary). The env var is the
255
263
  // operational lever for one-shot comparison runs without editing the config file.
@@ -348,6 +356,7 @@ export function computeResolvedOptions(opts) {
348
356
  undefined,
349
357
  purposeOption: opts.purpose?.trim() || undefined,
350
358
  labelOptions: opts.label ?? [],
359
+ summaryOnRun,
351
360
  };
352
361
  }
353
362
  const PUBLISH_AUTO_VALUES = ["always", "full-runs", "never"];
@@ -373,6 +382,179 @@ function resolvePublishAuto(repoValue) {
373
382
  }
374
383
  return "full-runs";
375
384
  }
385
+ // ---------------------------------------------------------------------------
386
+ // Phase 6 / DIAG-06 — post-run diagnosis summary helpers
387
+ // ---------------------------------------------------------------------------
388
+ /**
389
+ * Determine whether the post-run diagnosis summary hook should fire.
390
+ *
391
+ * 4-level precedence chain (D6-20):
392
+ * Level 1 — CLI flag (absolute): if `cliOpts.summary` is boolean, use it.
393
+ * Level 2 — AILF_INTERPRET_ON_RUN env var (absolute): strict "1"/"0" parse;
394
+ * anything else falls through (T-06-11 spoofing mitigation).
395
+ * Level 3 — config `summary.onRun` (absolute): "always" → true; "never" → false;
396
+ * "auto" or absent falls through to level 4.
397
+ * Level 4 — default auto: TTY && !CI (SC1 default-off in CI).
398
+ */
399
+ export function shouldRunPostSummary(cliOpts, resolvedOnRun) {
400
+ // Level 1: CLI flag wins absolutely
401
+ if (cliOpts.summary === true)
402
+ return true;
403
+ if (cliOpts.summary === false)
404
+ return false;
405
+ // Level 2: AILF_INTERPRET_ON_RUN env var (strict parse)
406
+ const envVal = process.env.AILF_INTERPRET_ON_RUN;
407
+ if (envVal === "1")
408
+ return true;
409
+ if (envVal === "0")
410
+ return false;
411
+ // Anything else (garbage, unset) falls through
412
+ // Level 3: config summary.onRun
413
+ if (resolvedOnRun === "always")
414
+ return true;
415
+ if (resolvedOnRun === "never")
416
+ return false;
417
+ // "auto" or undefined falls through
418
+ // Level 4: default auto — fire only when stdout is interactive and not in CI
419
+ return Boolean(process.stdout.isTTY) && process.env.CI !== "true";
420
+ }
421
+ /**
422
+ * Build a SynthesisCostTelemetry payload from a completed Diagnosis.
423
+ *
424
+ * Aggregates:
425
+ * - cost: sum of meta.cost across all cards (undefined treated as 0)
426
+ * - parseFailureCount: cards where status==="degraded" (parse failures)
427
+ * - parseFailureRate: parseFailureCount / total-cards (max 8 per D6-09)
428
+ * - perCard: per-card row with safe-extracted structured metadata
429
+ *
430
+ * Deliberately does NOT read card.body — only structured meta fields are
431
+ * persisted (T-06-14 PII guard per threat model).
432
+ */
433
+ // D6-09: denominator is always the fixed card-registry size, not cards.length.
434
+ // Using cards.length would allow parseFailureRate > 1.0 when the registry is
435
+ // a subset (e.g. test registries), violating the SynthesisCostTelemetrySchema
436
+ // min(0).max(1) constraint. Single edit point if the registry ever grows.
437
+ const CARD_REGISTRY_SIZE = 8;
438
+ export function buildSynthesisTelemetry(diagnosis) {
439
+ const cards = diagnosis.cards;
440
+ let totalCost = 0;
441
+ let parseFailureCount = 0;
442
+ const perCard = cards.map((card) => {
443
+ // "missing" cards have no `meta` — narrow with status guard
444
+ const meta = card.status !== "missing" ? card.meta : undefined;
445
+ const cost = meta?.cost ?? 0;
446
+ totalCost += cost;
447
+ // Parse failures produce status="degraded" (not "missing") in the current
448
+ // runner (diagnosis-runner.ts). A "missing" card is absence, not failure.
449
+ // If a future code path can produce status="missing" from a parse failure,
450
+ // this line must be updated and the parseFailed contract re-evaluated.
451
+ const parseFailed = card.status === "degraded";
452
+ if (parseFailed)
453
+ parseFailureCount++;
454
+ const row = {
455
+ cardType: card.cardType,
456
+ parseFailed,
457
+ cardVersion: meta?.cardVersion ?? "unknown",
458
+ generatedAt: meta?.generatedAt ?? new Date().toISOString(),
459
+ };
460
+ if (cost > 0)
461
+ row.cost = cost;
462
+ if (meta?.latencyMs !== undefined)
463
+ row.latencyMs = meta.latencyMs;
464
+ if (meta?.tokenUsage?.input !== undefined)
465
+ row.tokenInput = meta.tokenUsage.input;
466
+ if (meta?.tokenUsage?.output !== undefined)
467
+ row.tokenOutput = meta.tokenUsage.output;
468
+ return row;
469
+ });
470
+ return {
471
+ cost: totalCost,
472
+ parseFailureCount,
473
+ parseFailureRate: parseFailureCount / CARD_REGISTRY_SIZE,
474
+ perCard,
475
+ };
476
+ }
477
+ /**
478
+ * Run post-pipeline hooks after the pipeline completes.
479
+ *
480
+ * Fires after orchestratePipeline() + writePipelineResult() (D6-02).
481
+ * Hook failure prints to stderr but does NOT change exit code (D6-03).
482
+ * CI default-off: fires only when shouldRunPostSummary returns true (D6-20).
483
+ *
484
+ * @param ctx - App context (composition root wiring)
485
+ * @param result - Pipeline result (includes reportId when published)
486
+ * @param args - Hook options (cliOpts, summaryOnRun from config, optional runnerFactory for tests)
487
+ */
488
+ export async function runPostPipelineHooks(ctx, result, args) {
489
+ if (!shouldRunPostSummary(args.cliOpts, args.summaryOnRun))
490
+ return;
491
+ if (!result.reportId) {
492
+ process.stderr.write("ℹ️ No report published — skipping post-summary.\n");
493
+ return;
494
+ }
495
+ const reportId = result.reportId;
496
+ try {
497
+ // Build the runner — use injected factory (tests) or composition root (production)
498
+ let runner;
499
+ if (args.runnerFactory) {
500
+ runner = args.runnerFactory(ctx);
501
+ }
502
+ else {
503
+ const { getDiagnosisRunner } = await import("../composition-root.js");
504
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
505
+ runner = getDiagnosisRunner(ctx);
506
+ }
507
+ // Read the stored report — needed by the runner for version metadata
508
+ const report = await ctx.reportStore?.read(reportId);
509
+ if (!report) {
510
+ process.stderr.write(`ℹ️ Report not found: ${reportId} — skipping post-summary.\n`);
511
+ return;
512
+ }
513
+ // Derive version metadata from the stored report (same approach as interpret.ts)
514
+ const rec = report;
515
+ const summary = rec.summary;
516
+ const versions = summary?.versions;
517
+ const versionedInputs = {
518
+ graderJudgmentsVersion: typeof versions?.graderJudgmentsVersion === "string"
519
+ ? versions.graderJudgmentsVersion
520
+ : "unknown",
521
+ ensembleVersion: typeof versions?.ensembleVersion === "string"
522
+ ? versions.ensembleVersion
523
+ : "unknown",
524
+ diagnosisVersion: typeof versions?.diagnosisVersion === "string"
525
+ ? versions.diagnosisVersion
526
+ : "unknown",
527
+ cardVersion: typeof versions?.cardVersion === "string"
528
+ ? versions.cardVersion
529
+ : "unknown",
530
+ };
531
+ // Run the diagnosis
532
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
533
+ const diagnosis = await runner.run({
534
+ report: report,
535
+ versions: versionedInputs,
536
+ refresh: false,
537
+ });
538
+ // Print per-card summary lines to stdout (D6-04 single formatter)
539
+ for (const card of diagnosis.cards) {
540
+ process.stdout.write(`${formatCardSummaryLine(card)}\n`);
541
+ }
542
+ // Build and write synthesis telemetry back to the report doc (D6-08)
543
+ // patchSynthesis is now part of ReportStorePort (CR-01) — guard on store
544
+ // presence only; absent store means no report store is configured (expected).
545
+ if (ctx.reportStore) {
546
+ const telemetry = buildSynthesisTelemetry(diagnosis);
547
+ await ctx.reportStore.patchSynthesis(reportId, telemetry);
548
+ }
549
+ else {
550
+ process.stderr.write("ℹ️ No reportStore configured — synthesis telemetry not written to Sanity.\n");
551
+ }
552
+ }
553
+ catch (err) {
554
+ const msg = err instanceof Error ? err.message : String(err);
555
+ process.stderr.write(`⚠️ Diagnosis failed: ${msg}. Run \`ailf interpret ${reportId}\` to retry.\n`);
556
+ }
557
+ }
376
558
  /** Resolve and validate the --task-source flag value. */
377
559
  function resolveTaskSourceType(raw) {
378
560
  if (!raw || raw === "content-lake")
@@ -471,6 +653,11 @@ export async function executePipeline(cliOpts) {
471
653
  const steps = buildStepSequence(ctx, pipelineStart);
472
654
  const result = await orchestratePipeline(ctx, steps);
473
655
  writePipelineResult(result, config.outputDir);
656
+ // Phase 6 / DIAG-06: post-run hook fires after artifacts are written (D6-02)
657
+ await runPostPipelineHooks(ctx, result, {
658
+ cliOpts,
659
+ summaryOnRun: config.summaryOnRun,
660
+ });
474
661
  process.exit(result.success ? 0 : 1);
475
662
  }
476
663
  const o = resolveOptions(cliOpts);
@@ -510,6 +697,11 @@ export async function executePipeline(cliOpts) {
510
697
  const steps = buildStepSequence(ctx, pipelineStart);
511
698
  const result = await orchestratePipeline(ctx, steps);
512
699
  writePipelineResult(result, o.outputDir);
700
+ // Phase 6 / DIAG-06: post-run hook fires after artifacts are written (D6-02)
701
+ await runPostPipelineHooks(ctx, result, {
702
+ cliOpts,
703
+ summaryOnRun: o.summaryOnRun,
704
+ });
513
705
  process.exit(result.success ? 0 : 1);
514
706
  }
515
707
  // ---------------------------------------------------------------------------
@@ -47,6 +47,8 @@ export interface PipelineCliOptions {
47
47
  publish?: boolean;
48
48
  publishTag?: string;
49
49
  remoteCache?: boolean;
50
+ /** Phase 6 / DIAG-06: post-run diagnosis summary toggle. Undefined when neither flag is passed. */
51
+ summary?: boolean;
50
52
  sanityDocument: string[];
51
53
  sanityPerspective?: string;
52
54
  search?: string;
@@ -43,6 +43,8 @@ export function createRunCommand() {
43
43
  .option("-p, --publish", "Write report to Sanity + fan out to sinks (auto-enabled for full runs when report store is configured)")
44
44
  .option("--no-publish", "Suppress auto-publishing")
45
45
  .option("--publish-tag <tag>", "Label for published report")
46
+ .option("--summary", "Force post-run diagnosis summary (overrides config and CI default-off)")
47
+ .option("--no-summary", "Suppress post-run diagnosis summary")
46
48
  .option("--config <path>", "Load pipeline config from a TS/JS/YAML/JSON file (overrides most CLI flags)")
47
49
  .option("-o, --output <path>", "Write PR comment markdown to file")
48
50
  .option("--promptfoo-url <url>", "Promptfoo share URL for report")
@@ -275,6 +275,9 @@ export async function orchestratePipeline(ctx, steps) {
275
275
  belowCritical: state.belowCritical,
276
276
  durationMs,
277
277
  promptfooUrls: state.promptfooUrls,
278
+ // Phase 6 / DIAG-06 — thread reportId from state so the post-run hook in
279
+ // pipeline-action.ts can target the published Content Lake document.
280
+ reportId: state.reportId,
278
281
  steps: results,
279
282
  success: true,
280
283
  testSummary: state.testSummary,
@@ -15,6 +15,7 @@
15
15
  * @see docs/design-docs/report-store/domain-model.md
16
16
  */
17
17
  import type { SanityClient } from "@sanity/client";
18
+ import type { SynthesisCostTelemetry } from "./_vendor/ailf-core/index.d.ts";
18
19
  import type { ComparisonReport, ISOTimestamp, LineageQuery, Report, ReportId, ReportProvenance, ScoreSummary } from "./pipeline/types.js";
19
20
  /**
20
21
  * Result of an auto-comparison, bundling the ComparisonReport with the
@@ -89,6 +90,22 @@ export declare class ReportStore {
89
90
  * @see docs/design-docs/report-store/architecture.md — Auto-comparison
90
91
  */
91
92
  findComparableBaseline(query: LineageQuery): Promise<null | Report>;
93
+ /**
94
+ * Fetch the most recent report from the Content Lake.
95
+ *
96
+ * Mirrors the API gateway's `ReportStoreApi.latest()` signature
97
+ * (`packages/api/src/lib/sanity.ts`). Used by `ailf interpret latest`
98
+ * when no explicit report ID is supplied.
99
+ *
100
+ * @param query Optional narrowing by `mode` and/or `source.name`.
101
+ * @returns The most recent matching report, or null if none exist
102
+ * or on API failure. Schema-validation errors are advisory (logged
103
+ * and null-returned) per the same rationale as `findByFingerprint`.
104
+ */
105
+ latest(query?: {
106
+ mode?: string;
107
+ source?: string;
108
+ }): Promise<null | Report>;
92
109
  /**
93
110
  * Read a report by its ID.
94
111
  *
@@ -108,6 +125,15 @@ export declare class ReportStore {
108
125
  * runtime schema gate. Schema drift is a bug, not an outage.
109
126
  */
110
127
  write(report: Report): Promise<null | ReportId>;
128
+ /**
129
+ * Patch synthesis telemetry onto a published report (Phase 6 / DIAG-06).
130
+ * Non-fatal on Sanity failure — the on-disk reportSnapshot artifact
131
+ * remains the source of truth. Mirrors `write()` (L379–411) for
132
+ * error handling.
133
+ *
134
+ * Document _id is `report-${reportId}` (see `toSanityReportDoc` L559).
135
+ */
136
+ patchSynthesis(reportId: ReportId, telemetry: SynthesisCostTelemetry): Promise<void>;
111
137
  /**
112
138
  * Query error arrays from the last N reports for chronic failure detection.
113
139
  *
@@ -207,6 +207,50 @@ export class ReportStore {
207
207
  return null;
208
208
  }
209
209
  }
210
+ /**
211
+ * Fetch the most recent report from the Content Lake.
212
+ *
213
+ * Mirrors the API gateway's `ReportStoreApi.latest()` signature
214
+ * (`packages/api/src/lib/sanity.ts`). Used by `ailf interpret latest`
215
+ * when no explicit report ID is supplied.
216
+ *
217
+ * @param query Optional narrowing by `mode` and/or `source.name`.
218
+ * @returns The most recent matching report, or null if none exist
219
+ * or on API failure. Schema-validation errors are advisory (logged
220
+ * and null-returned) per the same rationale as `findByFingerprint`.
221
+ */
222
+ async latest(query) {
223
+ try {
224
+ let groq = `*[_type == $type`;
225
+ const params = { type: REPORT_TYPE };
226
+ if (query?.mode) {
227
+ groq += ` && provenance.mode == $mode`;
228
+ params.mode = query.mode;
229
+ }
230
+ if (query?.source) {
231
+ groq += ` && provenance.source.name == $source`;
232
+ params.source = query.source;
233
+ }
234
+ groq += `] | order(completedAt desc) [0]`;
235
+ const doc = await this.client.fetch(groq, params);
236
+ return doc ? toReport(doc) : null;
237
+ }
238
+ catch (error) {
239
+ // Advisory lookup — a malformed prior doc must not break a read-only
240
+ // CLI invocation. Log and return null so the caller surfaces a
241
+ // user-friendly "no report found" error instead of a Zod stack trace.
242
+ if (error instanceof ReportSchemaValidationError) {
243
+ logAdvisoryQuerySchemaFailure({
244
+ query: "latest",
245
+ context: { mode: query?.mode, sourceName: query?.source },
246
+ error,
247
+ });
248
+ return null;
249
+ }
250
+ console.warn(` ⚠️ Failed to fetch latest report from Sanity: ${error instanceof Error ? error.message : String(error)}`);
251
+ return null;
252
+ }
253
+ }
210
254
  /**
211
255
  * Read a report by its ID.
212
256
  *
@@ -264,6 +308,25 @@ export class ReportStore {
264
308
  return null;
265
309
  }
266
310
  }
311
+ /**
312
+ * Patch synthesis telemetry onto a published report (Phase 6 / DIAG-06).
313
+ * Non-fatal on Sanity failure — the on-disk reportSnapshot artifact
314
+ * remains the source of truth. Mirrors `write()` (L379–411) for
315
+ * error handling.
316
+ *
317
+ * Document _id is `report-${reportId}` (see `toSanityReportDoc` L559).
318
+ */
319
+ async patchSynthesis(reportId, telemetry) {
320
+ try {
321
+ await this.client
322
+ .patch(`report-${reportId}`)
323
+ .set({ "summary.synthesis": { diagnosis: telemetry } })
324
+ .commit();
325
+ }
326
+ catch (error) {
327
+ console.warn(` ⚠️ Failed to patch synthesis telemetry on report ${reportId}: ${error instanceof Error ? error.message : String(error)}`);
328
+ }
329
+ }
267
330
  /**
268
331
  * Query error arrays from the last N reports for chronic failure detection.
269
332
  *
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@sanity/ailf",
3
- "version": "6.0.0",
3
+ "version": "6.1.0",
4
4
  "private": false,
5
5
  "publishConfig": {
6
6
  "access": "public"