@sanity/ailf 1.0.0 → 2.0.1

package/dist/config/rubrics.ts

@@ -0,0 +1,225 @@
+/**
+ * rubrics.ts — Centralized rubric templates for LLM grading assertions.
+ *
+ * Tasks reference these templates by key and provide only their unique
+ * criteria bullet points. The pipeline assembles the full rubric text
+ * at expansion time.
+ *
+ * @see docs/design-docs/structured-dimensions.md
+ * @see docs/design-docs/uniform-dimension-scoring.md
+ */
+
+import { defineRubrics } from "../_vendor/ailf-core/index.js"
+
+export default defineRubrics({
+  templates: {
+    // ── Core literacy dimensions ────────────────────────────
+    "task-completion": {
+      dimension: "task-completion",
+      header: "Score task completion from 0 to 100:",
+      scale: [
+        "0: Couldn't attempt — missing critical information",
+        "20: Attempted but fundamentally wrong approach",
+        "50: Partial implementation — major functional gaps",
+        "80: Mostly complete — minor issues or missing edge cases",
+        "100: Fully functional code — works as expected",
+      ],
+      criteria_label: "Must demonstrate:",
+    },
+    "code-correctness": {
+      dimension: "code-correctness",
+      header: "Score code correctness from 0 to 100:",
+      scale: [
+        "0: Broken code, syntax errors, or deprecated APIs",
+        "30: Works but uses anti-patterns or inefficient approaches",
+        "50: Works but not idiomatic",
+        "80: Follows most best practices",
+        "100: Follows all best practices, idiomatic implementation",
+      ],
+      criteria_label: "Check for:",
+    },
+    "doc-coverage": {
+      dimension: "doc-coverage",
+      header: "Score documentation coverage from 0 to 100:",
+      scale: [
+        "0: Had to hallucinate/guess most implementation details",
+        "30: Significant gaps — filled with assumptions",
+        "50: Some gaps — inferred from partial information",
+        "80: Minor gaps — almost everything was documented",
+        "100: Complete coverage — all necessary info was in docs",
+      ],
+    },
+
+    // ── MCP server dimensions ───────────────────────────────
+    "mcp-input-validation": {
+      dimension: "input-validation",
+      header: "Score MCP tool input correctness from 0 to 100:",
+      scale: [
+        "0: Completely wrong tool inputs — missing or invalid parameters",
+        "25: Attempted correct inputs but with significant errors",
+        "50: Partially correct — some required fields present, others wrong",
+        "75: Mostly correct — minor issues with optional parameters",
+        "100: Perfect tool inputs — all parameters correct and well-formed",
+      ],
+      criteria_label: "Evaluate:",
+    },
+    "mcp-output-correctness": {
+      dimension: "output-correctness",
+      header: "Score MCP tool output handling from 0 to 100:",
+      scale: [
+        "0: Completely failed to use tool outputs",
+        "25: Used outputs but misinterpreted the data",
+        "50: Partially correct interpretation of tool outputs",
+        "75: Mostly correct — minor misuse of returned data",
+        "100: Perfect output handling — correctly interpreted all tool responses",
+      ],
+      criteria_label: "Check for:",
+    },
+    "mcp-error-handling": {
+      dimension: "error-handling",
+      header: "Score MCP error handling from 0 to 100:",
+      scale: [
+        "0: Crashed or ignored errors entirely",
+        "25: Acknowledged error but took wrong corrective action",
+        "50: Partial error handling — recovered from some errors",
+        "75: Good error handling — recovered gracefully with minor issues",
+        "100: Excellent — handled all errors appropriately with clear messaging",
+      ],
+      criteria_label: "Evaluate:",
+    },
+    "mcp-security": {
+      dimension: "security",
+      header: "Score MCP security compliance from 0 to 100:",
+      scale: [
+        "0: Attempted unauthorized operations or leaked sensitive data",
+        "25: Made questionable tool calls outside the expected scope",
+        "50: Stayed within scope but with minor security concerns",
+        "75: Good security — no unauthorized access, minor concerns",
+        "100: Perfect security — only used authorized tools with safe inputs",
+      ],
+      criteria_label: "Check for:",
+    },
+
+    // ── Knowledge probe dimensions ──────────────────────────
+    "factual-correctness": {
+      dimension: "factual-correctness",
+      header: "Score factual correctness from 0 to 100:",
+      scale: [
+        "0: Completely wrong — fabricated facts, hallucinated APIs, incorrect syntax",
+        "25: Mostly wrong — some real concepts but significant factual errors",
+        "50: Mixed — some correct facts alongside notable inaccuracies",
+        "75: Mostly correct — minor inaccuracies or imprecisions",
+        "100: Fully correct — all statements are accurate and verifiable",
+      ],
+      criteria_label: "Verify:",
+    },
+    completeness: {
+      dimension: "completeness",
+      header: "Score knowledge completeness from 0 to 100:",
+      scale: [
+        "0: Superficial — only knows the name, no substantive knowledge",
+        "25: Minimal — knows basic concepts but misses most important details",
+        "50: Partial — covers some key aspects but has significant gaps",
+        "75: Good coverage — covers most key aspects with minor gaps",
+        "100: Comprehensive — thorough coverage of all important aspects",
+      ],
+      criteria_label: "Check coverage of:",
+    },
+    currency: {
+      dimension: "currency",
+      header: "Score knowledge currency (up-to-dateness) from 0 to 100:",
+      scale: [
+        "0: Severely outdated — references deprecated APIs or removed features",
+        "25: Mostly outdated — aware of old version but not recent changes",
+        "50: Partially current — knows some recent changes but misses others",
+        "75: Mostly current — knows recent API but minor gaps on latest features",
+        "100: Fully current — references latest APIs, patterns, and best practices",
+      ],
+      criteria_label: "Check for:",
+    },
+
+    // ── Agent harness dimensions ────────────────────────────
+    "process-quality": {
+      dimension: "process-quality",
+      header:
+        "Score the agent's process quality from 0 to 100 (advisory — does not gate pass/fail):",
+      scale: [
+        "0: Chaotic process — random tool calls, no planning, no error handling",
+        "25: Poor process — some structure but significant inefficiencies",
+        "50: Adequate process — gets the job done but not efficiently",
+        "75: Good process — reads before writing, handles errors, incremental changes",
+        "100: Excellent process — optimal tool usage, clear planning, graceful recovery",
+      ],
+      criteria_label: "Evaluate:",
+    },
+    "agent-output": {
+      dimension: "agent-output",
+      header: "Score the agent's final output from 0 to 100:",
+      scale: [
+        "0: No useful output produced — task completely failed",
+        "25: Partial output — attempted but with fundamental errors",
+        "50: Usable output — works but with significant issues",
+        "75: Good output — mostly correct with minor issues",
+        "100: Excellent output — fully correct, clean, and complete",
+      ],
+      criteria_label: "Check for:",
+    },
+    "agent-tool-usage": {
+      dimension: "tool-usage",
+      header: "Score the agent's tool usage from 0 to 100:",
+      scale: [
+        "0: Completely wrong tool usage — called wrong tools or with invalid inputs",
+        "25: Poor tool usage — correct tools but wrong parameters or sequencing",
+        "50: Adequate — correct tools and basic parameters, some inefficiency",
+        "75: Good — efficient tool usage with proper error handling",
+        "100: Excellent — optimal tool selection, correct inputs, minimal redundancy",
+      ],
+      criteria_label: "Evaluate:",
+    },
+  },
+
+  // ── Named scoring profiles ────────────────────────────────
+  profiles: {
+    default: {
+      "task-completion": 0.5,
+      "code-correctness": 0.25,
+      "doc-coverage": 0.25,
+    },
+    "output-only": {
+      "task-completion": 0.6,
+      "code-correctness": 0.4,
+    },
+    "mcp-behavior": {
+      "input-validation": 0.25,
+      "output-correctness": 0.35,
+      "error-handling": 0.25,
+      security: 0.15,
+    },
+    "knowledge-probe": {
+      "factual-correctness": 0.45,
+      completeness: 0.35,
+      currency: 0.2,
+    },
+    "agent-harness": {
+      "agent-output": 0.45,
+      "tool-usage": 0.4,
+      "process-quality": 0.15,
+    },
+  },
+
+  // ── Mode-to-profile bindings ──────────────────────────────
+  // Literacy mode uses variant sub-keys because 'baseline' and 'agentic'
+  // are scoring variants within the same canonical mode, not separate modes.
+  "mode-profiles": {
+    "literacy": {
+      baseline: { gold: "default", baseline: "output-only" },
+      agentic: { gold: "default" },
+    },
+    "mcp-server": { gold: "mcp-behavior" },
+    "knowledge-probe": { gold: "knowledge-probe" },
+    "agent-harness": { gold: "agent-harness" },
+  },
+
+  footer:
+    'Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}',
+})

package/dist/config/schedules.ts

@@ -0,0 +1,47 @@
+/**
+ * schedules.ts — Scheduled evaluation configuration.
+ *
+ * Each schedule defines a recurring pipeline run with its own source,
+ * mode, and delivery preferences. The GitHub Actions cron workflow reads
+ * this config to determine which evaluations to run and when.
+ *
+ * Cron expressions use UTC timezone (GitHub Actions standard).
+ *
+ * @see docs/design-docs/report-store/implementation.md — Phase 5
+ */
+
+import { defineSchedules, env } from "../_vendor/ailf-core/index.js"
+
+export default defineSchedules({
+  schedules: [
+    // Daily baseline — track score trends against production docs
+    {
+      name: "daily-baseline",
+      cron: "0 2 * * *", // 2:00 AM UTC, every day
+      mode: "baseline",
+      source: "production",
+      publish: true,
+      compare: true,
+      enabled: true,
+    },
+
+    // Weekly full decomposition — complete floor/ceiling/actual report
+    {
+      name: "weekly-full",
+      cron: "0 3 * * 0", // 3:00 AM UTC, every Sunday
+      mode: "full",
+      source: "production",
+      publish: true,
+      compare: true,
+      enabled: true,
+    },
+  ],
+
+  // Digest — aggregates reports into periodic summaries
+  digest: {
+    enabled: true,
+    cron: "0 9 * * 1", // 9:00 AM UTC, every Monday
+    lookbackDays: 7,
+    slackWebhookUrl: env("SLACK_WEBHOOK_URL", ""),
+  },
+})

package/dist/config/sinks.ts

@@ -0,0 +1,37 @@
+/**
+ * sinks.ts — Report delivery sink configuration.
+ *
+ * Sinks receive published evaluation reports and deliver them to external
+ * systems (BigQuery, Slack, GitHub, webhooks, etc.).
+ *
+ * Sinks are fire-and-forget: a sink failure is logged but never blocks
+ * the pipeline. The Sanity Content Lake is the system of record.
+ *
+ * Sinks activate only when their required environment variables are present.
+ * A developer running locally with no env vars gets zero sinks.
+ *
+ * @see docs/design-docs/report-store/sink-architecture.md
+ */
+
+import { defineSinks } from "../_vendor/ailf-core/index.js"
+
+export default defineSinks({
+  sinks: [
+    // All sinks are currently disabled (commented out in YAML).
+    // Uncomment and configure as needed:
+    // BigQuery — disabled; Airbyte ELT is the primary delivery mechanism
+    // { type: "bigquery", enabled: false, project: env("BIGQUERY_PROJECT", "data-platform-302218"),
+    //   dataset: env("BIGQUERY_DATASET", "ailf"), credentials: env("GOOGLE_APPLICATION_CREDENTIALS", "") },
+    // Slack — regression alerts to configured channels
+    // { type: "slack", enabled: true, webhookUrl: env("SLACK_WEBHOOK_URL", ""),
+    //   channel: "#docs-ai-literacy",
+    //   routing: { critical: "#docs-alerts", warning: "#docs-team",
+    //              regression: "#docs-team", digest: "#docs-weekly" } },
+    // GitHub PR comments
+    // { type: "github-comment", enabled: false, token: env("GITHUB_TOKEN", "") },
+    // Webhook — generic HTTP relay
+    // { type: "webhook", enabled: false, url: env("AILF_WEBHOOK_URL", ""),
+    //   headers: { Authorization: `Bearer ${env("AILF_WEBHOOK_TOKEN", "")}` },
+    //   routing: { critical: true } },
+  ],
+})

package/dist/config/sources.ts

@@ -0,0 +1,21 @@
+/**
+ * sources.ts — Documentation source definitions for AILF evaluations.
+ *
+ * Default sources (production, branch, local) are provided by the
+ * sanity-literacy preset registered in the composition root. This file
+ * exists as an override point — any sources defined here are merged with
+ * (and take precedence over) preset-provided sources.
+ *
+ * To add a custom source, define it here:
+ *
+ *   export default defineSources([
+ *     { name: "my-docs", baseUrl: "https://docs.example.com", ... },
+ *   ])
+ *
+ * @see packages/eval/src/pipeline/compiler/presets/sanity-literacy.ts
+ * @see docs/archive/exec-plans/dynamic-doc-sources.md
+ */
+
+import { defineSources } from "../_vendor/ailf-core/index.js"
+
+export default defineSources([])

package/dist/config/thresholds.ts

@@ -0,0 +1,61 @@
+/**
+ * thresholds.ts — Quality thresholds for readiness gates and regression alerts.
+ *
+ * Used by:
+ * - `npx @sanity/ailf pipeline --readiness` (launch readiness checklist)
+ * - `npx @sanity/ailf pipeline --publish` (severity-aware sink routing)
+ * - `npx @sanity/ailf pipeline --compare` (regression alerting)
+ *
+ * @see docs/archive/exec-plans/scenario-matrix-implementation/phase-5-readiness-thresholds.md
+ */
+
+import { defineThresholds } from "../_vendor/ailf-core/index.js"
+
+export default defineThresholds({
+  // Global defaults (apply to all areas unless overridden)
+  defaults: {
+    composite: 50,
+    dimensions: {
+      "task-completion": 40,
+      "code-correctness": 30,
+      "doc-coverage": 30,
+    },
+    "doc-lift": 0, // docs must not hurt
+    ceiling: 40, // minimum ceiling score (doc quality floor)
+  },
+
+  // Per-area overrides (inherit from defaults, override specific values)
+  areas: {
+    groq: {
+      composite: 60, // GROQ is critical — higher bar
+      dimensions: {
+        "task-completion": 50,
+      },
+    },
+    // "visual-editing": {
+    //   composite: 45, // currently at 36, set achievable near-term target
+    // },
+  },
+
+  // Regression thresholds (for comparison reports)
+  regression: {
+    composite: -3, // alert if composite drops more than 3 points
+    "per-area": -5, // alert if any area drops more than 5 points
+    "per-dimension": -8, // alert if any dimension drops more than 8 points
+  },
+
+  // Severity classification
+  severity: {
+    critical: {
+      "composite-below": 30,
+      "negative-doc-lift": true,
+    },
+    warning: {
+      "composite-below": 50,
+      "regression-exceeds": -3,
+    },
+    info: {
+      "composite-below": 60,
+    },
+  },
+})

package/dist/index.d.ts

@@ -0,0 +1,41 @@
+/**
+ * @sanity/ailf — Public API for the AI Literacy Framework.
+ *
+ * This module is the entry point for external consumers who import from
+ * `@sanity/ailf`. It re-exports the authoring API needed to write task
+ * definitions, configuration files, and validate task YAML.
+ *
+ * ## Task authoring
+ *
+ * ```typescript
+ * import { defineTask } from "@sanity/ailf"
+ *
+ * export default defineTask({
+ *   id: "groq-projection-basics",
+ *   mode: "literacy",
+ *   title: "GROQ Projection Basics",
+ *   area: "groq",
+ *   prompt: { text: "Write GROQ queries..." },
+ *   assertions: [
+ *     { type: "llm-rubric", template: "task-completion", criteria: ["..."] },
+ *   ],
+ * })
+ * ```
+ *
+ * ## Configuration authoring
+ *
+ * ```typescript
+ * import { defineConfig, env } from "@sanity/ailf"
+ *
+ * export default defineConfig({
+ *   projectId: env("SANITY_PROJECT_ID"),
+ *   dataset: env("SANITY_DATASET"),
+ * })
+ * ```
+ */
+export { defineConfig, defineFeatures, defineModels, definePricingTable, definePreset, definePrompts, defineRubrics, defineSchedules, defineSinks, defineSources, defineTask, defineThresholds, } from "./_vendor/ailf-core/index.d.ts";
+export type { PricingEntry, PromptEntry, SourceEntry } from "./_vendor/ailf-core/index.d.ts";
+export { env } from "./_vendor/ailf-core/index.d.ts";
+export type { AgentHarnessTaskDefinition, CustomTaskDefinition, GeneralizedAssertionDefinition, GeneralizedDocRef, GeneralizedTaskDefinition, GeneralizedTemplatedAssertion, GeneralizedValueAssertion, IdDocRef, KnowledgeProbeTaskDefinition, LiteracyTaskDefinition, MCPServerTaskDefinition, PathDocRef, PerspectiveDocRef, RubricRef, SlugDocRef, TaskCommonFields, TaskDifficulty, TaskOptions, TaskProviderConfig, TaskStatus, } from "./_vendor/ailf-core/index.d.ts";
+export { CanonicalTaskFileSchema, CanonicalTaskSchema, CURATED_ASSERTION_TYPES, detectLegacyFieldNames, parseCanonicalTaskFile, RUBRIC_TEMPLATE_NAMES, type CanonicalTask, type CuratedAssertionType, type RubricTemplateName, } from "./adapters/task-sources/repo-schemas.js";
+export { formatValidationResult, validateCanonicalTasks, type ValidationMessage, type ValidationResult, } from "./adapters/task-sources/repo-validation.js";

package/dist/index.js

@@ -0,0 +1,48 @@
+/**
+ * @sanity/ailf — Public API for the AI Literacy Framework.
+ *
+ * This module is the entry point for external consumers who import from
+ * `@sanity/ailf`. It re-exports the authoring API needed to write task
+ * definitions, configuration files, and validate task YAML.
+ *
+ * ## Task authoring
+ *
+ * ```typescript
+ * import { defineTask } from "@sanity/ailf"
+ *
+ * export default defineTask({
+ *   id: "groq-projection-basics",
+ *   mode: "literacy",
+ *   title: "GROQ Projection Basics",
+ *   area: "groq",
+ *   prompt: { text: "Write GROQ queries..." },
+ *   assertions: [
+ *     { type: "llm-rubric", template: "task-completion", criteria: ["..."] },
+ *   ],
+ * })
+ * ```
+ *
+ * ## Configuration authoring
+ *
+ * ```typescript
+ * import { defineConfig, env } from "@sanity/ailf"
+ *
+ * export default defineConfig({
+ *   projectId: env("SANITY_PROJECT_ID"),
+ *   dataset: env("SANITY_DATASET"),
+ * })
+ * ```
+ */
+// ---------------------------------------------------------------------------
+// Configuration helpers (define* identity functions for typed authoring)
+// ---------------------------------------------------------------------------
+export { defineConfig, defineFeatures, defineModels, definePricingTable, definePreset, definePrompts, defineRubrics, defineSchedules, defineSinks, defineSources, defineTask, defineThresholds, } from "./_vendor/ailf-core/index.js";
+// ---------------------------------------------------------------------------
+// Environment helper
+// ---------------------------------------------------------------------------
+export { env } from "./_vendor/ailf-core/index.js";
+// ---------------------------------------------------------------------------
+// Validation — for programmatic validation of task YAML
+// ---------------------------------------------------------------------------
+export { CanonicalTaskFileSchema, CanonicalTaskSchema, CURATED_ASSERTION_TYPES, detectLegacyFieldNames, parseCanonicalTaskFile, RUBRIC_TEMPLATE_NAMES, } from "./adapters/task-sources/repo-schemas.js";
+export { formatValidationResult, validateCanonicalTasks, } from "./adapters/task-sources/repo-validation.js";

package/dist/lib/agent-behavior-report.d.ts

@@ -0,0 +1,8 @@
+/**
+ * lib/agent-behavior-report.ts — DEPRECATED re-export shim.
+ * @deprecated Import from ../pipeline/agent-behavior-report.js instead.
+ */
+import "dotenv/config";
+export { analyzeResults, CANONICAL_DOC_MAP, detectFeatureArea, } from "../pipeline/agent-behavior-report.js";
+export type { AnalysisResult, FeatureAnalysis, TaskBehavior, TestResult, } from "../pipeline/agent-behavior-report.js";
+export declare function main(resultsPathArg?: string): void;

package/dist/lib/agent-behavior-report.js

@@ -0,0 +1,185 @@
+/**
+ * lib/agent-behavior-report.ts — DEPRECATED re-export shim.
+ * @deprecated Import from ../pipeline/agent-behavior-report.js instead.
+ */
+// oxlint-disable-next-line import/no-unassigned-import -- side-effect: loads .env into process.env
+import "dotenv/config";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
+import { dirname, join } from "path";
+export { analyzeResults, CANONICAL_DOC_MAP, detectFeatureArea, } from "../pipeline/agent-behavior-report.js";
+import { analyzeResults, } from "../pipeline/agent-behavior-report.js";
+export function main(resultsPathArg) {
+    const ROOT = join(dirname(new URL(import.meta.url).pathname), "..", "..");
+    const resultsPath = resultsPathArg ??
+        process.argv[2] ??
+        join(ROOT, "results", "latest", "eval-results.json");
+    if (!existsSync(resultsPath)) {
+        console.error(`Results file not found: ${resultsPath}`);
+        console.error("Run an evaluation first: pnpm eval:observed");
+        process.exit(1);
+    }
+    console.log(`Reading results from: ${resultsPath}`);
+    console.log();
+    const json = JSON.parse(readFileSync(resultsPath, "utf-8"));
+    const rawResults = Array.isArray(json.results)
+        ? json.results
+        : json.results.results;
+    const analysis = analyzeResults(rawResults);
+    if (!analysis.hasData) {
+        console.log("No agent behavior data found in the results.");
+        console.log("Make sure you ran the evaluation with the observed config:");
+        console.log("  pnpm eval:observed");
+        process.exit(0);
+    }
+    printReport(analysis);
+    // Persist detailed report as JSON
+    const outDir = join(ROOT, "results", "latest");
+    mkdirSync(outDir, { recursive: true });
+    const reportData = {
+        features: analysis.features.map((f) => ({
+            avgDocPages: f.avgDocPages,
+            avgNetworkMs: f.avgNetworkMs,
+            avgSearches: f.avgSearches,
+            canonicalCoverage: f.canonicalCoverage,
+            canonicalSlugs: f.canonicalSlugs,
+            docSlugsVisited: f.allDocSlugs,
+            externalDomains: f.allExternalDomains,
+            feature: f.feature,
+            searchQueries: f.allSearchQueries,
+            taskCount: f.tasks.length,
+        })),
+        tasks: analysis.tasks.map((t) => ({
+            behavior: t.behavior,
+            description: t.description,
+            feature: t.feature,
+            hasDocs: t.hasDocs,
+        })),
+        timestamp: new Date().toISOString(),
+        totalTasks: analysis.tasks.length,
+    };
+    writeFileSync(join(outDir, "agent-behavior-report.json"), JSON.stringify(reportData, null, 2));
+    console.log("Agent behavior report written to results/latest/agent-behavior-report.json");
+}
+// ---------------------------------------------------------------------------
+// Report output (kept in shim for backward compat)
+// ---------------------------------------------------------------------------
+function printReport(analysis) {
+    console.log("=".repeat(80));
+    console.log("            AGENT BEHAVIOR OBSERVATION REPORT");
+    console.log("=".repeat(80));
+    console.log();
+    // ---- Overview table ----
+    console.log("OVERVIEW BY FEATURE AREA");
+    console.log("-".repeat(80));
+    const h = "| Feature Area        | Tasks | Avg Docs | Avg Search | Avg Net(ms) | Canon% |";
+    const sep = "|---------------------|-------|----------|------------|-------------|--------|";
+    console.log(h);
+    console.log(sep);
+    for (const f of analysis.features) {
+        console.log(`| ${f.feature.padEnd(19)} | ` +
+            `${f.tasks.length.toString().padStart(5)} | ` +
+            `${f.avgDocPages.toFixed(1).padStart(8)} | ` +
+            `${f.avgSearches.toFixed(1).padStart(10)} | ` +
+            `${Math.round(f.avgNetworkMs).toString().padStart(11)} | ` +
+            `${(f.canonicalCoverage * 100).toFixed(0).padStart(5)}% |`);
+    }
+    console.log();
+    // ---- Canonical coverage breakdown ----
+    console.log("CANONICAL DOCUMENTATION COVERAGE");
+    console.log("-".repeat(80));
+    console.log();
+    for (const f of analysis.features) {
+        console.log(`  ${f.feature} (${(f.canonicalCoverage * 100).toFixed(0)}% canonical coverage):`);
+        if (f.canonicalSlugs.length === 0) {
+            console.log("    (no canonical docs defined)");
+        }
+        else {
+            for (const slug of f.canonicalSlugs) {
+                const found = f.allDocSlugs.some((visited) => visited.includes(slug));
+                const marker = found ? "[x]" : "[ ]";
+                console.log(`    ${marker} ${slug}`);
+            }
+        }
+        if (f.allDocSlugs.length > 0) {
+            const nonCanonical = f.allDocSlugs.filter((slug) => !f.canonicalSlugs.some((c) => slug.includes(c)));
+            if (nonCanonical.length > 0) {
+                console.log("    Additional docs visited:");
+                for (const slug of nonCanonical) {
+                    console.log(`      + ${slug}`);
+                }
+            }
+        }
+        console.log();
+    }
+    // ---- Search strategy ----
+    const allSearches = analysis.features.flatMap((f) => f.allSearchQueries);
+    if (allSearches.length > 0) {
+        console.log("SEARCH STRATEGY");
+        console.log("-".repeat(80));
+        console.log();
+        for (const f of analysis.features) {
+            if (f.allSearchQueries.length === 0)
+                continue;
+            console.log(`  ${f.feature}:`);
+            for (const q of f.allSearchQueries) {
+                console.log(`    -> "${q}"`);
+            }
+        }
+        console.log();
+    }
+    // ---- Per-task detail ----
+    console.log("PER-TASK DETAIL");
+    console.log("-".repeat(80));
+    console.log();
+    for (const f of analysis.features) {
+        console.log(`  ## ${f.feature}`);
+        console.log();
+        for (const t of f.tasks) {
+            const variant = t.hasDocs ? "[gold]" : "[baseline]";
+            console.log(`  ${variant} ${t.description}`);
+            console.log(`    Requests: ${t.behavior.totalRequests} | ` +
+                `Doc pages: ${t.behavior.docPagesVisited} | ` +
+                `Searches: ${t.behavior.searchesPerformed} | ` +
+                `External: ${t.behavior.externalRequestCount}`);
+            if (t.behavior.docSlugsVisited.length > 0) {
+                console.log(`    Docs: ${t.behavior.docSlugsVisited.join(", ")}`);
+            }
+            if (t.behavior.uniqueSearchQueries.length > 0) {
+                console.log(`    Queries: ${t.behavior.uniqueSearchQueries.map((q) => `"${q}"`).join(", ")}`);
+            }
+            console.log();
+        }
+    }
+    // ---- External domains ----
+    const allDomains = [
+        ...new Set(analysis.features.flatMap((f) => f.allExternalDomains)),
+    ];
+    if (allDomains.length > 0) {
+        console.log("EXTERNAL DOMAINS");
+        console.log("-".repeat(80));
+        console.log();
+        for (const d of allDomains) {
+            console.log(`  - ${d}`);
+        }
+        console.log();
+    }
+    // ---- Summary stats ----
+    console.log("OVERALL STATISTICS");
+    console.log("-".repeat(80));
+    console.log();
+    const totalTasks = analysis.tasks.length;
+    const tasksUsingDocs = analysis.tasks.filter((t) => t.behavior.usedDocs).length;
+    const tasksUsingSearch = analysis.tasks.filter((t) => t.behavior.usedSearch).length;
+    const avgCanonical = analysis.features.reduce((s, f) => s + f.canonicalCoverage, 0) /
+        (analysis.features.length || 1);
+    console.log(`  Total tasks observed:    ${totalTasks}`);
+    console.log(`  Tasks that used docs:    ${tasksUsingDocs}/${totalTasks} (${((tasksUsingDocs / totalTasks) * 100).toFixed(0)}%)`);
+    console.log(`  Tasks that used search:  ${tasksUsingSearch}/${totalTasks} (${((tasksUsingSearch / totalTasks) * 100).toFixed(0)}%)`);
+    console.log(`  Avg canonical coverage:  ${(avgCanonical * 100).toFixed(1)}%`);
+    console.log();
+}
+// Only run when invoked directly (not when imported)
+if (process.argv[1]?.endsWith("agent-behavior-report.ts") ||
+    process.argv[1]?.endsWith("agent-behavior-report.js")) {
+    main();
+}