@sanity/ailf 1.0.0 → 2.0.1

package/dist/adapters/task-sources/repo-validation.js

@@ -1,8 +1,129 @@
 /**
- * repo-validation.ts — Re-exports semantic validation from @sanity/ailf-tasks.
+ * repo-validation.ts — Semantic validation for task definitions.
  *
- * The validation logic is the single source of truth in @sanity/ailf-tasks.
- * This file re-exports so existing eval-package importers don't need
- * to change their import paths.
+ * Checks that go beyond Zod schema parsing:
+ * - Assertion types are in the curated set
+ * - Rubric template names resolve to known templates
+ * - Doc ref slugs look reasonable (slugs, not URLs)
+ * - Tasks have at least one LLM rubric assertion (recommended)
+ * - Tasks have a prompt text (recommended)
+ *
+ * These produce warnings, not errors — the pipeline can still run
+ * with imperfect tasks. Only structural failures (caught by Zod) block.
+ *
+ * Previously this file re-exported from @sanity/ailf-tasks. That package
+ * has been eliminated — all validation logic now lives here.
+ */
+import { CURATED_ASSERTION_TYPES, RUBRIC_TEMPLATE_NAMES, } from "./repo-schemas.js";
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Run semantic validation on an array of parsed canonical tasks.
+ *
+ * Returns warnings for issues that don't block execution (unknown feature
+ * areas, unresolved slugs) and errors for issues that would cause pipeline
+ * failures (completely missing required fields — though Zod catches most).
+ */
+export function validateCanonicalTasks(tasks) {
+    const errors = [];
+    const warnings = [];
+    // Check for duplicate IDs
+    const seenIds = new Set();
+    for (const task of tasks) {
+        if (seenIds.has(task.id)) {
+            errors.push({
+                taskId: task.id,
+                field: "id",
+                message: `Duplicate task ID "${task.id}"`,
+            });
+        }
+        seenIds.add(task.id);
+    }
+    for (const task of tasks) {
+        const assertions = task.assertions ?? [];
+        // Check assertion types
+        for (let i = 0; i < assertions.length; i++) {
+            const assertion = assertions[i];
+            if (!CURATED_ASSERTION_TYPES.includes(assertion.type)) {
+                warnings.push({
+                    taskId: task.id,
+                    field: `assertions[${i}].type`,
+                    message: `Unknown assertion type "${assertion.type}". ` +
+                        `Valid types: ${CURATED_ASSERTION_TYPES.join(", ")}`,
+                });
+            }
+            // Check rubric template for llm-rubric assertions
+            if (assertion.type === "llm-rubric" && "template" in assertion) {
+                const template = assertion.template;
+                if (!RUBRIC_TEMPLATE_NAMES.includes(template)) {
+                    warnings.push({
+                        taskId: task.id,
+                        field: `assertions[${i}].template`,
+                        message: `Unknown rubric template "${template}". ` +
+                            `Valid templates: ${RUBRIC_TEMPLATE_NAMES.join(", ")}`,
+                    });
+                }
+            }
+        }
+        // Check canonical doc refs look reasonable
+        const docs = task.context?.docs ?? [];
+        for (let i = 0; i < docs.length; i++) {
+            const doc = docs[i];
+            // Slug refs: warn if they look like URLs or paths
+            if ("slug" in doc && !("id" in doc) && typeof doc.slug === "string") {
+                if (doc.slug.includes("/") || doc.slug.includes("http")) {
+                    warnings.push({
+                        taskId: task.id,
+                        field: `context.docs[${i}].slug`,
+                        message: `Slug "${doc.slug}" looks like a URL or path — use 'path' type for paths or 'slug' for document slugs (e.g., "groq-introduction")`,
+                    });
+                }
+            }
+        }
+        // Check task has at least one llm-rubric assertion (recommended but not required)
+        const hasLlmRubric = assertions.some((a) => a.type === "llm-rubric");
+        if (!hasLlmRubric) {
+            warnings.push({
+                taskId: task.id,
+                field: "assertions",
+                message: "No llm-rubric assertion found. Tasks should have at least one scored rubric for meaningful evaluation.",
+            });
+        }
+        // Check prompt text exists
+        if (!task.prompt?.text) {
+            warnings.push({
+                taskId: task.id,
+                field: "prompt.text",
+                message: "No task prompt found in prompt.text. The LLM will receive an empty implementation request.",
+            });
+        }
+    }
+    return {
+        valid: errors.length === 0,
+        errors,
+        warnings,
+    };
+}
+/**
+ * Format validation results for console output.
  */
-export { detectSnakeCaseFields, formatValidationResult, validateRepoTasks, } from "../../_vendor/ailf-tasks/index.js";
+export function formatValidationResult(result) {
+    const lines = [];
+    if (result.errors.length > 0) {
+        lines.push("Errors:");
+        for (const e of result.errors) {
+            lines.push(`  [${e.taskId}] ${e.field}: ${e.message}`);
+        }
+    }
+    if (result.warnings.length > 0) {
+        lines.push("Warnings:");
+        for (const w of result.warnings) {
+            lines.push(`  [${w.taskId}] ${w.field}: ${w.message}`);
+        }
+    }
+    if (result.valid && result.warnings.length === 0) {
+        lines.push("All tasks pass validation");
+    }
+    return lines.join("\n");
+}

package/dist/adapters/task-sources/task-file-loader.d.ts

@@ -1,10 +1,9 @@
 /**
  * TaskFileLoader — loads task definitions from TypeScript files.
  *
- * Supplements the existing YAML-based task loading by supporting
- * `*.task.ts` and `*.task.js` files in task directories. Files are
- * loaded via jiti and validated through the RepoTaskSchema from
- * @sanity/ailf-tasks.
+ * Supports `*.task.ts` and `*.task.js` files in task directories. Files
+ * are loaded via jiti and expected to export GeneralizedTaskDefinition
+ * objects authored with `defineTask()`.
  *
  * TS task files export a single task or an array of tasks:
  *
@@ -20,9 +19,6 @@
  * ]
  * ```
  *
- * The loader integrates into the existing RepoTaskSource adapter — TS
- * task files are discovered alongside YAML files in the same directory.
- *
  * @see docs/design-docs/architecture-overhaul/typescript-configuration.md
  */
 /** A raw task object loaded from a TS file (pre-validation) */
@@ -55,6 +51,13 @@ export declare function discoverTsTaskFiles(tasksDir: string): string[];
  * @returns The loaded task(s), or throws on load failure
  */
 export declare function loadTsTaskFile(filePath: string): Promise<RawTsTask>;
+/**
+ * Synchronously load task definitions from a single TS/JS task file.
+ *
+ * Uses jiti's synchronous require-style loading instead of async import().
+ * Needed by resolve-mappings.ts which is called from sync contexts.
+ */
+export declare function loadTsTaskFileSync(filePath: string): RawTsTask;
 /**
  * Load all TS task files from a directory.
  *

package/dist/adapters/task-sources/task-file-loader.js

@@ -1,10 +1,9 @@
 /**
  * TaskFileLoader — loads task definitions from TypeScript files.
  *
- * Supplements the existing YAML-based task loading by supporting
- * `*.task.ts` and `*.task.js` files in task directories. Files are
- * loaded via jiti and validated through the RepoTaskSchema from
- * @sanity/ailf-tasks.
+ * Supports `*.task.ts` and `*.task.js` files in task directories. Files
+ * are loaded via jiti and expected to export GeneralizedTaskDefinition
+ * objects authored with `defineTask()`.
  *
  * TS task files export a single task or an array of tasks:
  *
@@ -20,13 +19,12 @@
  * ]
  * ```
  *
- * The loader integrates into the existing RepoTaskSource adapter — TS
- * task files are discovered alongside YAML files in the same directory.
- *
  * @see docs/design-docs/architecture-overhaul/typescript-configuration.md
  */
 import { existsSync, readdirSync } from "fs";
+import { pathToFileURL } from "node:url";
 import { resolve } from "path";
+import { createJiti } from "jiti";
 import { loadTsConfig } from "../config-sources/ts-config-loader.js";
 /**
  * Discover TS/JS task files in a directory.
@@ -67,6 +65,22 @@ export async function loadTsTaskFile(filePath) {
     const tasks = Array.isArray(value) ? value : [value];
     return { filePath, tasks };
 }
+/**
+ * Synchronously load task definitions from a single TS/JS task file.
+ *
+ * Uses jiti's synchronous require-style loading instead of async import().
+ * Needed by resolve-mappings.ts which is called from sync contexts.
+ */
+export function loadTsTaskFileSync(filePath) {
+    const jiti = createJiti(pathToFileURL(filePath).href, {
+        interopDefault: true,
+        requireCache: true,
+    });
+    const mod = jiti(filePath);
+    const value = mod && typeof mod === "object" && "default" in mod ? mod.default : mod;
+    const tasks = Array.isArray(value) ? value : [value];
+    return { filePath, tasks };
+}
 /**
  * Load all TS task files from a directory.
  *

package/dist/agent-observer/test-imports.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Final validation — ensures all agent-observer modules work together
+ * and the full data pipeline (record → classify → summarize) is correct.
+ *
+ * Run: tsx src/agent-observer/test-imports.ts
+ */
+export {};

package/dist/agent-observer/test-imports.js

@@ -0,0 +1,185 @@
+/**
+ * Final validation — ensures all agent-observer modules work together
+ * and the full data pipeline (record → classify → summarize) is correct.
+ *
+ * Run: tsx src/agent-observer/test-imports.ts
+ */
+import { classifyRequests, extractDocSlug, extractSearchQuery, extractApiEndpoint, extractDomain, extractPageTitle, isDocPageRequest, isSearchRequest, isSanityApiRequest, } from "./classifier.js";
+import { RequestRecorder } from "./proxy.js";
+import { default as InstrumentedProvider } from "./provider.js";
+// ─── Test data ───────────────────────────────────────────────────────────────
+const now = Date.now();
+const mockRequests = [
+    // 1. Doc page visit
+    {
+        headers: {},
+        latencyMs: 234,
+        method: "GET",
+        responsePreview: "<html><head><title>Create a Schema - Sanity</title></head>...",
+        responseSize: 45000,
+        seq: 0,
+        statusCode: 200,
+        timestamp: new Date(now).toISOString(),
+        url: "https://www.sanity.io/docs/create-a-schema-and-configure-sanity-studio",
+    },
+    // 2. Another doc page
+    {
+        headers: {},
+        latencyMs: 180,
+        method: "GET",
+        responsePreview: "<html><head><title>Object Type - Sanity Docs</title></head>...",
+        responseSize: 32000,
+        seq: 1,
+        statusCode: 200,
+        timestamp: new Date(now + 100).toISOString(),
+        url: "https://www.sanity.io/docs/schema-types/object-type",
+    },
+    // 3. Search query
+    {
+        headers: {},
+        latencyMs: 450,
+        method: "GET",
+        responseSize: 12000,
+        seq: 2,
+        statusCode: 200,
+        timestamp: new Date(now + 200).toISOString(),
+        url: "https://www.sanity.io/search?q=visual+editing+preview",
+    },
+    // 4. Sanity API call (GROQ query — must NOT be classified as search)
+    {
+        headers: {},
+        latencyMs: 320,
+        method: "GET",
+        responseSize: 8500,
+        seq: 3,
+        statusCode: 200,
+        timestamp: new Date(now + 300).toISOString(),
+        url: 'https://api.sanity.io/v2021-03-25/data/query/production?query=*[_type=="article"]',
+    },
+    // 5. CDN API call
+    {
+        headers: {},
+        latencyMs: 85,
+        method: "GET",
+        responseSize: 150000,
+        seq: 4,
+        statusCode: 200,
+        timestamp: new Date(now + 350).toISOString(),
+        url: "https://cdn.sanity.io/images/abc123/production/image-xyz.jpg",
+    },
+    // 6. External request (npm docs)
+    {
+        headers: {},
+        latencyMs: 300,
+        method: "GET",
+        responseSize: 20000,
+        seq: 5,
+        statusCode: 200,
+        timestamp: new Date(now + 400).toISOString(),
+        url: "https://docs.npmjs.com/cli/install",
+    },
+    // 7. Algolia search
+    {
+        body: JSON.stringify({ query: "presentation tool setup" }),
+        headers: {},
+        latencyMs: 150,
+        method: "POST",
+        responseSize: 5000,
+        seq: 6,
+        statusCode: 200,
+        timestamp: new Date(now + 500).toISOString(),
+        url: "https://abc123.algolia.net/1/indexes/sanity_docs/query",
+    },
+    // 8. Google search
+    {
+        headers: {},
+        latencyMs: 200,
+        method: "GET",
+        responseSize: 80000,
+        seq: 7,
+        statusCode: 200,
+        timestamp: new Date(now + 600).toISOString(),
+        url: "https://www.google.com/search?q=sanity+studio+custom+tool",
+    },
+    // 9. Failed request (should be skipped)
+    {
+        headers: {},
+        latencyMs: 0,
+        method: "GET",
+        responseSize: 0,
+        seq: 8,
+        statusCode: 0,
+        timestamp: new Date(now + 700).toISOString(),
+        url: "https://www.sanity.io/docs/nonexistent-page",
+    },
+];
+// ─── Run tests ───────────────────────────────────────────────────────────────
+let passed = 0;
+let failed = 0;
+function assert(condition, msg) {
+    if (condition) {
+        console.log(`  ✅ ${msg}`);
+        passed++;
+    }
+    else {
+        console.log(`  ❌ FAIL: ${msg}`);
+        failed++;
+    }
+}
+console.log("\n═══ Agent Observer — Final Validation ═══\n");
+// --- Individual function tests ---
+console.log("1. Individual detection functions:");
+assert(isDocPageRequest(mockRequests[0]) === true, "Doc page detected");
+assert(isDocPageRequest(mockRequests[3]) === false, "API call NOT detected as doc page");
+assert(isSearchRequest(mockRequests[2]) === true, "Search detected");
+assert(isSearchRequest(mockRequests[3]) === false, "API call NOT detected as search");
+assert(isSanityApiRequest(mockRequests[3]) === true, "API call detected");
+assert(isSanityApiRequest(mockRequests[0]) === false, "Doc page NOT detected as API call");
+// --- Slug extraction ---
+console.log("\n2. Metadata extraction:");
+assert(extractDocSlug("https://www.sanity.io/docs/create-a-schema-and-configure-sanity-studio") === "create-a-schema-and-configure-sanity-studio", "Doc slug extracted correctly");
+assert(extractDocSlug("https://www.sanity.io/docs/schema-types/object-type") ===
+    "schema-types/object-type", "Nested doc slug extracted correctly");
+assert(extractSearchQuery(mockRequests[2]) === "visual editing preview", `Search query extracted: "${extractSearchQuery(mockRequests[2])}"`);
+assert(extractApiEndpoint("https://api.sanity.io/v2021-03-25/data/query/production?query=*") === "/data/query/production?query=*", `API endpoint extracted: "${extractApiEndpoint("https://api.sanity.io/v2021-03-25/data/query/production?query=*")}"`);
+assert(extractDomain("https://docs.npmjs.com/cli/install") === "docs.npmjs.com", "Domain extracted correctly");
+assert(extractPageTitle("<html><head><title>Create a Schema - Sanity</title></head>") === "Create a Schema - Sanity", "Page title extracted correctly");
+// --- Full classification pipeline ---
+console.log("\n3. Full classification pipeline:");
+const classified = classifyRequests(mockRequests);
+assert(classified.docPageVisits.length === 2, `Doc pages: ${classified.docPageVisits.length} (expected 2)`);
+assert(classified.searchQueries.length === 3, `Searches: ${classified.searchQueries.length} (expected 3 — site search + algolia + google)`);
+assert(classified.apiCalls.length === 2, `API calls: ${classified.apiCalls.length} (expected 2 — GROQ query + CDN)`);
+assert(classified.externalRequests.length === 1, `External: ${classified.externalRequests.length} (expected 1 — npmjs)`);
+// Verify the API ?query= param was NOT classified as a search
+const searchUrls = classified.searchQueries.map((s) => s.url);
+assert(!searchUrls.includes('https://api.sanity.io/v2021-03-25/data/query/production?query=*[_type=="article"]'), "API ?query= param NOT misclassified as search");
+// Verify the failed request was skipped
+const allUrls = [
+    ...classified.docPageVisits.map((d) => d.url),
+    ...classified.searchQueries.map((s) => s.url),
+    ...classified.apiCalls.map((a) => a.url),
+    ...classified.externalRequests.map((e) => e.url),
+];
+assert(!allUrls.includes("https://www.sanity.io/docs/nonexistent-page"), "Failed request (status 0) was skipped");
+// --- RequestRecorder ---
+console.log("\n4. RequestRecorder:");
+const recorder = new RequestRecorder({ includePatterns: [/sanity\.io/] });
+assert(typeof recorder.start === "function", "RequestRecorder.start() exists");
+assert(typeof recorder.stop === "function", "RequestRecorder.stop() exists");
+// --- InstrumentedProvider ---
+console.log("\n5. InstrumentedProvider:");
+const provider = new InstrumentedProvider({ config: {}, id: "test-validation" });
+assert(provider.id() === "instrumented:test-validation", `Provider ID: "${provider.id()}"`);
+assert(typeof provider.callApi === "function", "Provider.callApi() exists");
+// --- Summary ---
+console.log(`\n${"═".repeat(50)}`);
+console.log(`Results: ${passed} passed, ${failed} failed, ${passed + failed} total`);
+if (failed > 0) {
+    console.log("\n⚠️  Some tests failed!");
+    process.exit(1);
+}
+else {
+    console.log("\n✅ All tests passed! Agent observer system is ready.");
+    process.exit(0);
+}

package/dist/artifact-capture/comparator.d.ts

@@ -0,0 +1,22 @@
+/**
+ * CaptureComparator — compares two capture directories and produces a diff report.
+ *
+ * Reads manifest.json from both directories and computes:
+ * - Inventory diff (added/removed/common artifacts)
+ * - Content diff (structural or strict, for common artifacts)
+ * - Score comparison (from score-summary.json)
+ * - Timing comparison (from pipeline-context.json)
+ * - Metadata comparison (mode, variant, config keys)
+ * - Security scan (regex for leaked secrets)
+ *
+ * Implementation for the types defined in @sanity/ailf-core.
+ */
+import type { CaptureDiffReport, ComparisonOptions } from "../_vendor/ailf-core/index.d.ts";
+/**
+ * Compare two capture directories and produce a structured diff report.
+ *
+ * @param baselineDir - Path to the baseline capture directory (contains manifest.json)
+ * @param experimentDir - Path to the experiment capture directory
+ * @param opts - Comparison options (mode, thresholds, etc.)
+ */
+export declare function compareCaptures(baselineDir: string, experimentDir: string, opts?: Partial<ComparisonOptions>): CaptureDiffReport;