@sanity/ailf 0.1.0

package/dist/agent-observer/proxy.d.ts

@@ -0,0 +1,91 @@
+/**
+ * proxy.ts
+ *
+ * A lightweight HTTP recording proxy that captures all outbound requests
+ * made by an AI agent during evaluation. Instead of being an actual network
+ * proxy (which requires HTTPS MITM certificates), this module provides:
+ *
+ *   1. A `fetch` wrapper that records every request/response
+ *   2. A session manager that groups observations by test case
+ *
+ * This design works with any agent framework that uses `fetch` or can be
+ * configured to use a custom HTTP client. For agents that use a real browser,
+ * see the Chrome DevTools Protocol integration (future work).
+ *
+ * Usage:
+ *   const recorder = new RequestRecorder()
+ *   recorder.start('session-123')
+ *
+ *   // The agent makes requests through the recorder's fetch wrapper
+ *   const response = await recorder.fetch('https://sanity.io/docs/groq')
+ *
+ *   const log = recorder.stop()
+ *   // → AgentBehaviorLog with all requests classified
+ */
+import type { ObservedRequest, AgentBehaviorLog } from "./types.js";
+export interface RecorderOptions {
+    /** Headers to capture from requests (case-insensitive). Default: common ones */
+    captureHeaders?: string[];
+    /** Whether to capture response body previews. Default: true */
+    captureResponsePreview?: boolean;
+    /** Filter: skip requests matching these URL patterns. Default: skip none.
+     *  Accepts RegExp or string (strings are auto-converted to case-insensitive RegExp). */
+    excludePatterns?: (RegExp | string)[];
+    /** Filter: only record requests matching these URL patterns. Default: record all.
+     *  Accepts RegExp or string (strings are auto-converted to case-insensitive RegExp). */
+    includePatterns?: (RegExp | string)[];
+    /** Maximum request body bytes to capture. Default: 4096 */
+    maxBodyBytes?: number;
+    /** Maximum response body bytes to capture in preview. Default: 2048 */
+    maxPreviewBytes?: number;
+}
+export declare class RequestRecorder {
+    private observations;
+    private options;
+    private provider;
+    private running;
+    private seq;
+    private sessionId;
+    private startTime;
+    private taskDescription;
+    constructor(options?: RecorderOptions);
+    /**
+     * A `fetch`-compatible wrapper that records the request and response,
+     * then returns the real response unmodified.
+     *
+     * Drop-in replacement: `const response = await recorder.fetch(url, init)`
+     */
+    fetch(input: Request | string | URL, init?: RequestInit): Promise<Response>;
+    /**
+     * Whether the recorder is currently active.
+     */
+    isRunning(): boolean;
+    /**
+     * Get current observations without stopping the recorder.
+     */
+    peek(): ObservedRequest[];
+    /**
+     * Record an externally-observed request/response pair.
+     *
+     * Use this when you can't wrap `fetch` directly but can observe traffic
+     * (e.g., via browser DevTools Protocol, mitmproxy logs, etc.).
+     */
+    record(observation: Omit<ObservedRequest, "seq">): void;
+    /**
+     * Reset the recorder for reuse without creating a new instance.
+     */
+    reset(): void;
+    /**
+     * Start a new recording session. Call this before the agent runs a task.
+     */
+    start(sessionId: string, provider: string, taskDescription: string): void;
+    /**
+     * Stop recording and return the complete behavior log with
+     * classified observations and summary statistics.
+     */
+    stop(): AgentBehaviorLog;
+    private computeSummary;
+    private extractBody;
+    private extractHeaders;
+    private redactSensitive;
+}

package/dist/agent-observer/proxy.js

@@ -0,0 +1,321 @@
+/**
+ * proxy.ts
+ *
+ * A lightweight HTTP recording proxy that captures all outbound requests
+ * made by an AI agent during evaluation. Instead of being an actual network
+ * proxy (which requires HTTPS MITM certificates), this module provides:
+ *
+ *   1. A `fetch` wrapper that records every request/response
+ *   2. A session manager that groups observations by test case
+ *
+ * This design works with any agent framework that uses `fetch` or can be
+ * configured to use a custom HTTP client. For agents that use a real browser,
+ * see the Chrome DevTools Protocol integration (future work).
+ *
+ * Usage:
+ *   const recorder = new RequestRecorder()
+ *   recorder.start('session-123')
+ *
+ *   // The agent makes requests through the recorder's fetch wrapper
+ *   const response = await recorder.fetch('https://sanity.io/docs/groq')
+ *
+ *   const log = recorder.stop()
+ *   // → AgentBehaviorLog with all requests classified
+ */
+import { classifyRequests } from "./classifier.js";
+const DEFAULT_OPTIONS = {
+    captureHeaders: [
+        "accept",
+        "content-type",
+        "user-agent",
+        "authorization",
+        "x-sanity-token",
+    ],
+    captureResponsePreview: true,
+    excludePatterns: [
+        // Skip common noise
+        /\.(woff2?|ttf|eot)(\?|$)/i,
+        /\.(png|jpg|jpeg|gif|svg|ico|webp)(\?|$)/i,
+        /favicon/i,
+    ],
+    includePatterns: [],
+    maxBodyBytes: 4096,
+    maxPreviewBytes: 2048,
+};
+// ---------------------------------------------------------------------------
+// RequestRecorder
+// ---------------------------------------------------------------------------
+export class RequestRecorder {
+    observations = [];
+    options;
+    provider = "";
+    running = false;
+    seq = 0;
+    sessionId = "";
+    startTime = 0;
+    taskDescription = "";
+    constructor(options) {
+        const merged = { ...DEFAULT_OPTIONS, ...options };
+        // Coerce string patterns from YAML config into RegExp instances
+        if (merged.includePatterns) {
+            merged.includePatterns = merged.includePatterns.map(toRegExp);
+        }
+        if (merged.excludePatterns) {
+            merged.excludePatterns = merged.excludePatterns.map(toRegExp);
+        }
+        this.options = merged;
+    }
+    /**
+     * A `fetch`-compatible wrapper that records the request and response,
+     * then returns the real response unmodified.
+     *
+     * Drop-in replacement: `const response = await recorder.fetch(url, init)`
+     */
+    async fetch(input, init) {
+        const reqStart = Date.now();
+        const url = typeof input === "string"
+            ? input
+            : input instanceof URL
+                ? input.toString()
+                : input.url;
+        const method = init?.method ??
+            (typeof input !== "string" && !(input instanceof URL)
+                ? input.method
+                : "GET") ??
+            "GET";
+        let response;
+        let error = null;
+        try {
+            response = await globalThis.fetch(input, init);
+        }
+        catch (err) {
+            error = err;
+            // Record the failed request
+            this.record({
+                body: await this.extractBody(init?.body),
+                contentType: undefined,
+                headers: this.extractHeaders(init?.headers),
+                latencyMs: Date.now() - reqStart,
+                method: method.toUpperCase(),
+                responsePreview: `Error: ${error.message}`,
+                responseSize: 0,
+                statusCode: 0,
+                timestamp: new Date(reqStart).toISOString(),
+                url,
+            });
+            throw error;
+        }
+        const latencyMs = Date.now() - reqStart;
+        // Clone the response so we can read the body without consuming it
+        const clone = response.clone();
+        let responseSize = 0;
+        let responsePreview;
+        if (this.options.captureResponsePreview) {
+            try {
+                const text = await clone.text();
+                responseSize = new TextEncoder().encode(text).length;
+                responsePreview = text.slice(0, this.options.maxPreviewBytes);
+            }
+            catch {
+                // Body might not be text — that's fine
+                responseSize = parseInt(response.headers.get("content-length") ?? "0", 10);
+            }
+        }
+        this.record({
+            body: await this.extractBody(init?.body),
+            contentType: response.headers.get("content-type") ?? undefined,
+            headers: this.extractHeaders(init?.headers),
+            latencyMs,
+            method: method.toUpperCase(),
+            responsePreview,
+            responseSize,
+            statusCode: response.status,
+            timestamp: new Date(reqStart).toISOString(),
+            url,
+        });
+        return response;
+    }
+    /**
+     * Whether the recorder is currently active.
+     */
+    isRunning() {
+        return this.running;
+    }
+    /**
+     * Get current observations without stopping the recorder.
+     */
+    peek() {
+        return [...this.observations];
+    }
+    /**
+     * Record an externally-observed request/response pair.
+     *
+     * Use this when you can't wrap `fetch` directly but can observe traffic
+     * (e.g., via browser DevTools Protocol, mitmproxy logs, etc.).
+     */
+    record(observation) {
+        if (!this.running)
+            return;
+        const url = observation.url;
+        // Apply filters
+        if (this.options.includePatterns.length > 0) {
+            if (!this.options.includePatterns.some((p) => p.test(url)))
+                return;
+        }
+        if (this.options.excludePatterns.some((p) => p.test(url)))
+            return;
+        this.observations.push({
+            ...observation,
+            // Truncate body if needed
+            body: observation.body?.slice(0, this.options.maxBodyBytes),
+            responsePreview: observation.responsePreview?.slice(0, this.options.maxPreviewBytes),
+            seq: this.seq++,
+        });
+    }
+    /**
+     * Reset the recorder for reuse without creating a new instance.
+     */
+    reset() {
+        this.observations = [];
+        this.seq = 0;
+        this.sessionId = "";
+        this.provider = "";
+        this.taskDescription = "";
+        this.startTime = 0;
+        this.running = false;
+    }
+    /**
+     * Start a new recording session. Call this before the agent runs a task.
+     */
+    start(sessionId, provider, taskDescription) {
+        this.sessionId = sessionId;
+        this.provider = provider;
+        this.taskDescription = taskDescription;
+        this.observations = [];
+        this.seq = 0;
+        this.startTime = Date.now();
+        this.running = true;
+    }
+    /**
+     * Stop recording and return the complete behavior log with
+     * classified observations and summary statistics.
+     */
+    stop() {
+        this.running = false;
+        const endTime = Date.now();
+        const classified = classifyRequests(this.observations);
+        const summary = this.computeSummary();
+        return {
+            endedAt: new Date(endTime).toISOString(),
+            networkDurationMs: this.observations.reduce((sum, r) => sum + r.latencyMs, 0),
+            provider: this.provider,
+            requests: this.observations,
+            sessionId: this.sessionId,
+            startedAt: new Date(this.startTime).toISOString(),
+            taskDescription: this.taskDescription,
+            totalDurationMs: endTime - this.startTime,
+            ...classified,
+            summary,
+        };
+    }
+    // -------------------------------------------------------------------------
+    // Private helpers
+    // -------------------------------------------------------------------------
+    computeSummary() {
+        const classified = classifyRequests(this.observations);
+        const uniqueUrls = new Set(this.observations.map((r) => r.url));
+        const docSlugs = [
+            ...new Set(classified.docPageVisits.map((d) => d.slug)),
+        ].filter(Boolean);
+        const searchQueries = [
+            ...new Set(classified.searchQueries.map((s) => s.query)),
+        ].filter(Boolean);
+        const extDomains = [
+            ...new Set(classified.externalRequests.map((e) => e.domain)),
+        ];
+        return {
+            apiCallCount: classified.apiCalls.length,
+            docPagesVisited: classified.docPageVisits.length,
+            docSlugsVisited: docSlugs,
+            externalDomains: extDomains,
+            externalRequestCount: classified.externalRequests.length,
+            searchesPerformed: classified.searchQueries.length,
+            totalBytesDownloaded: this.observations.reduce((sum, r) => sum + r.responseSize, 0),
+            totalNetworkMs: this.observations.reduce((sum, r) => sum + r.latencyMs, 0),
+            totalRequests: this.observations.length,
+            uniqueSearchQueries: searchQueries,
+            uniqueUrls: uniqueUrls.size,
+            usedDocs: classified.docPageVisits.length > 0,
+            usedSearch: classified.searchQueries.length > 0,
+        };
+    }
+    async extractBody(body) {
+        if (!body)
+            return undefined;
+        if (typeof body === "string") {
+            return body.slice(0, this.options.maxBodyBytes);
+        }
+        // For other body types, try to convert
+        try {
+            if (body instanceof ArrayBuffer) {
+                return new TextDecoder()
+                    .decode(body)
+                    .slice(0, this.options.maxBodyBytes);
+            }
+            if (body instanceof Blob) {
+                const text = await body.text();
+                return text.slice(0, this.options.maxBodyBytes);
+            }
+        }
+        catch {
+            return "[binary body]";
+        }
+        return undefined;
+    }
+    extractHeaders(headers) {
+        const result = {};
+        if (!headers)
+            return result;
+        const allowed = new Set(this.options.captureHeaders.map((h) => h.toLowerCase()));
+        if (headers instanceof Headers) {
+            headers.forEach((value, key) => {
+                if (allowed.has(key.toLowerCase())) {
+                    result[key.toLowerCase()] = this.redactSensitive(key, value);
+                }
+            });
+        }
+        else if (Array.isArray(headers)) {
+            for (const [key, value] of headers) {
+                if (allowed.has(key.toLowerCase())) {
+                    result[key.toLowerCase()] = this.redactSensitive(key, value);
+                }
+            }
+        }
+        else {
+            for (const [key, value] of Object.entries(headers)) {
+                if (allowed.has(key.toLowerCase())) {
+                    result[key.toLowerCase()] = this.redactSensitive(key, value);
+                }
+            }
+        }
+        return result;
+    }
+    redactSensitive(key, value) {
+        const sensitive = ["authorization", "x-sanity-token", "cookie"];
+        if (sensitive.includes(key.toLowerCase())) {
+            return value.slice(0, 8) + "...[REDACTED]";
+        }
+        return value;
+    }
+}
+/**
+ * Coerce a pattern that may be a string (from YAML config) into a RegExp.
+ * If it is already a RegExp, return as-is.
+ */
+function toRegExp(pattern) {
+    if (pattern instanceof RegExp)
+        return pattern;
+    // Simple containment match — treat string as a literal substring
+    const escaped = String(pattern).replace(/[-/\\^$*+?.()|[\]{}]/g, "\\$&");
+    return new RegExp(escaped, "i");
+}

package/dist/agent-observer/test-imports.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Final validation — ensures all agent-observer modules work together
+ * and the full data pipeline (record → classify → summarize) is correct.
+ *
+ * Run: tsx src/agent-observer/test-imports.ts
+ */
+export {};

package/dist/agent-observer/test-imports.js

@@ -0,0 +1,185 @@
+/**
+ * Final validation — ensures all agent-observer modules work together
+ * and the full data pipeline (record → classify → summarize) is correct.
+ *
+ * Run: tsx src/agent-observer/test-imports.ts
+ */
+import { classifyRequests, extractDocSlug, extractSearchQuery, extractApiEndpoint, extractDomain, extractPageTitle, isDocPageRequest, isSearchRequest, isSanityApiRequest, } from "./classifier.js";
+import { RequestRecorder } from "./proxy.js";
+import { default as InstrumentedProvider } from "./provider.js";
+// ─── Test data ───────────────────────────────────────────────────────────────
+const now = Date.now();
+const mockRequests = [
+    // 1. Doc page visit
+    {
+        headers: {},
+        latencyMs: 234,
+        method: "GET",
+        responsePreview: "<html><head><title>Create a Schema - Sanity</title></head>...",
+        responseSize: 45000,
+        seq: 0,
+        statusCode: 200,
+        timestamp: new Date(now).toISOString(),
+        url: "https://www.sanity.io/docs/create-a-schema-and-configure-sanity-studio",
+    },
+    // 2. Another doc page
+    {
+        headers: {},
+        latencyMs: 180,
+        method: "GET",
+        responsePreview: "<html><head><title>Object Type - Sanity Docs</title></head>...",
+        responseSize: 32000,
+        seq: 1,
+        statusCode: 200,
+        timestamp: new Date(now + 100).toISOString(),
+        url: "https://www.sanity.io/docs/schema-types/object-type",
+    },
+    // 3. Search query
+    {
+        headers: {},
+        latencyMs: 450,
+        method: "GET",
+        responseSize: 12000,
+        seq: 2,
+        statusCode: 200,
+        timestamp: new Date(now + 200).toISOString(),
+        url: "https://www.sanity.io/search?q=visual+editing+preview",
+    },
+    // 4. Sanity API call (GROQ query — must NOT be classified as search)
+    {
+        headers: {},
+        latencyMs: 320,
+        method: "GET",
+        responseSize: 8500,
+        seq: 3,
+        statusCode: 200,
+        timestamp: new Date(now + 300).toISOString(),
+        url: 'https://api.sanity.io/v2021-03-25/data/query/production?query=*[_type=="article"]',
+    },
+    // 5. CDN API call
+    {
+        headers: {},
+        latencyMs: 85,
+        method: "GET",
+        responseSize: 150000,
+        seq: 4,
+        statusCode: 200,
+        timestamp: new Date(now + 350).toISOString(),
+        url: "https://cdn.sanity.io/images/abc123/production/image-xyz.jpg",
+    },
+    // 6. External request (npm docs)
+    {
+        headers: {},
+        latencyMs: 300,
+        method: "GET",
+        responseSize: 20000,
+        seq: 5,
+        statusCode: 200,
+        timestamp: new Date(now + 400).toISOString(),
+        url: "https://docs.npmjs.com/cli/install",
+    },
+    // 7. Algolia search
+    {
+        body: JSON.stringify({ query: "presentation tool setup" }),
+        headers: {},
+        latencyMs: 150,
+        method: "POST",
+        responseSize: 5000,
+        seq: 6,
+        statusCode: 200,
+        timestamp: new Date(now + 500).toISOString(),
+        url: "https://abc123.algolia.net/1/indexes/sanity_docs/query",
+    },
+    // 8. Google search
+    {
+        headers: {},
+        latencyMs: 200,
+        method: "GET",
+        responseSize: 80000,
+        seq: 7,
+        statusCode: 200,
+        timestamp: new Date(now + 600).toISOString(),
+        url: "https://www.google.com/search?q=sanity+studio+custom+tool",
+    },
+    // 9. Failed request (should be skipped)
+    {
+        headers: {},
+        latencyMs: 0,
+        method: "GET",
+        responseSize: 0,
+        seq: 8,
+        statusCode: 0,
+        timestamp: new Date(now + 700).toISOString(),
+        url: "https://www.sanity.io/docs/nonexistent-page",
+    },
+];
+// ─── Run tests ───────────────────────────────────────────────────────────────
+let passed = 0;
+let failed = 0;
+function assert(condition, msg) {
+    if (condition) {
+        console.log(`  ✅ ${msg}`);
+        passed++;
+    }
+    else {
+        console.log(`  ❌ FAIL: ${msg}`);
+        failed++;
+    }
+}
+console.log("\n═══ Agent Observer — Final Validation ═══\n");
+// --- Individual function tests ---
+console.log("1. Individual detection functions:");
+assert(isDocPageRequest(mockRequests[0]) === true, "Doc page detected");
+assert(isDocPageRequest(mockRequests[3]) === false, "API call NOT detected as doc page");
+assert(isSearchRequest(mockRequests[2]) === true, "Search detected");
+assert(isSearchRequest(mockRequests[3]) === false, "API call NOT detected as search");
+assert(isSanityApiRequest(mockRequests[3]) === true, "API call detected");
+assert(isSanityApiRequest(mockRequests[0]) === false, "Doc page NOT detected as API call");
+// --- Slug extraction ---
+console.log("\n2. Metadata extraction:");
+assert(extractDocSlug("https://www.sanity.io/docs/create-a-schema-and-configure-sanity-studio") === "create-a-schema-and-configure-sanity-studio", "Doc slug extracted correctly");
+assert(extractDocSlug("https://www.sanity.io/docs/schema-types/object-type") ===
+    "schema-types/object-type", "Nested doc slug extracted correctly");
+assert(extractSearchQuery(mockRequests[2]) === "visual editing preview", `Search query extracted: "${extractSearchQuery(mockRequests[2])}"`);
+assert(extractApiEndpoint("https://api.sanity.io/v2021-03-25/data/query/production?query=*") === "/data/query/production?query=*", `API endpoint extracted: "${extractApiEndpoint("https://api.sanity.io/v2021-03-25/data/query/production?query=*")}"`);
+assert(extractDomain("https://docs.npmjs.com/cli/install") === "docs.npmjs.com", "Domain extracted correctly");
+assert(extractPageTitle("<html><head><title>Create a Schema - Sanity</title></head>") === "Create a Schema - Sanity", "Page title extracted correctly");
+// --- Full classification pipeline ---
+console.log("\n3. Full classification pipeline:");
+const classified = classifyRequests(mockRequests);
+assert(classified.docPageVisits.length === 2, `Doc pages: ${classified.docPageVisits.length} (expected 2)`);
+assert(classified.searchQueries.length === 3, `Searches: ${classified.searchQueries.length} (expected 3 — site search + algolia + google)`);
+assert(classified.apiCalls.length === 2, `API calls: ${classified.apiCalls.length} (expected 2 — GROQ query + CDN)`);
+assert(classified.externalRequests.length === 1, `External: ${classified.externalRequests.length} (expected 1 — npmjs)`);
+// Verify the API ?query= param was NOT classified as a search
+const searchUrls = classified.searchQueries.map((s) => s.url);
+assert(!searchUrls.includes('https://api.sanity.io/v2021-03-25/data/query/production?query=*[_type=="article"]'), "API ?query= param NOT misclassified as search");
+// Verify the failed request was skipped
+const allUrls = [
+    ...classified.docPageVisits.map((d) => d.url),
+    ...classified.searchQueries.map((s) => s.url),
+    ...classified.apiCalls.map((a) => a.url),
+    ...classified.externalRequests.map((e) => e.url),
+];
+assert(!allUrls.includes("https://www.sanity.io/docs/nonexistent-page"), "Failed request (status 0) was skipped");
+// --- RequestRecorder ---
+console.log("\n4. RequestRecorder:");
+const recorder = new RequestRecorder({ includePatterns: [/sanity\.io/] });
+assert(typeof recorder.start === "function", "RequestRecorder.start() exists");
+assert(typeof recorder.stop === "function", "RequestRecorder.stop() exists");
+// --- InstrumentedProvider ---
+console.log("\n5. InstrumentedProvider:");
+const provider = new InstrumentedProvider({ config: {}, id: "test-validation" });
+assert(provider.id() === "instrumented:test-validation", `Provider ID: "${provider.id()}"`);
+assert(typeof provider.callApi === "function", "Provider.callApi() exists");
+// --- Summary ---
+console.log(`\n${"═".repeat(50)}`);
+console.log(`Results: ${passed} passed, ${failed} failed, ${passed + failed} total`);
+if (failed > 0) {
+    console.log("\n⚠️  Some tests failed!");
+    process.exit(1);
+}
+else {
+    console.log("\n✅ All tests passed! Agent observer system is ready.");
+    process.exit(0);
+}