@intx/inference-discovery 0.1.2

package/src/runner.ts

@@ -0,0 +1,242 @@
+import path from "node:path";
+import fs from "node:fs/promises";
+import type { Capability, CapabilityIntent } from "./catalog";
+import { detectResponseKind } from "./content-type";
+import { buildManifest } from "./manifest";
+import type { CaptureStep, CapturedResponse, ProviderPlugin } from "./plugin";
+import {
+  writeCapture,
+  type RequestBody,
+  type ResponseBody,
+  type WriteCaptureInput,
+} from "./write-capture";
+
+export type FetchLike = (
+  input: string,
+  init: {
+    method: string;
+    headers: Record<string, string>;
+    body: string | Uint8Array;
+  },
+) => Promise<Response>;
+
+export interface RunCaptureOpts {
+  plugin: ProviderPlugin;
+  model: string;
+  capability: Capability;
+  intent: CapabilityIntent;
+  outDir: string;
+  now?: () => Date;
+  fetch?: FetchLike;
+}
+
+const REASONING_TRACE_CAPABILITY_PREFIXES = [
+  "reasoning-content",
+  "redacted-thinking",
+] as const;
+
+function shouldEmitReasoningTrace(capability: Capability): boolean {
+  for (const prefix of REASONING_TRACE_CAPABILITY_PREFIXES) {
+    if (capability.startsWith(prefix)) return true;
+  }
+  return false;
+}
+
+function headersToObject(headers: Headers): Record<string, string> {
+  const out: Record<string, string> = {};
+  headers.forEach((value, key) => {
+    out[key] = value;
+  });
+  return out;
+}
+
+const defaultFetch: FetchLike = (input, init) =>
+  fetch(input, {
+    method: init.method,
+    headers: init.headers,
+    body: init.body,
+  });
+
+function mergeHeaders(
+  defaults: Record<string, string>,
+  stepHeaders: Record<string, string>,
+  authHeaders: Record<string, string>,
+): Record<string, string> {
+  const authKeys = new Set(
+    Object.keys(authHeaders).map((k) => k.toLowerCase()),
+  );
+  for (const key of Object.keys(stepHeaders)) {
+    if (authKeys.has(key.toLowerCase())) {
+      throw new Error(
+        `capture step attempted to override plug-in auth header '${key}'; ` +
+          `auth headers are plug-in-wide and cannot be overridden per step`,
+      );
+    }
+  }
+  // Default → step overrides default → auth wins over everything.
+  return { ...defaults, ...stepHeaders, ...authHeaders };
+}
+
+function buildRequestForStep(
+  step: CaptureStep,
+  authHeaders: Record<string, string>,
+): {
+  method: string;
+  headers: Record<string, string>;
+  body: string | Uint8Array;
+  request: RequestBody;
+} {
+  const method = step.method ?? "POST";
+  const stepHeaders = step.headers ?? {};
+  if (step.kind === "raw") {
+    const headers = mergeHeaders(
+      { "Content-Type": step.contentType },
+      stepHeaders,
+      authHeaders,
+    );
+    return {
+      method,
+      headers,
+      body: step.body,
+      request: { kind: "raw", bytes: step.body, contentType: step.contentType },
+    };
+  }
+  const headers = mergeHeaders(
+    { "Content-Type": "application/json" },
+    stepHeaders,
+    authHeaders,
+  );
+  return {
+    method,
+    headers,
+    body: JSON.stringify(step.body),
+    request: { kind: "json", body: step.body },
+  };
+}
+
+async function captureStep(args: {
+  step: CaptureStep;
+  outDir: string;
+  plugin: ProviderPlugin;
+  capability: Capability;
+  doFetch: FetchLike;
+}): Promise<CapturedResponse> {
+  const { step, outDir, plugin, capability, doFetch } = args;
+
+  const stepDir =
+    step.subdir === null ? outDir : path.join(outDir, step.subdir);
+
+  const authHeaders = plugin.buildAuthHeaders();
+  const {
+    method,
+    headers: requestHeaders,
+    body,
+    request,
+  } = buildRequestForStep(step, authHeaders);
+
+  const response = await doFetch(step.url, {
+    method,
+    headers: requestHeaders,
+    body,
+  });
+
+  const responseHeaders = headersToObject(response.headers);
+  const kind = detectResponseKind(response.headers);
+
+  let captured: ResponseBody;
+  let parsedForGenerator: unknown | null;
+  let bytesForGenerator: Uint8Array | null;
+  // Read the body as raw bytes regardless of kind so the capture
+  // can write them verbatim. For JSON responses we also parse for
+  // the generator's reasoning-trace extraction path, but the bytes
+  // we write to disk are the original network bytes (not a
+  // re-serialised pretty-print of the parsed value).
+  const buf = await response.arrayBuffer();
+  const bytes = new Uint8Array(buf);
+  if (kind === "sse") {
+    captured = { kind: "sse", bytes };
+    parsedForGenerator = null;
+    bytesForGenerator = bytes;
+  } else {
+    const text = new TextDecoder().decode(bytes);
+    const parsed: unknown = JSON.parse(text);
+    captured = { kind: "json", bytes, parsed };
+    parsedForGenerator = parsed;
+    bytesForGenerator = null;
+  }
+
+  const captureInput: WriteCaptureInput = {
+    request,
+    requestHeaders,
+    redactRequestHeaders: plugin.redactRequestHeaders,
+    response: captured,
+    responseHeaders,
+    redactResponseHeaders: plugin.redactResponseHeaders,
+  };
+
+  await writeCapture(stepDir, captureInput);
+
+  if (
+    plugin.extractReasoningTrace !== undefined &&
+    shouldEmitReasoningTrace(capability) &&
+    parsedForGenerator !== null
+  ) {
+    const trace = plugin.extractReasoningTrace(parsedForGenerator);
+    if (trace !== null) {
+      await fs.writeFile(
+        path.join(stepDir, "reasoning-trace.json"),
+        `${JSON.stringify(trace, null, 2)}\n`,
+      );
+    }
+  }
+
+  return {
+    status: response.status,
+    headers: responseHeaders,
+    parsed: parsedForGenerator,
+    bytes: bytesForGenerator,
+  };
+}
+
+export async function runCapture(opts: RunCaptureOpts): Promise<void> {
+  const { plugin, model, capability, intent, outDir } = opts;
+  const doFetch = opts.fetch ?? defaultFetch;
+
+  const iterator = plugin.iterateCaptureSteps({ model, capability, intent });
+
+  let stepsExecuted = 0;
+  let iterResult = iterator.next();
+  while (!iterResult.done) {
+    const captured = await captureStep({
+      step: iterResult.value,
+      outDir,
+      plugin,
+      capability,
+      doFetch,
+    });
+    stepsExecuted += 1;
+    iterResult = iterator.next(captured);
+  }
+
+  if (stepsExecuted === 0) {
+    throw new Error(
+      `plug-in ${plugin.name} produced no capture steps for ${model}/${capability}`,
+    );
+  }
+
+  const manifestOpts: Parameters<typeof buildManifest>[0] = {
+    provider: plugin.name,
+    model,
+    capability,
+  };
+  if (opts.now !== undefined) {
+    manifestOpts.now = opts.now;
+  }
+  const manifest = buildManifest(manifestOpts);
+
+  await fs.mkdir(outDir, { recursive: true });
+  await fs.writeFile(
+    path.join(outDir, "manifest.json"),
+    `${JSON.stringify(manifest, null, 2)}\n`,
+  );
+}

package/src/write-capture.test.ts

@@ -0,0 +1,168 @@
+import { describe, test, expect, beforeEach, afterEach } from "bun:test";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { writeCapture } from "./write-capture";
+
+async function makeTempDir(): Promise<string> {
+  return await fs.mkdtemp(path.join(os.tmpdir(), "write-capture-test-"));
+}
+
+describe("writeCapture", () => {
+  let dir: string;
+
+  beforeEach(async () => {
+    dir = await makeTempDir();
+  });
+
+  afterEach(async () => {
+    await fs.rm(dir, { recursive: true, force: true });
+  });
+
+  test("writes JSON request body and response/metadata files", async () => {
+    // Use a non-default key order and trailing whitespace in the
+    // response bytes to prove writeCapture preserves the body
+    // byte-identical rather than re-formatting it via JSON.stringify.
+    const originalResponseText = `{"text":"hello","seq":1}  \n`;
+    const originalResponseBytes = new TextEncoder().encode(
+      originalResponseText,
+    );
+    await writeCapture(dir, {
+      request: {
+        kind: "json",
+        body: { messages: [{ role: "user", content: "hi" }] },
+      },
+      requestHeaders: {
+        "Content-Type": "application/json",
+        "X-Goog-Api-Key": "secret",
+      },
+      redactRequestHeaders: ["x-goog-api-key"],
+      response: {
+        kind: "json",
+        bytes: originalResponseBytes,
+        parsed: { text: "hello", seq: 1 },
+      },
+      responseHeaders: { "Content-Type": "application/json" },
+      redactResponseHeaders: [],
+    });
+
+    const entries = (await fs.readdir(dir)).sort();
+    expect(entries).toEqual([
+      "request-headers.json",
+      "request.json",
+      "response-headers.json",
+      "response.json",
+    ]);
+
+    const requestHeaders = JSON.parse(
+      await fs.readFile(path.join(dir, "request-headers.json"), "utf8"),
+    );
+    expect(requestHeaders["X-Goog-Api-Key"]).toBe("<REDACTED>");
+    expect(requestHeaders["Content-Type"]).toBe("application/json");
+
+    const writtenResponse = await fs.readFile(path.join(dir, "response.json"));
+    expect(Array.from(writtenResponse)).toEqual(
+      Array.from(originalResponseBytes),
+    );
+  });
+
+  test("writes a raw request body to request.bin verbatim", async () => {
+    const bytes = new Uint8Array([0x25, 0x50, 0x44, 0x46, 0x2d, 0x31, 0x2e]);
+    await writeCapture(dir, {
+      request: {
+        kind: "raw",
+        bytes,
+        contentType: "application/pdf",
+      },
+      requestHeaders: {
+        "Content-Type": "application/pdf",
+        "X-Goog-Api-Key": "secret",
+      },
+      redactRequestHeaders: ["x-goog-api-key"],
+      response: {
+        kind: "json",
+        bytes: new TextEncoder().encode('{"fileId":"abc"}'),
+        parsed: { fileId: "abc" },
+      },
+      responseHeaders: { "Content-Type": "application/json" },
+      redactResponseHeaders: [],
+    });
+
+    const entries = (await fs.readdir(dir)).sort();
+    expect(entries).toEqual([
+      "request-headers.json",
+      "request.bin",
+      "response-headers.json",
+      "response.json",
+    ]);
+
+    const written = await fs.readFile(path.join(dir, "request.bin"));
+    expect(Array.from(written)).toEqual(Array.from(bytes));
+
+    const requestHeaders = JSON.parse(
+      await fs.readFile(path.join(dir, "request-headers.json"), "utf8"),
+    );
+    expect(requestHeaders["X-Goog-Api-Key"]).toBe("<REDACTED>");
+    expect(requestHeaders["Content-Type"]).toBe("application/pdf");
+  });
+
+  test("writes SSE response as response.sse and not response.json", async () => {
+    const bytes = new TextEncoder().encode("data: hello\n\n");
+    await writeCapture(dir, {
+      request: { kind: "json", body: {} },
+      requestHeaders: {},
+      redactRequestHeaders: [],
+      response: { kind: "sse", bytes },
+      responseHeaders: { "Content-Type": "text/event-stream" },
+      redactResponseHeaders: [],
+    });
+    const entries = (await fs.readdir(dir)).sort();
+    expect(entries).toContain("response.sse");
+    expect(entries).not.toContain("response.json");
+    const written = await fs.readFile(path.join(dir, "response.sse"));
+    expect(new TextDecoder().decode(written)).toBe("data: hello\n\n");
+  });
+
+  test("redaction is case-insensitive on header names", async () => {
+    await writeCapture(dir, {
+      request: { kind: "json", body: {} },
+      requestHeaders: { Authorization: "Bearer xyz" },
+      redactRequestHeaders: ["AUTHORIZATION"],
+      response: {
+        kind: "json",
+        bytes: new TextEncoder().encode("{}"),
+        parsed: {},
+      },
+      responseHeaders: { "Set-Cookie": "abc=1" },
+      redactResponseHeaders: ["set-cookie"],
+    });
+
+    const requestHeaders = JSON.parse(
+      await fs.readFile(path.join(dir, "request-headers.json"), "utf8"),
+    );
+    expect(requestHeaders.Authorization).toBe("<REDACTED>");
+
+    const responseHeaders = JSON.parse(
+      await fs.readFile(path.join(dir, "response-headers.json"), "utf8"),
+    );
+    expect(responseHeaders["Set-Cookie"]).toBe("<REDACTED>");
+  });
+
+  test("creates target directory if it does not exist", async () => {
+    const nested = path.join(dir, "a", "b", "c");
+    await writeCapture(nested, {
+      request: { kind: "json", body: {} },
+      requestHeaders: {},
+      redactRequestHeaders: [],
+      response: {
+        kind: "json",
+        bytes: new TextEncoder().encode("{}"),
+        parsed: {},
+      },
+      responseHeaders: {},
+      redactResponseHeaders: [],
+    });
+    const entries = await fs.readdir(nested);
+    expect(entries).toContain("request.json");
+  });
+});

package/src/write-capture.ts

@@ -0,0 +1,83 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+
+/**
+ * Captured response body. The on-disk file (response.json or
+ * response.sse) is written from `bytes` verbatim so the recording
+ * is byte-identical to what the server sent — pretty-printing
+ * a parsed JSON body would lose original key order, trailing
+ * whitespace, and any content-length / signature semantics. The
+ * optional `parsed` field is kept for the discovery rig's
+ * extractReasoningTrace path, which needs the JSON-decoded value
+ * to pull provider-specific metadata; writeCapture itself never
+ * reads it.
+ */
+export type ResponseBody =
+  | { kind: "json"; bytes: Uint8Array; parsed: unknown }
+  | { kind: "sse"; bytes: Uint8Array };
+
+export type RequestBody =
+  | { kind: "json"; body: unknown }
+  | { kind: "raw"; bytes: Uint8Array; contentType: string };
+
+export interface WriteCaptureInput {
+  request: RequestBody;
+  requestHeaders: Record<string, string>;
+  redactRequestHeaders: readonly string[];
+  response: ResponseBody;
+  responseHeaders: Record<string, string>;
+  redactResponseHeaders: readonly string[];
+}
+
+const REDACTED = "<REDACTED>";
+
+function applyRedaction(
+  headers: Record<string, string>,
+  redactNames: readonly string[],
+): Record<string, string> {
+  const lowered = new Set(redactNames.map((name) => name.toLowerCase()));
+  const result: Record<string, string> = {};
+  for (const [key, value] of Object.entries(headers)) {
+    result[key] = lowered.has(key.toLowerCase()) ? REDACTED : value;
+  }
+  return result;
+}
+
+export async function writeCapture(
+  dir: string,
+  input: WriteCaptureInput,
+): Promise<void> {
+  await fs.mkdir(dir, { recursive: true });
+
+  const redactedRequest = applyRedaction(
+    input.requestHeaders,
+    input.redactRequestHeaders,
+  );
+  const redactedResponse = applyRedaction(
+    input.responseHeaders,
+    input.redactResponseHeaders,
+  );
+
+  if (input.request.kind === "json") {
+    await fs.writeFile(
+      path.join(dir, "request.json"),
+      `${JSON.stringify(input.request.body, null, 2)}\n`,
+    );
+  } else {
+    await fs.writeFile(path.join(dir, "request.bin"), input.request.bytes);
+  }
+  await fs.writeFile(
+    path.join(dir, "request-headers.json"),
+    `${JSON.stringify(redactedRequest, null, 2)}\n`,
+  );
+  await fs.writeFile(
+    path.join(dir, "response-headers.json"),
+    `${JSON.stringify(redactedResponse, null, 2)}\n`,
+  );
+
+  if (input.response.kind === "json") {
+    await fs.writeFile(path.join(dir, "response.json"), input.response.bytes);
+  } else {
+    await fs.writeFile(path.join(dir, "response.sse"), input.response.bytes);
+  }
+}

package/tsconfig.json

@@ -0,0 +1,4 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "include": ["src/**/*.ts"]
+}