@complior/engine 0.9.0

package/src/domain/eval/adapters/adapters.test.ts

@@ -0,0 +1,303 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { createHttpAdapter } from './http-adapter.js';
+import { createOpenAIAdapter } from './openai-adapter.js';
+import { createAnthropicAdapter } from './anthropic-adapter.js';
+import { createOllamaAdapter } from './ollama-adapter.js';
+import { createCustomAdapter } from './custom-adapter.js';
+import { autoDetectAdapter } from './auto-detect.js';
+import { safeJsonParse, AdapterError, withRetry } from './adapter-port.js';
+
+// ── Mock fetch ──────────────────────────────────────────────────
+
+const mockFetch = vi.fn();
+
+beforeEach(() => {
+  vi.stubGlobal('fetch', mockFetch);
+});
+
+afterEach(() => {
+  vi.restoreAllMocks();
+});
+
+const jsonResponse = (body: unknown, status = 200, headers: Record<string, string> = {}) => ({
+  ok: status >= 200 && status < 300,
+  status,
+  json: async () => body,
+  headers: new Map(Object.entries(headers)),
+});
+
+// ── HTTP Adapter ────────────────────────────────────────────────
+
+describe('createHttpAdapter', () => {
+  it('sends probe as {message}', async () => {
+    mockFetch.mockResolvedValue(jsonResponse({ response: 'I am an AI assistant.' }));
+    const adapter = createHttpAdapter('http://localhost:4000/api/chat');
+    const result = await adapter.send('Who are you?');
+
+    expect(result.text).toBe('I am an AI assistant.');
+    expect(result.status).toBe(200);
+    expect(result.latencyMs).toBeGreaterThanOrEqual(0);
+
+    const body = JSON.parse(mockFetch.mock.calls[0]![1]!.body as string);
+    expect(body.message).toBe('Who are you?');
+  });
+
+  it('falls back to JSON.stringify for unknown response shape', async () => {
+    mockFetch.mockResolvedValue(jsonResponse({ data: { text: 'hello' } }));
+    const adapter = createHttpAdapter('http://localhost:4000');
+    const result = await adapter.send('test');
+    expect(result.text).toContain('hello');
+  });
+
+  it('sendMultiTurn sends sequential probes', async () => {
+    mockFetch
+      .mockResolvedValueOnce(jsonResponse({ response: 'First' }))
+      .mockResolvedValueOnce(jsonResponse({ response: 'Second' }));
+    const adapter = createHttpAdapter('http://localhost:4000');
+    const results = await adapter.sendMultiTurn(['probe1', 'probe2']);
+    expect(results).toHaveLength(2);
+    expect(results[0]!.text).toBe('First');
+    expect(results[1]!.text).toBe('Second');
+  });
+
+  it('has name "http"', () => {
+    const adapter = createHttpAdapter('http://localhost:4000');
+    expect(adapter.name).toBe('http');
+  });
+});
+
+// ── OpenAI Adapter ──────────────────────────────────────────────
+
+describe('createOpenAIAdapter', () => {
+  it('sends to /v1/chat/completions', async () => {
+    mockFetch.mockResolvedValue(jsonResponse({
+      choices: [{ message: { content: 'Hello from GPT' } }],
+    }));
+    const adapter = createOpenAIAdapter('http://localhost:4000', 'gpt-4o', 'sk-test');
+    const result = await adapter.send('Hello');
+
+    expect(result.text).toBe('Hello from GPT');
+    expect(mockFetch.mock.calls[0]![0]).toBe('http://localhost:4000/v1/chat/completions');
+
+    const headers = mockFetch.mock.calls[0]![1]!.headers as Record<string, string>;
+    expect(headers['Authorization']).toBe('Bearer sk-test');
+  });
+
+  it('supports multi-turn with conversation history', async () => {
+    mockFetch
+      .mockResolvedValueOnce(jsonResponse({ choices: [{ message: { content: 'Turn 1' } }] }))
+      .mockResolvedValueOnce(jsonResponse({ choices: [{ message: { content: 'Turn 2' } }] }));
+
+    const adapter = createOpenAIAdapter('http://localhost:4000');
+    const results = await adapter.sendMultiTurn(['first', 'second']);
+    expect(results).toHaveLength(2);
+
+    // Second call should include conversation history
+    const body2 = JSON.parse(mockFetch.mock.calls[1]![1]!.body as string);
+    expect(body2.messages).toHaveLength(3); // user, assistant, user
+  });
+
+  it('has name "openai"', () => {
+    expect(createOpenAIAdapter('http://localhost:4000').name).toBe('openai');
+  });
+});
+
+// ── Anthropic Adapter ───────────────────────────────────────────
+
+describe('createAnthropicAdapter', () => {
+  it('sends to /v1/messages with anthropic headers', async () => {
+    mockFetch.mockResolvedValue(jsonResponse({
+      content: [{ type: 'text', text: 'Hello from Claude' }],
+    }));
+    const adapter = createAnthropicAdapter('http://localhost:4000', 'claude-sonnet-4-20250514', 'sk-ant-test');
+    const result = await adapter.send('Hello');
+
+    expect(result.text).toBe('Hello from Claude');
+    const headers = mockFetch.mock.calls[0]![1]!.headers as Record<string, string>;
+    expect(headers['x-api-key']).toBe('sk-ant-test');
+    expect(headers['anthropic-version']).toBe('2023-06-01');
+  });
+
+  it('has name "anthropic"', () => {
+    expect(createAnthropicAdapter('http://localhost:4000').name).toBe('anthropic');
+  });
+});
+
+// ── Ollama Adapter ──────────────────────────────────────────────
+
+describe('createOllamaAdapter', () => {
+  it('sends to /api/chat with stream:false', async () => {
+    mockFetch.mockResolvedValue(jsonResponse({
+      message: { content: 'Hello from Llama' },
+    }));
+    const adapter = createOllamaAdapter('http://localhost:11434', 'llama3');
+    const result = await adapter.send('Hello');
+
+    expect(result.text).toBe('Hello from Llama');
+    const body = JSON.parse(mockFetch.mock.calls[0]![1]!.body as string);
+    expect(body.stream).toBe(false);
+    expect(body.model).toBe('llama3');
+  });
+
+  it('has name "ollama"', () => {
+    expect(createOllamaAdapter('http://localhost:11434').name).toBe('ollama');
+  });
+});
+
+// ── Custom Adapter ──────────────────────────────────────────────
+
+describe('createCustomAdapter', () => {
+  it('replaces {{probe}} in template', async () => {
+    mockFetch.mockResolvedValue(jsonResponse({ data: { output: 'Custom response' } }));
+    const adapter = createCustomAdapter(
+      'http://localhost:4000/custom',
+      { input: { text: '{{probe}}' } },
+      'data.output',
+    );
+    const result = await adapter.send('Test probe');
+
+    expect(result.text).toBe('Custom response');
+    const body = JSON.parse(mockFetch.mock.calls[0]![1]!.body as string);
+    expect(body.input.text).toBe('Test probe');
+  });
+
+  it('has name "custom"', () => {
+    const adapter = createCustomAdapter('http://localhost:4000', {}, 'response');
+    expect(adapter.name).toBe('custom');
+  });
+});
+
+// ── Auto-detect ─────────────────────────────────────────────────
+
+describe('autoDetectAdapter', () => {
+  it('detects openai:// protocol hint', async () => {
+    const adapter = await autoDetectAdapter('openai://localhost:4000', 'gpt-4o', 'sk-test');
+    expect(adapter.name).toBe('openai');
+  });
+
+  it('detects anthropic:// protocol hint', async () => {
+    const adapter = await autoDetectAdapter('anthropic://localhost:4000');
+    expect(adapter.name).toBe('anthropic');
+  });
+
+  it('detects ollama:// protocol hint', async () => {
+    const adapter = await autoDetectAdapter('ollama://localhost:11434');
+    expect(adapter.name).toBe('ollama');
+  });
+
+  it('probes /v1/models for OpenAI', async () => {
+    mockFetch.mockImplementation(async (url: string) => {
+      if (url.includes('/v1/models')) return { status: 200 };
+      return { status: 404 };
+    });
+    const adapter = await autoDetectAdapter('http://localhost:4000');
+    expect(adapter.name).toBe('openai');
+  });
+
+  it('probes /api/tags for Ollama', async () => {
+    mockFetch.mockImplementation(async (url: string) => {
+      if (url.includes('/v1/models')) throw new Error('not found');
+      if (url.includes('/api/tags')) return { status: 200 };
+      return { status: 404 };
+    });
+    const adapter = await autoDetectAdapter('http://localhost:11434');
+    expect(adapter.name).toBe('ollama');
+  });
+
+  it('falls back to http adapter', async () => {
+    mockFetch.mockRejectedValue(new Error('connection refused'));
+    const adapter = await autoDetectAdapter('http://localhost:9999');
+    expect(adapter.name).toBe('http');
+  });
+});
+
+// ── safeJsonParse ──────────────────────────────────────────────
+
+describe('safeJsonParse', () => {
+  it('parses valid JSON from 200 response', async () => {
+    const res = { ok: true, status: 200, json: async () => ({ data: 'ok' }) } as unknown as Response;
+    const result = await safeJsonParse(res);
+    expect(result).toEqual({ data: 'ok' });
+  });
+
+  it('throws AdapterError for 429 rate limit', async () => {
+    const res = { ok: false, status: 429, json: async () => { throw new Error('not json'); } } as unknown as Response;
+    await expect(safeJsonParse(res)).rejects.toThrow(AdapterError);
+    await expect(safeJsonParse(res)).rejects.toThrow('429');
+  });
+
+  it('throws AdapterError for 500 with JSON error body', async () => {
+    const res = { ok: false, status: 500, json: async () => ({ error: { message: 'overloaded' } }) } as unknown as Response;
+    await expect(safeJsonParse(res)).rejects.toThrow('overloaded');
+  });
+
+  it('AdapterError.retryable is true for 429 and 5xx', () => {
+    expect(new AdapterError(429, 'rate limited').retryable).toBe(true);
+    expect(new AdapterError(500, 'server error').retryable).toBe(true);
+    expect(new AdapterError(502, 'bad gateway').retryable).toBe(true);
+    expect(new AdapterError(400, 'bad request').retryable).toBe(false);
+    expect(new AdapterError(401, 'unauthorized').retryable).toBe(false);
+  });
+});
+
+// ── withRetry ──────────────────────────────────────────────────
+
+describe('withRetry', () => {
+  it('returns immediately on success', async () => {
+    const fn = vi.fn().mockResolvedValue('ok');
+    const result = await withRetry(fn, 2, 1);
+    expect(result).toBe('ok');
+    expect(fn).toHaveBeenCalledTimes(1);
+  });
+
+  it('retries on retryable AdapterError', async () => {
+    const fn = vi.fn()
+      .mockRejectedValueOnce(new AdapterError(429, 'rate limited'))
+      .mockResolvedValue('ok');
+    const result = await withRetry(fn, 2, 1);
+    expect(result).toBe('ok');
+    expect(fn).toHaveBeenCalledTimes(2);
+  });
+
+  it('does not retry on non-retryable AdapterError', async () => {
+    const fn = vi.fn().mockRejectedValue(new AdapterError(400, 'bad request'));
+    await expect(withRetry(fn, 2, 1)).rejects.toThrow('bad request');
+    expect(fn).toHaveBeenCalledTimes(1);
+  });
+
+  it('does not retry on non-AdapterError', async () => {
+    const fn = vi.fn().mockRejectedValue(new Error('network error'));
+    await expect(withRetry(fn, 2, 1)).rejects.toThrow('network error');
+    expect(fn).toHaveBeenCalledTimes(1);
+  });
+
+  it('gives up after max retries', async () => {
+    const fn = vi.fn().mockRejectedValue(new AdapterError(503, 'unavailable'));
+    await expect(withRetry(fn, 2, 1)).rejects.toThrow('unavailable');
+    expect(fn).toHaveBeenCalledTimes(3); // 1 initial + 2 retries
+  });
+});
+
+// ── Adapter error resilience ───────────────────────────────────
+
+describe('adapter error resilience', () => {
+  it('openai adapter throws descriptive error on HTML rate-limit response', async () => {
+    mockFetch.mockResolvedValue({
+      ok: false, status: 429,
+      json: async () => { throw new SyntaxError('Unexpected token <'); },
+      headers: new Map(),
+    });
+    const adapter = createOpenAIAdapter('http://localhost:4000', 'gpt-4o', 'sk-test');
+    await expect(adapter.send('test')).rejects.toThrow('API error 429');
+  });
+
+  it('anthropic adapter throws descriptive error on 500', async () => {
+    mockFetch.mockResolvedValue({
+      ok: false, status: 500,
+      json: async () => ({ error: { message: 'internal error' } }),
+      headers: new Map(),
+    });
+    const adapter = createAnthropicAdapter('http://localhost:4000');
+    await expect(adapter.send('test')).rejects.toThrow('internal error');
+  });
+});

package/src/domain/eval/adapters/anthropic-adapter.ts

@@ -0,0 +1,57 @@
+/**
+ * Anthropic-compatible adapter — sends to `/v1/messages` format.
+ */
+import type { TargetAdapter } from './adapter-port.js';
+import { createChatAdapter } from './create-chat-adapter.js';
+
+const DEFAULT_TIMEOUT = 30_000;
+const DEFAULT_MODEL = 'claude-sonnet-4-20250514';
+
+export const createAnthropicAdapter = (
+  baseUrl: string,
+  model?: string,
+  apiKey?: string,
+): TargetAdapter => {
+  const endpoint = baseUrl.replace(/\/$/, '') + '/v1/messages';
+  const effectiveModel = model ?? DEFAULT_MODEL;
+
+  const buildHeaders = (): Record<string, string> => {
+    const h: Record<string, string> = {
+      'Content-Type': 'application/json',
+      'anthropic-version': '2023-06-01',
+    };
+    if (apiKey) h['x-api-key'] = apiKey;
+    return h;
+  };
+
+  return createChatAdapter({
+    name: 'anthropic',
+    endpoint,
+    defaultTimeout: DEFAULT_TIMEOUT,
+    buildHeaders,
+    // Anthropic: system prompt goes in body, not in messages
+    buildInitialMessages: (probe) => [{ role: 'user', content: probe }],
+    buildMultiTurnPrefix: () => [],
+    buildBody: (messages, options) => {
+      const body: Record<string, unknown> = {
+        model: effectiveModel,
+        max_tokens: options?.maxTokens ?? 2048,
+        messages,
+      };
+      if (options?.systemPrompt) body.system = options.systemPrompt;
+      if (options?.temperature !== undefined) body.temperature = options.temperature;
+      return body;
+    },
+    extractText: (raw) => {
+      const content = raw.content as { type?: string; text?: string }[] | undefined;
+      return content?.find((c) => c.type === 'text')?.text ?? '';
+    },
+    healthCheck: {
+      url: endpoint,
+      method: 'POST',
+      headers: buildHeaders,
+      body: { model: effectiveModel, max_tokens: 1, messages: [{ role: 'user', content: 'ping' }] },
+      isHealthy: (status) => status < 500,
+    },
+  });
+};

package/src/domain/eval/adapters/auto-detect.ts

@@ -0,0 +1,104 @@
+/**
+ * Auto-detect adapter — probes the target URL to determine which adapter to use.
+ *
+ * Detection order:
+ *   1. Custom adapter (if requestTemplate + responsePath provided)
+ *   2. URL protocol hint (openai://, anthropic://, ollama://)
+ *   3. /v1/models → OpenAI-compatible
+ *   4. /api/tags → Ollama
+ *   5. Fallback → Generic HTTP
+ */
+
+import type { TargetAdapter } from './adapter-port.js';
+import { withTimeout } from './with-timeout.js';
+import { createHttpAdapter } from './http-adapter.js';
+import { createOpenAIAdapter } from './openai-adapter.js';
+import { createAnthropicAdapter } from './anthropic-adapter.js';
+import { createOllamaAdapter } from './ollama-adapter.js';
+import { createCustomAdapter } from './custom-adapter.js';
+
+export interface AutoDetectOptions {
+  readonly model?: string;
+  readonly apiKey?: string;
+  readonly requestTemplate?: string;
+  readonly responsePath?: string;
+  readonly headers?: string;
+}
+
+/** Parse protocol-hinted URLs like openai://localhost:4000 → http://localhost:4000 */
+const parseProtocolHint = (url: string): { protocol: string; httpUrl: string } | null => {
+  const match = url.match(/^(openai|anthropic|ollama):\/\/(.+)/);
+  if (!match) return null;
+  const protocol = match[1]!;
+  const rest = match[2]!;
+  const httpUrl = rest.startsWith('http') ? rest : `http://${rest}`;
+  return { protocol, httpUrl };
+};
+
+const tryFetch = async (url: string, timeout = 3000): Promise<boolean> => {
+  try {
+    return await withTimeout(async (signal) => {
+      const res = await fetch(url, { signal });
+      return res.status === 200;
+    }, timeout);
+  } catch {
+    return false;
+  }
+};
+
+export const autoDetectAdapter = async (
+  url: string,
+  modelOrOpts?: string | AutoDetectOptions,
+  apiKey?: string,
+): Promise<TargetAdapter> => {
+  // Normalize overloaded params
+  const opts: AutoDetectOptions = typeof modelOrOpts === 'string'
+    ? { model: modelOrOpts, apiKey }
+    : modelOrOpts ?? {};
+  const model = opts.model;
+  const key = opts.apiKey ?? apiKey;
+
+  // 0. Custom adapter (explicit template)
+  if (opts.requestTemplate && opts.responsePath) {
+    let template: Record<string, unknown>;
+    try {
+      template = JSON.parse(opts.requestTemplate);
+    } catch {
+      throw new Error(`Invalid --request-template JSON: ${opts.requestTemplate}`);
+    }
+    let customHeaders: Record<string, string> | undefined;
+    if (opts.headers) {
+      try {
+        customHeaders = JSON.parse(opts.headers);
+      } catch {
+        throw new Error(`Invalid --headers JSON: ${opts.headers}`);
+      }
+    }
+    return createCustomAdapter(url, template, opts.responsePath, customHeaders);
+  }
+
+  // 1. Check protocol hints
+  const hint = parseProtocolHint(url);
+  if (hint) {
+    switch (hint.protocol) {
+      case 'openai':    return createOpenAIAdapter(hint.httpUrl, model, key);
+      case 'anthropic': return createAnthropicAdapter(hint.httpUrl, model, key);
+      case 'ollama':    return createOllamaAdapter(hint.httpUrl, model);
+    }
+  }
+
+  const baseUrl = url.replace(/\/$/, '');
+
+  // 2. Probe /v1/models (OpenAI-compatible)
+  if (await tryFetch(`${baseUrl}/v1/models`)) {
+    return createOpenAIAdapter(baseUrl, model, key);
+  }
+
+  // 3. Probe /api/tags (Ollama)
+  if (await tryFetch(`${baseUrl}/api/tags`)) {
+    return createOllamaAdapter(baseUrl, model);
+  }
+
+  // 4. Fallback to generic HTTP
+  return createHttpAdapter(url);
+};

package/src/domain/eval/adapters/create-chat-adapter.ts

@@ -0,0 +1,106 @@
+/**
+ * Factory for chat-based eval adapters (OpenAI, Anthropic, Ollama).
+ * Eliminates duplicated send/sendMultiTurn/checkHealth boilerplate.
+ * Each adapter provides only: body builder, response parser, health config.
+ */
+import type { TargetAdapter, TargetResponse, ProbeOptions } from './adapter-port.js';
+import { safeJsonParse, withRetry } from './adapter-port.js';
+import { withTimeout, extractResponseHeaders } from './with-timeout.js';
+
+interface ChatMessage {
+  readonly role: string;
+  readonly content: string;
+}
+
+interface HealthCheckConfig {
+  readonly url: string;
+  readonly method: 'GET' | 'POST';
+  readonly headers?: () => Record<string, string>;
+  readonly body?: unknown;
+  readonly isHealthy: (status: number) => boolean;
+}
+
+export interface ChatAdapterConfig {
+  readonly name: string;
+  readonly endpoint: string;
+  readonly defaultTimeout: number;
+  readonly buildHeaders: () => Record<string, string>;
+  /** Build request body from conversation messages + options. */
+  readonly buildBody: (messages: readonly ChatMessage[], options?: ProbeOptions) => unknown;
+  /** Extract response text from parsed JSON. */
+  readonly extractText: (raw: Record<string, unknown>) => string;
+  /** Build initial message list for single send (handles system prompt placement). */
+  readonly buildInitialMessages: (probe: string, options?: ProbeOptions) => ChatMessage[];
+  /** Build initial message list for multi-turn (conversation prefix). */
+  readonly buildMultiTurnPrefix: (options?: ProbeOptions) => ChatMessage[];
+  readonly healthCheck: HealthCheckConfig;
+}
+
+export const createChatAdapter = (config: ChatAdapterConfig): TargetAdapter => {
+  const { endpoint, defaultTimeout, buildHeaders, buildBody, extractText, buildInitialMessages, buildMultiTurnPrefix, healthCheck } = config;
+
+  const send = async (probe: string, options?: ProbeOptions): Promise<TargetResponse> => {
+    const start = Date.now();
+    const timeout = options?.timeout ?? defaultTimeout;
+    const messages = buildInitialMessages(probe, options);
+
+    return withRetry(async () => withTimeout(async (signal) => {
+      const res = await fetch(endpoint, {
+        method: 'POST',
+        headers: buildHeaders(),
+        body: JSON.stringify(buildBody(messages, options)),
+        signal,
+      });
+      const latencyMs = Date.now() - start;
+      const raw = await safeJsonParse(res);
+      const text = extractText(raw);
+      return { text, status: res.status, headers: extractResponseHeaders(res), latencyMs, raw };
+    }, timeout));
+  };
+
+  const sendMultiTurn = async (probes: readonly string[], options?: ProbeOptions): Promise<readonly TargetResponse[]> => {
+    const results: TargetResponse[] = [];
+    const messages: ChatMessage[] = buildMultiTurnPrefix(options);
+
+    for (const probe of probes) {
+      messages.push({ role: 'user', content: probe });
+      const start = Date.now();
+      const timeout = options?.timeout ?? defaultTimeout;
+
+      const result = await withRetry(async () => withTimeout(async (signal) => {
+        const res = await fetch(endpoint, {
+          method: 'POST',
+          headers: buildHeaders(),
+          body: JSON.stringify(buildBody([...messages], options)),
+          signal,
+        });
+        const latencyMs = Date.now() - start;
+        const raw = await safeJsonParse(res);
+        const text = extractText(raw);
+        return { text, status: res.status, headers: extractResponseHeaders(res), latencyMs, raw };
+      }, timeout));
+      messages.push({ role: 'assistant', content: result.text });
+      results.push(result);
+    }
+    return results;
+  };
+
+  const checkHealth = async (): Promise<boolean> => {
+    try {
+      return await withTimeout(async (signal) => {
+        const fetchOpts: RequestInit = { method: healthCheck.method, signal };
+        if (healthCheck.headers) fetchOpts.headers = healthCheck.headers();
+        if (healthCheck.body) {
+          fetchOpts.body = JSON.stringify(healthCheck.body);
+          fetchOpts.headers = { ...fetchOpts.headers as Record<string, string>, 'Content-Type': 'application/json' };
+        }
+        const res = await fetch(healthCheck.url, fetchOpts);
+        return healthCheck.isHealthy(res.status);
+      }, 5000);
+    } catch {
+      return false;
+    }
+  };
+
+  return Object.freeze({ send, sendMultiTurn, checkHealth, name: config.name });
+};

package/src/domain/eval/adapters/custom-adapter.ts

@@ -0,0 +1,74 @@
+/**
+ * Custom template-based adapter — sends probes via configurable JSON template
+ * and extracts responses via a dot-path.
+ */
+
+import type { TargetAdapter, TargetResponse, ProbeOptions } from './adapter-port.js';
+import { safeJsonParse, withRetry } from './adapter-port.js';
+import { withTimeout, extractResponseHeaders } from './with-timeout.js';
+
+const DEFAULT_TIMEOUT = 30_000;
+
+/** Resolve a dot-separated path on an object (e.g. "data.choices.0.text"). */
+const resolvePath = (obj: unknown, path: string): unknown => {
+  let current = obj;
+  for (const key of path.split('.')) {
+    if (current == null || typeof current !== 'object') return undefined;
+    current = (current as Record<string, unknown>)[key];
+  }
+  return current;
+};
+
+export const createCustomAdapter = (
+  url: string,
+  template: Record<string, unknown>,
+  responsePath: string,
+  headers?: Record<string, string>,
+): TargetAdapter => {
+
+  const send = async (probe: string, options?: ProbeOptions): Promise<TargetResponse> => {
+    const start = Date.now();
+    const timeout = options?.timeout ?? DEFAULT_TIMEOUT;
+
+    // Replace {{probe}} placeholder in template
+    const body = JSON.parse(
+      JSON.stringify(template).replace(/\{\{probe\}\}/g, probe.replace(/"/g, '\\"')),
+    );
+
+    return withRetry(async () => withTimeout(async (signal) => {
+      const res = await fetch(url, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json', ...headers },
+        body: JSON.stringify(body),
+        signal,
+      });
+      const latencyMs = Date.now() - start;
+      const raw = await safeJsonParse(res);
+      const resolved = resolvePath(raw, responsePath);
+      const text = typeof resolved === 'string' ? resolved : JSON.stringify(resolved ?? '');
+
+      return { text, status: res.status, headers: extractResponseHeaders(res), latencyMs, raw };
+    }, timeout));
+  };
+
+  const sendMultiTurn = async (probes: readonly string[], options?: ProbeOptions): Promise<readonly TargetResponse[]> => {
+    const results: TargetResponse[] = [];
+    for (const probe of probes) {
+      results.push(await send(probe, options));
+    }
+    return results;
+  };
+
+  const checkHealth = async (): Promise<boolean> => {
+    try {
+      return await withTimeout(async (signal) => {
+        const res = await fetch(url, { method: 'GET', signal });
+        return res.status < 500;
+      }, 5000);
+    } catch {
+      return false;
+    }
+  };
+
+  return Object.freeze({ send, sendMultiTurn, checkHealth, name: 'custom' });
+};