impel-cli 0.7.0

package/src/doctor.js

@@ -0,0 +1,546 @@
+import { randomUUID } from "node:crypto";
+
+import { normalizeGatewayUrl, redactSecretText, resolveDefaultGateway } from "./config.js";
+import { tenantCredential } from "./tenants.js";
+
+export const DOCTOR_PROVIDERS = ["claude", "codex"];
+export const DEFAULT_TTFT_BUDGET_MS = Object.freeze({ claude: 6_000, codex: 5_000 });
+
+const MAX_ERROR_BODY_BYTES = 64 * 1024;
+const MAX_OUTPUT_CHARS = 16 * 1024;
+const MAX_PENDING_SSE_CHARS = 1024 * 1024;
+const MAX_STREAM_BYTES = 4 * 1024 * 1024;
+const GATEWAY_REQUEST_ID_RE = /^impel-[a-f0-9]{32}$/u;
+const POOL_STATES = new Set(["healthy", "exhausted", "rate_limited", "expired", "no_seat"]);
+
+function round(value) {
+  return Number.isFinite(value) ? Math.round(value) : null;
+}
+
+function percentile(values, proportion) {
+  const sorted = values.filter(Number.isFinite).sort((a, b) => a - b);
+  if (!sorted.length) return null;
+  return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * proportion) - 1)];
+}
+
+function safeErrorMessage(payload, fallback) {
+  const candidate = payload?.error?.message || payload?.error || payload?.message || fallback;
+  const text = typeof candidate === "string" ? candidate : JSON.stringify(candidate);
+  return redactSecretText(String(text || fallback).replace(/\s+/gu, " ")).slice(0, 300);
+}
+
+async function readTextLimited(response, limit = MAX_ERROR_BODY_BYTES) {
+  if (!response.body) return "";
+  const reader = response.body.getReader();
+  const decoder = new TextDecoder();
+  let text = "";
+  let bytes = 0;
+  while (bytes < limit) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    bytes += value.byteLength;
+    text += decoder.decode(value, { stream: true });
+    if (bytes >= limit) {
+      await reader.cancel().catch(() => {});
+      break;
+    }
+  }
+  text += decoder.decode();
+  return text.slice(0, limit);
+}
+
+async function readError(response) {
+  const text = await readTextLimited(response);
+  let payload = null;
+  try {
+    payload = text ? JSON.parse(text) : null;
+  } catch {
+    // Do not echo arbitrary upstream response bodies into terminal output.
+  }
+  return safeErrorMessage(payload, `${response.status} ${response.statusText || "gateway error"}`.trim());
+}
+
+function consumeSseBlock(provider, block, state, startedAt, now) {
+  const data = block
+    .split(/\r\n|\r|\n/gu)
+    .filter((line) => line.startsWith("data:"))
+    .map((line) => line.slice(5).trimStart())
+    .join("\n")
+    .trim();
+  if (!data) return;
+  if (data === "[DONE]") {
+    state.doneSentinel = true;
+    return;
+  }
+
+  let event;
+  try {
+    event = JSON.parse(data);
+  } catch {
+    state.error ||= "gateway returned malformed JSON in the SSE stream";
+    return;
+  }
+
+  let delta = "";
+  if (provider === "claude") {
+    if (!state.responseId && typeof event?.message?.id === "string") {
+      state.responseId = event.message.id.slice(0, 512) || null;
+    }
+    if (event?.type === "message_stop") state.terminalEvent = event.type;
+    if (event?.type === "content_block_delta" && event?.delta?.type === "text_delta") {
+      delta = typeof event.delta.text === "string" ? event.delta.text : "";
+    }
+  } else {
+    if (!state.responseId && typeof event?.response?.id === "string") {
+      state.responseId = event.response.id.slice(0, 512) || null;
+    }
+    if (event?.type === "response.completed") {
+      if (event?.response?.status && event.response.status !== "completed") {
+        state.error ||= `provider completed the response with status ${redactSecretText(event.response.status)}`;
+      } else {
+        state.terminalEvent = event.type;
+      }
+    }
+    if (["response.failed", "response.incomplete", "response.cancelled"].includes(event?.type)) {
+      state.error ||= safeErrorMessage(event?.response || event, `provider emitted ${event.type}`);
+    }
+    if (event?.type === "response.output_text.delta") {
+      delta = typeof event.delta === "string" ? event.delta : "";
+    }
+  }
+  if (event?.type === "error") {
+    state.error ||= safeErrorMessage(event, "provider emitted an SSE error event");
+  }
+
+  if (!delta) return;
+  state.ttftMs ??= now() - startedAt;
+  if (state.answer.length < MAX_OUTPUT_CHARS) {
+    const remaining = MAX_OUTPUT_CHARS - state.answer.length;
+    state.answer += delta.slice(0, remaining);
+    if (delta.length > remaining) state.outputTruncated = true;
+  } else {
+    state.outputTruncated = true;
+  }
+}
+
+async function streamProbe({ provider, url, bearer, model, tenantId, attempt, timeoutMs, fetchImpl, uuid, now }) {
+  // Keep customer/org names out of the provider-visible synthetic prompt.
+  const clientRequestId = `impel-doctor-${provider}-${attempt}-${uuid()}`;
+  const expected = `ACK ${clientRequestId}`;
+  const prompt = `Synthetic Impel production routing check ${clientRequestId}. Reply with exactly: ${expected}`;
+  const body = provider === "claude"
+    ? {
+        model,
+        max_tokens: 96,
+        stream: true,
+        messages: [{ role: "user", content: prompt }],
+      }
+    : {
+        model,
+        input: [{ role: "user", content: [{ type: "input_text", text: prompt }] }],
+        store: false,
+        stream: true,
+      };
+
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), timeoutMs);
+  const startedAt = now();
+  let responseMetadata = {
+    requestId: null,
+    echoedClientRequestId: null,
+    accountId: null,
+    orgId: null,
+    status: null,
+    headersMs: null,
+    contentType: "",
+  };
+  try {
+    const response = await fetchImpl(url, {
+      method: "POST",
+      headers: {
+        accept: "text/event-stream",
+        authorization: `Bearer ${bearer}`,
+        "content-type": "application/json",
+        "x-request-id": clientRequestId,
+        ...(provider === "claude" ? { "anthropic-version": "2023-06-01" } : {}),
+      },
+      body: JSON.stringify(body),
+      signal: controller.signal,
+    });
+    const headersMs = now() - startedAt;
+    responseMetadata = {
+      requestId: response.headers.get("x-impel-request-id") || null,
+      echoedClientRequestId: response.headers.get("x-impel-client-request-id") || null,
+      accountId: response.headers.get("x-impel-account-id") || null,
+      orgId: response.headers.get("x-impel-org-id") || null,
+      status: response.status,
+      headersMs: round(headersMs),
+      contentType: response.headers.get("content-type") || "",
+    };
+    if (!response.ok || !response.body) {
+      return {
+        tenantId,
+        provider,
+        attempt,
+        model,
+        ...responseMetadata,
+        clientRequestId,
+        responseId: null,
+        firstByteMs: round(headersMs),
+        ttftMs: null,
+        totalMs: round(now() - startedAt),
+        exactAck: false,
+        outputChars: 0,
+        outputTruncated: false,
+        terminalEvent: null,
+        doneSentinel: false,
+        receivedBytes: 0,
+        error: await readError(response),
+      };
+    }
+
+    const state = {
+      answer: "",
+      responseId: null,
+      ttftMs: null,
+      outputTruncated: false,
+      terminalEvent: null,
+      doneSentinel: false,
+      error: null,
+    };
+    const reader = response.body.getReader();
+    const decoder = new TextDecoder();
+    let pending = "";
+    let firstByteMs = null;
+    let receivedBytes = 0;
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      firstByteMs ??= now() - startedAt;
+      receivedBytes += value.byteLength;
+      if (receivedBytes > MAX_STREAM_BYTES) {
+        await reader.cancel().catch(() => {});
+        throw new Error("gateway response exceeded the production probe stream limit");
+      }
+      pending += decoder.decode(value, { stream: true });
+      if (pending.length > MAX_PENDING_SSE_CHARS) {
+        await reader.cancel().catch(() => {});
+        throw new Error("gateway returned an invalid oversized SSE event");
+      }
+      const blocks = pending.split(/(?:(?:\r\n|\r|\n)){2}/gu);
+      pending = blocks.pop() || "";
+      for (const block of blocks) consumeSseBlock(provider, block, state, startedAt, now);
+    }
+    pending += decoder.decode();
+    if (pending) consumeSseBlock(provider, pending, state, startedAt, now);
+
+    const exactAck = state.answer.trim() === expected;
+    const validationError = state.error
+      || (!state.responseId ? "provider stream did not include a response id" : null)
+      || (!state.terminalEvent ? "provider stream ended without a successful terminal event" : null)
+      || (!GATEWAY_REQUEST_ID_RE.test(responseMetadata.requestId || "")
+        ? "response did not include a valid gateway-owned request id"
+        : null)
+      || (responseMetadata.requestId === clientRequestId
+        ? "gateway reused the caller correlation id as its request id"
+        : null)
+      || (responseMetadata.echoedClientRequestId !== clientRequestId
+        ? "response did not echo the caller correlation id"
+        : null);
+
+    return {
+      tenantId,
+      provider,
+      attempt,
+      model,
+      ...responseMetadata,
+      clientRequestId,
+      responseId: state.responseId,
+      firstByteMs: round(firstByteMs ?? headersMs),
+      ttftMs: round(state.ttftMs),
+      totalMs: round(now() - startedAt),
+      exactAck,
+      outputChars: state.answer.length,
+      outputTruncated: state.outputTruncated,
+      terminalEvent: state.terminalEvent,
+      doneSentinel: state.doneSentinel,
+      receivedBytes,
+      error: validationError,
+    };
+  } catch (error) {
+    const timedOut = error?.name === "AbortError";
+    return {
+      tenantId,
+      provider,
+      attempt,
+      model,
+      ...responseMetadata,
+      clientRequestId,
+      responseId: null,
+      firstByteMs: null,
+      ttftMs: null,
+      totalMs: round(now() - startedAt),
+      exactAck: false,
+      outputChars: 0,
+      outputTruncated: false,
+      terminalEvent: null,
+      doneSentinel: false,
+      receivedBytes: 0,
+      error: timedOut
+        ? `timed out after ${timeoutMs}ms`
+        : redactSecretText(error?.message || error).slice(0, 300),
+    };
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+function modelReady(model) {
+  return model?.available !== false && model?.ready !== false && model?.readiness?.ready !== false;
+}
+
+function providerCatalogStatuses(payload) {
+  if (!payload || typeof payload !== "object" || !Object.hasOwn(payload, "provider_status")) {
+    return { statuses: {}, malformed: false };
+  }
+  const rawStatuses = payload.provider_status;
+  if (!rawStatuses || typeof rawStatuses !== "object" || Array.isArray(rawStatuses)) {
+    return { statuses: {}, malformed: true };
+  }
+  const statuses = {};
+  for (const provider of DOCTOR_PROVIDERS) {
+    if (!Object.hasOwn(rawStatuses, provider)) continue;
+    const raw = rawStatuses[provider];
+    if (!raw
+        || typeof raw !== "object"
+        || Array.isArray(raw)
+        || !POOL_STATES.has(raw.state)
+        || typeof raw.routable !== "boolean") {
+      return { statuses: {}, malformed: true };
+    }
+    statuses[provider] = { state: raw.state, routable: raw.routable };
+  }
+  return { statuses, malformed: false };
+}
+
+export function recordSucceeded(record) {
+  return record.status === 200
+    && !record.error
+    && record.exactAck
+    && Boolean(record.responseId)
+    && Boolean(record.terminalEvent)
+    && GATEWAY_REQUEST_ID_RE.test(record.requestId || "")
+    && record.requestId !== record.clientRequestId
+    && record.echoedClientRequestId === record.clientRequestId
+    && Boolean(record.accountId)
+    && record.orgId === record.tenantId
+    && record.contentType.toLowerCase().includes("text/event-stream");
+}
+
+function selectModel(models, provider) {
+  const candidates = models.filter((model) => model?.provider === provider && modelReady(model));
+  return candidates.find((model) => model.default)?.id || candidates[0]?.id || null;
+}
+
+function summarizeProvider(records, provider, ttftBudgetMs, catalogStatus, skipped) {
+  const selected = records.filter((record) => record.provider === provider);
+  const successful = selected.filter(recordSucceeded);
+  const withinBudget = successful.filter((record) => record.ttftMs !== null && record.ttftMs <= ttftBudgetMs);
+  return {
+    catalogState: catalogStatus?.state || null,
+    routable: catalogStatus?.routable ?? null,
+    skipped,
+    attempted: selected.length,
+    successful: successful.length,
+    withinLatencyBudget: withinBudget.length,
+    ttftBudgetMs,
+    ttftP50Ms: percentile(successful.map((record) => record.ttftMs), 0.5),
+    ttftP95Ms: percentile(successful.map((record) => record.ttftMs), 0.95),
+    totalP50Ms: percentile(successful.map((record) => record.totalMs), 0.5),
+    totalP95Ms: percentile(successful.map((record) => record.totalMs), 0.95),
+  };
+}
+
+export async function probeTenant({
+  config,
+  tenantId,
+  productAccess,
+  providers = DOCTOR_PROVIDERS,
+  attempts = 1,
+  timeoutMs = 120_000,
+  ttftBudgets = DEFAULT_TTFT_BUDGET_MS,
+  strictLatency = false,
+  fetchImpl = fetch,
+  uuid = randomUUID,
+  now = () => performance.now(),
+}) {
+  const gatewayUrl = normalizeGatewayUrl(config.gatewayUrl || resolveDefaultGateway());
+  const bearer = tenantCredential(config.pat, tenantId);
+  const catalogStartedAt = now();
+  const catalogController = new AbortController();
+  const catalogTimeoutMs = Math.min(timeoutMs, 30_000);
+  const catalogTimeout = setTimeout(() => catalogController.abort(), catalogTimeoutMs);
+  let response;
+  let catalogText;
+  try {
+    response = await fetchImpl(new URL("/v1/models", gatewayUrl), {
+      headers: { accept: "application/json", authorization: `Bearer ${bearer}` },
+      signal: catalogController.signal,
+    });
+    catalogText = await readTextLimited(response);
+  } catch (error) {
+    const timedOut = error?.name === "AbortError";
+    return {
+      tenantId,
+      productAccess,
+      catalog: {
+        status: null,
+        durationMs: round(now() - catalogStartedAt),
+        models: 0,
+        error: timedOut
+          ? `model catalog timed out after ${catalogTimeoutMs}ms`
+          : redactSecretText(error?.message || error).slice(0, 300),
+      },
+      records: [],
+      providers: {},
+      passed: false,
+    };
+  } finally {
+    clearTimeout(catalogTimeout);
+  }
+
+  let catalogPayload = null;
+  try {
+    catalogPayload = catalogText ? JSON.parse(catalogText) : null;
+  } catch {
+    // Report a typed catalog error below.
+  }
+  const models = Array.isArray(catalogPayload?.data) ? catalogPayload.data : [];
+  const catalogOrgId = typeof catalogPayload?.org_id === "string" ? catalogPayload.org_id : null;
+  const catalogProductAccess = typeof catalogPayload?.product_access === "string"
+    ? catalogPayload.product_access
+    : null;
+  const catalogScopeMatches = catalogOrgId === tenantId && catalogProductAccess === productAccess;
+  const providerStatusResult = providerCatalogStatuses(catalogPayload);
+  const providerStatus = providerStatusResult.statuses;
+  const catalog = {
+    status: response.status,
+    durationMs: round(now() - catalogStartedAt),
+    models: models.length,
+    orgId: catalogOrgId,
+    productAccess: catalogProductAccess,
+    providerStatus,
+    error: response.ok && catalogPayload
+      ? null
+      : response.ok
+        ? "gateway returned an invalid model catalog"
+        : safeErrorMessage(catalogPayload, `${response.status} ${response.statusText || "gateway error"}`.trim()),
+  };
+  if (!response.ok || !catalogPayload) {
+    return { tenantId, productAccess, catalog, records: [], providers: {}, passed: false };
+  }
+  if (!catalogScopeMatches) {
+    catalog.error = "model catalog tenant or product access did not match the live selection";
+    return { tenantId, productAccess, catalog, records: [], providers: {}, passed: false };
+  }
+  if (providerStatusResult.malformed) {
+    catalog.error = "model catalog returned malformed provider status metadata";
+    return { tenantId, productAccess, catalog, records: [], providers: {}, passed: false };
+  }
+  const contradictoryProvider = providers.find((provider) => {
+    const status = providerStatus[provider];
+    return Boolean(selectModel(models, provider))
+      && Boolean(status)
+      && (status.state !== "healthy" || status.routable !== true);
+  });
+  if (contradictoryProvider) {
+    catalog.error = `model catalog advertised a ready ${contradictoryProvider} model while its provider status was not healthy and routable`;
+    return { tenantId, productAccess, catalog, records: [], providers: {}, passed: false };
+  }
+
+  const records = [];
+  const skippedProviders = new Set();
+  for (const provider of providers) {
+    const model = selectModel(models, provider);
+    if (!model) {
+      const status = providerStatus[provider];
+      if (status?.state === "no_seat" && status.routable === false) {
+        skippedProviders.add(provider);
+        continue;
+      }
+      for (let attempt = 1; attempt <= attempts; attempt += 1) {
+        records.push({
+          tenantId,
+          provider,
+          attempt,
+          model: null,
+          requestId: null,
+          clientRequestId: null,
+          echoedClientRequestId: null,
+          responseId: null,
+          accountId: null,
+          orgId: null,
+          status: null,
+          headersMs: null,
+          firstByteMs: null,
+          ttftMs: null,
+          totalMs: null,
+          exactAck: false,
+          outputChars: 0,
+          outputTruncated: false,
+          terminalEvent: null,
+          doneSentinel: false,
+          receivedBytes: 0,
+          contentType: "",
+          error: "no ready model advertised for this tenant and provider",
+        });
+      }
+      continue;
+    }
+    const url = new URL(provider === "claude" ? "/anthropic/v1/messages" : "/v1/responses", gatewayUrl);
+    for (let attempt = 1; attempt <= attempts; attempt += 1) {
+      records.push(await streamProbe({
+        provider,
+        url,
+        bearer,
+        model,
+        tenantId,
+        attempt,
+        timeoutMs,
+        fetchImpl,
+        uuid,
+        now,
+      }));
+    }
+  }
+
+  const providerSummary = Object.fromEntries(
+    providers.map((provider) => [
+      provider,
+      summarizeProvider(
+        records,
+        provider,
+        ttftBudgets[provider],
+        providerStatus[provider],
+        skippedProviders.has(provider),
+      ),
+    ]),
+  );
+  const probedProviders = providers.filter((provider) => !skippedProviders.has(provider));
+  const requestChecksPassed = probedProviders.length > 0
+    && records.length === probedProviders.length * attempts
+    && records.every(recordSucceeded);
+  const latencyPassed = probedProviders.length > 0 && probedProviders.every((provider) => {
+    const summary = providerSummary[provider];
+    return summary.attempted > 0 && summary.withinLatencyBudget === summary.attempted;
+  });
+  return {
+    tenantId,
+    productAccess,
+    catalog,
+    records,
+    providers: providerSummary,
+    latencyPassed,
+    passed: Boolean(productAccess) && requestChecksPassed && (!strictLatency || latencyPassed),
+  };
+}