@cortad/connector 0.6.0-rc.1 → 0.6.0-rc.2

package/lib/history/braintrust.mjs

@@ -0,0 +1,83 @@
+import { said } from "./langfuse.mjs";
+import { answered, grouped, msOf, rootOf, threadIn, userIn } from "./vendor.mjs";
+
+// The app's recent conversations as Braintrust logged them, read with the key the app already has,
+// sent to Braintrust's own API and nowhere else. The project comes from GET /v1/project, its logs
+// from POST /v1/project_logs/{project_id}/fetch with the cursor each page hands back (Braintrust API
+// reference, "Fetch project logs"). A trace's root span is one reply.
+const PAGE = 100;
+const MAX_TRACES = 3_000;
+
+export const BRAINTRUST = {
+  id: "braintrust",
+  name: "Braintrust",
+  needs: ["BRAINTRUST_API_KEY"],
+  hostOf: (env) => env.BRAINTRUST_API_URL || "https://api.braintrust.dev",
+  read: readBraintrust,
+};
+
+// The project to read: BRAINTRUST_PARENT as Braintrust's OpenTelemetry exporter takes it, else
+// BRAINTRUST_PROJECT. An app that names its project in code has neither.
+export function projectIn(env) {
+  const [kind, ...rest] = (env.BRAINTRUST_PARENT ?? "").split(":");
+  const value = rest.join(":");
+  if (kind === "project_id" && value) return { id: value };
+  if (kind === "project_name" && value) return { name: value };
+  return env.BRAINTRUST_PROJECT ? { name: env.BRAINTRUST_PROJECT } : {};
+}
+
+const isRoot = (event) => event.is_root ?? !event.span_parents?.length;
+
+// Braintrust's OpenAI wrapper logs a reply as the completion's choices.
+const replyOf = (output) => (Array.isArray(output) ? output.map((choice) => choice?.message ?? choice) : output);
+
+export async function readBraintrust({ env, since, fetchImpl = fetch }) {
+  const root = rootOf(BRAINTRUST.hostOf(env));
+  const host = new URL(root).host;
+  const headers = { authorization: `Bearer ${env.BRAINTRUST_API_KEY}`, "content-type": "application/json" };
+  const ask = (path, init = {}) => answered(fetchImpl, `${root}${path}`, { ...init, headers }, "BRAINTRUST");
+  const id = await projectId(ask, env, host);
+
+  const roots = [];
+  const seen = new Set();
+  const from = since.getTime();
+  for (let cursor = null; roots.length < MAX_TRACES;) {
+    const body = JSON.stringify({ limit: PAGE, ...(cursor ? { cursor } : {}) });
+    const page = await ask(`/v1/project_logs/${encodeURIComponent(id)}/fetch`, { method: "POST", body });
+    const events = page.events ?? [];
+    // A later page can give a row again at an older version; the first seen is the newest.
+    for (const event of events) {
+      if (seen.has(event.id)) continue;
+      seen.add(event.id);
+      if (isRoot(event) && msOf(event.created) >= from) roots.push(event);
+    }
+    cursor = page.cursor;
+    // ponytail: pages run newest change first, so a page of only older rows ends the read and a bulk
+    // edit of old logs would end it early. POST /btql with a created filter is exact if that matters.
+    if (!cursor || events.every((event) => msOf(event.created) < from)) break;
+  }
+  const { conversations, withoutText } = grouped(roots.map((event) => ({
+    conversation: threadIn(event.metadata) || event.root_span_id,
+    user: userIn(event.metadata),
+    at: event.created,
+    question: said(event.input, "user"),
+    answer: said(replyOf(event.output), "assistant"),
+  })));
+  return { conversations, host, read: roots.length, withoutText };
+}
+
+async function projectId(ask, env, host) {
+  const wanted = projectIn(env);
+  if (wanted.id) return wanted.id;
+  const org = env.BRAINTRUST_ORG_NAME ? `&org_name=${encodeURIComponent(env.BRAINTRUST_ORG_NAME)}` : "";
+  if (wanted.name) {
+    const { objects } = await ask(`/v1/project?project_name=${encodeURIComponent(wanted.name)}${org}`);
+    const project = objects?.find((p) => p.name === wanted.name);
+    if (!project) throw new Error(`${host} has no project named ${wanted.name}`);
+    return project.id;
+  }
+  // With no project named, the one project the key sees is the app's.
+  const { objects = [] } = await ask(`/v1/project?limit=2${org}`);
+  if (objects.length === 1) return objects[0].id;
+  throw new Error(objects.length ? "the key in your env sees more than one Braintrust project; set BRAINTRUST_PROJECT to the one to read" : `${host} has no project for the key in your env`);
+}

package/lib/history/helicone.mjs

@@ -0,0 +1,59 @@
+import { said } from "./langfuse.mjs";
+import { addressOf, answered, grouped, rootOf } from "./vendor.mjs";
+
+// The app's recent conversations as Helicone logged its model calls, read with the key the app already
+// has, sent to Helicone's own API and nowhere else. Requests come from POST
+// /v1/request/query-clickhouse (Helicone API reference, "Get Requests"), newest first by offset. A
+// body Helicone keeps in its store is fetched from the request's signed_body_url, with no key.
+const PAGE = 100;
+const MAX_REQUESTS = 3_000;
+const NOT_A_CHAT = /\/(?:embeddings|moderations)(?:$|[/?])/;
+
+export const HELICONE = {
+  id: "helicone",
+  name: "Helicone",
+  needs: ["HELICONE_API_KEY"],
+  // Helicone's own SDKs send a key made in the EU to the EU API.
+  hostOf: (env) => (env.HELICONE_API_KEY?.startsWith("sk-helicone-eu-") ? "https://eu.api.helicone.ai" : "https://api.helicone.ai"),
+  read: readHelicone,
+};
+
+// A body as Helicone answers it inline: parsed JSON, or nothing when it says the body is kept elsewhere.
+export function inline(body) {
+  let value = body;
+  if (typeof body === "string") {
+    try { value = JSON.parse(body); } catch { return body; }
+  }
+  return value && typeof value === "object" && "helicone_message" in value ? undefined : value;
+}
+
+async function bodiesOf(row, fetchImpl) {
+  if (!row.signed_body_url) return { request: inline(row.request_body), response: inline(row.response_body) };
+  const stored = await answered(fetchImpl, addressOf(row.signed_body_url).href, {}, null);
+  return { request: stored?.request, response: stored?.response };
+}
+
+export async function readHelicone({ env, since, fetchImpl = fetch }) {
+  const root = rootOf(HELICONE.hostOf(env));
+  const headers = { authorization: `Bearer ${env.HELICONE_API_KEY}`, "content-type": "application/json" };
+  const filter = { request_response_rmt: { request_created_at: { gte: since.toISOString() } } };
+  const turns = [];
+  let read = 0;
+  for (let offset = 0; offset < MAX_REQUESTS; offset += PAGE) {
+    const body = JSON.stringify({ filter, offset, limit: PAGE, sort: { created_at: "desc" } });
+    const page = await answered(fetchImpl, `${root}/v1/request/query-clickhouse`, { method: "POST", headers, body }, "HELICONE");
+    const rows = page.data ?? [];
+    read += rows.length;
+    const chats = rows.filter((row) => !NOT_A_CHAT.test(row.request_path ?? ""));
+    const bodies = await Promise.all(chats.map((row) => bodiesOf(row, fetchImpl)));
+    chats.forEach((row, i) => turns.push({
+      conversation: row.request_properties?.["Helicone-Session-Id"] || row.request_id,
+      user: row.request_user_id,
+      at: row.request_created_at,
+      question: said(bodies[i].request, "user"),
+      answer: said(bodies[i].response, "assistant"),
+    }));
+    if (rows.length < PAGE) break;
+  }
+  return { ...grouped(turns), host: new URL(root).host, read };
+}

package/lib/history/langsmith.mjs

@@ -0,0 +1,62 @@
+import { said } from "./langfuse.mjs";
+import { answered, grouped, rootOf, threadIn, userIn } from "./vendor.mjs";
+
+// The app's recent conversations as LangSmith holds them, read with the key the app already has, sent
+// to LangSmith's own address and nowhere else. The project is found by name with GET /api/v1/sessions;
+// its root runs, one reply each, come from POST /api/v2/runs/query (LangSmith API reference, "Query
+// runs"). v1 runs/query is deprecated and answers 501 on SmithDB deployments.
+const PAGE = 100;
+const MAX_RUNS = 3_000;
+const SELECTS = ["ID", "START_TIME", "INPUTS", "OUTPUTS", "METADATA", "THREAD_ID"];
+
+export const LANGSMITH = {
+  id: "langsmith",
+  name: "LangSmith",
+  needs: ["LANGSMITH_API_KEY"],
+  hostOf: (env) => env.LANGSMITH_ENDPOINT || env.LANGCHAIN_ENDPOINT || "https://api.smith.langchain.com",
+  read: readLangSmith,
+};
+
+// LangChain writes a message's role as its type, or as the class of a serialized message.
+const ROLE_OF_TYPE = { human: "user", HumanMessage: "user", ai: "assistant", AIMessage: "assistant", AIMessageChunk: "assistant" };
+
+export function asChat(value) {
+  if (!value || typeof value !== "object" || !Array.isArray(value.messages)) return value;
+  return {
+    ...value,
+    messages: value.messages.map((m) => {
+      if (!m || typeof m !== "object" || m.role) return m;
+      const role = ROLE_OF_TYPE[m.type === "constructor" ? m.id?.at(-1) : m.type];
+      return role ? { role, content: m.kwargs?.content ?? m.content } : m;
+    }),
+  };
+}
+
+export async function readLangSmith({ env, since, fetchImpl = fetch }) {
+  // Self-hosted endpoints are given with their /api/v1; every path below starts at the root.
+  const root = rootOf(LANGSMITH.hostOf(env)).replace(/\/api(?:\/v1)?$/, "");
+  const host = new URL(root).host;
+  const headers = { "x-api-key": env.LANGSMITH_API_KEY, ...(env.LANGSMITH_WORKSPACE_ID ? { "x-tenant-id": env.LANGSMITH_WORKSPACE_ID } : {}) };
+  const ask = (path, init = {}) => answered(fetchImpl, `${root}${path}`, { ...init, headers: { ...headers, ...init.headers } }, "LANGSMITH");
+
+  const name = env.LANGSMITH_PROJECT || env.LANGCHAIN_PROJECT || "default";
+  const project = (await ask(`/api/v1/sessions?name=${encodeURIComponent(name)}`)).find((p) => p.name === name);
+  if (!project) throw new Error(`${host} has no project named ${name}`);
+
+  const runs = [];
+  for (let cursor = null; runs.length < MAX_RUNS;) {
+    const body = { project_ids: [project.id], is_root: true, min_start_time: since.toISOString(), page_size: PAGE, selects: SELECTS, ...(cursor ? { cursor } : {}) };
+    const page = await ask("/api/v2/runs/query", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) });
+    runs.push(...(page.items ?? []));
+    cursor = page.next_cursor;
+    if (!cursor) break;
+  }
+  const { conversations, withoutText } = grouped(runs.map((run) => ({
+    conversation: run.thread_id || threadIn(run.metadata) || run.id,
+    user: userIn(run.metadata),
+    at: run.start_time,
+    question: said(asChat(run.inputs), "user"),
+    answer: said(asChat(run.outputs), "assistant"),
+  })));
+  return { conversations, host, read: runs.length, withoutText };
+}

package/lib/history/opik.mjs

@@ -0,0 +1,42 @@
+import { said } from "./langfuse.mjs";
+import { answered, grouped, rootOf, userIn } from "./vendor.mjs";
+
+// The app's recent conversations as Opik traced them, read with the key the app already has, sent to
+// Opik's own API and nowhere else. Traces, one reply each and newest first, come from GET
+// /v1/private/traces by page (Opik REST API, "Get traces by project_name or project_id").
+const PAGE = 100;
+const MAX_TRACES = 3_000;
+
+export const OPIK = {
+  id: "opik",
+  name: "Opik",
+  needs: ["OPIK_API_KEY", "OPIK_WORKSPACE"],
+  hostOf: (env) => env.OPIK_URL_OVERRIDE || "https://www.comet.com/opik/api",
+  read: readOpik,
+};
+
+// Experiments, the playground, optimizers and evaluators write traces too; the app's own say "sdk".
+const fromTheApp = (trace) => !trace.source || trace.source === "sdk";
+
+export async function readOpik({ env, since, fetchImpl = fetch }) {
+  const root = rootOf(OPIK.hostOf(env));
+  const headers = { authorization: env.OPIK_API_KEY, "Comet-Workspace": env.OPIK_WORKSPACE };
+  const project = env.OPIK_PROJECT_NAME || "Default Project";
+  const traces = [];
+  for (let page = 1; traces.length < MAX_TRACES; page += 1) {
+    // Attachments stay references; only the text is read.
+    const query = new URLSearchParams({ project_name: project, page: String(page), size: String(PAGE), from_time: since.toISOString(), strip_attachments: "true" });
+    const body = await answered(fetchImpl, `${root}/v1/private/traces?${query}`, { headers }, "OPIK");
+    const content = body.content ?? [];
+    traces.push(...content);
+    if (content.length < PAGE) break;
+  }
+  const { conversations, withoutText } = grouped(traces.filter(fromTheApp).map((trace) => ({
+    conversation: trace.thread_id || trace.id,
+    user: userIn(trace.metadata),
+    at: trace.start_time,
+    question: said(trace.input, "user"),
+    answer: said(trace.output, "assistant"),
+  })));
+  return { conversations, host: new URL(root).host, read: traces.length, withoutText };
+}

package/lib/history/phoenix.mjs

@@ -0,0 +1,58 @@
+import { said } from "./langfuse.mjs";
+import { answered, grouped, rootOf } from "./vendor.mjs";
+
+// The app's recent conversations as Phoenix traced them, read with the key the app already has, sent
+// to the app's own Phoenix and nowhere else. Root spans, one reply each and newest first, come from
+// GET /v1/projects/{project_identifier}/spans with the cursor each page hands back (Phoenix REST API,
+// "List spans with simple filters").
+const PAGE = 100;
+const MAX_SPANS = 3_000;
+const OTLP_PATH = /\/+v1\/traces\/?$/;
+
+export const PHOENIX = {
+  id: "phoenix",
+  name: "Phoenix",
+  needs: ["PHOENIX_COLLECTOR_ENDPOINT"],
+  // As Phoenix's own client finds the server: its API address, else the collector's less the OTLP path.
+  hostOf: (env) => env.PHOENIX_ENDPOINT || env.PHOENIX_COLLECTOR_ENDPOINT?.replace(OTLP_PATH, "") || "http://localhost:6006",
+  read: readPhoenix,
+};
+
+// OpenInference flattens a span's chat into keys such as llm.input_messages.0.message.role.
+export function messagesIn(attributes, prefix) {
+  const byIndex = new Map();
+  for (const [key, value] of Object.entries(attributes ?? {})) {
+    const part = key.startsWith(`${prefix}.`) && /^(\d+)\.message\.(role|content)$/.exec(key.slice(prefix.length + 1));
+    if (part) byIndex.set(Number(part[1]), { ...byIndex.get(Number(part[1])), [part[2]]: value });
+  }
+  return [...byIndex].sort(([a], [b]) => a - b).map(([, message]) => message);
+}
+
+// A span's input or output: parsed when its mime type says JSON, else the text as written.
+export function valueIn(attributes, kind) {
+  const value = attributes?.[`${kind}.value`];
+  if (attributes?.[`${kind}.mime_type`] !== "application/json" || typeof value !== "string") return value;
+  try { return JSON.parse(value); } catch { return value; }
+}
+
+export async function readPhoenix({ env, since, fetchImpl = fetch }) {
+  const root = rootOf(PHOENIX.hostOf(env));
+  const project = encodeURIComponent(env.PHOENIX_PROJECT || env.PHOENIX_PROJECT_NAME || "default");
+  const headers = env.PHOENIX_API_KEY ? { authorization: `Bearer ${env.PHOENIX_API_KEY}` } : {};
+  const spans = [];
+  for (let cursor = null; spans.length < MAX_SPANS;) {
+    const query = new URLSearchParams({ parent_id: "null", sort: "start_time", order: "desc", start_time: since.toISOString(), limit: String(PAGE), ...(cursor ? { cursor } : {}) });
+    const page = await answered(fetchImpl, `${root}/v1/projects/${project}/spans?${query}`, { headers }, "PHOENIX");
+    spans.push(...(page.data ?? []));
+    cursor = page.next_cursor;
+    if (!cursor) break;
+  }
+  const { conversations, withoutText } = grouped(spans.map(({ attributes: a = {}, context, start_time: at }) => ({
+    conversation: a["session.id"] || context?.trace_id,
+    user: a["user.id"],
+    at,
+    question: said(messagesIn(a, "llm.input_messages"), "user") || said(valueIn(a, "input"), "user"),
+    answer: said(messagesIn(a, "llm.output_messages"), "assistant") || said(valueIn(a, "output"), "assistant"),
+  })));
+  return { conversations, host: new URL(root).host, read: spans.length, withoutText };
+}

package/lib/history/vendor.mjs

@@ -0,0 +1,68 @@
+// What the history readers share: the vendor's address checked before a key goes to it, its JSON
+// answer with a busy answer waited out, and its records made into conversations.
+const LOCAL = new Set(["localhost", "127.0.0.1"]);
+const TRIES = 4;
+const MAX_WAIT_MS = 30_000;
+const THREAD_KEYS = ["thread_id", "session_id", "conversation_id"];
+
+const sleep = (ms) => new Promise((resolve) => { setTimeout(resolve, ms); });
+
+// An address a vendor's key may go to: https, or http to this computer only.
+export function addressOf(raw) {
+  const url = URL.canParse(raw) ? new URL(raw) : null;
+  if (!url || url.username || url.password) throw new Error("an address in your env is not a plain URL");
+  if (url.protocol !== "https:" && !(url.protocol === "http:" && LOCAL.has(url.hostname))) throw new Error(`${url.host} is not reached over https`);
+  return url;
+}
+
+// The API's root with no trailing slash, query or fragment, for paths to join onto.
+export function rootOf(raw) {
+  const url = addressOf(raw);
+  return `${url.origin}${url.pathname.replace(/\/+$/, "")}`;
+}
+
+// The vendor's JSON. A 429 is tried again a few times, after as long as the vendor asks (up to 30
+// seconds) or longer each time. `keys` names whose keys went with the request; a signed URL carries none.
+export async function answered(fetchImpl, url, init, keys) {
+  const { host } = new URL(url);
+  for (let attempt = 1; ; attempt += 1) {
+    const res = await fetchImpl(url, { ...init, signal: AbortSignal.timeout(30_000) });
+    if (res.status === 429 && attempt < TRIES) { await sleep(waitOf(res, attempt)); continue; }
+    if (keys && (res.status === 401 || res.status === 403)) throw new Error(`${host} refused the ${keys} keys in your env`);
+    if (!res.ok) throw new Error(`${host} answered ${res.status}`);
+    return res.json();
+  }
+}
+
+function waitOf(res, attempt) {
+  const seconds = Number(res.headers.get("retry-after") || Number.NaN);
+  return Number.isFinite(seconds) && seconds >= 0 ? Math.min(seconds * 1000, MAX_WAIT_MS) : 1000 * 2 ** (attempt - 1);
+}
+
+// The conversation and the person an app named in a record's metadata, if it named them.
+export const threadIn = (metadata) => THREAD_KEYS.map((k) => metadata?.[k]).find((v) => typeof v === "string" && v) ?? null;
+export const userIn = (metadata) => (typeof metadata?.user_id === "string" && metadata.user_id) || null;
+
+// A vendor's time in milliseconds. Every vendor here keeps UTC, and Helicone writes it with no zone.
+export const msOf = (time) => Date.parse(/(?:Z|[+-]\d{2}:?\d{2})$/i.test(time) ? time : `${String(time).replace(" ", "T")}Z`);
+
+const isoOf = (ms) => (Number.isFinite(ms) ? new Date(ms).toISOString() : null);
+
+// Turns as conversations. A turn is one record: the person's newest message and the app's reply, the
+// reply a millisecond later so the two keep their order. A turn with neither is counted, not kept.
+export function grouped(turns) {
+  const byConversation = new Map();
+  let withoutText = 0;
+  for (const turn of turns) {
+    const question = turn.question.trim();
+    const answer = turn.answer.trim();
+    if (!question && !answer) { withoutText += 1; continue; }
+    const held = byConversation.get(turn.conversation) ?? { id: turn.conversation, messages: [] };
+    if (!held.user && turn.user) held.user = turn.user;
+    const at = msOf(turn.at);
+    if (question) held.messages.push({ role: "user", text: question, at: isoOf(at) });
+    if (answer) held.messages.push({ role: "assistant", text: answer, at: isoOf(at + 1) });
+    byConversation.set(turn.conversation, held);
+  }
+  return { conversations: [...byConversation.values()], withoutText };
+}

package/lib/intelligence.mjs

@@ -8,14 +8,19 @@ import { homeOf } from "./home.mjs";
 import { pair } from "./intelligence-pair.mjs";
 import { canSet, hostOf, HOSTS, setKey } from "./hosts.mjs";
 import { looksLikeDevelopment, selected, spanOf } from "./history/contract.mjs";
+import { BRAINTRUST } from "./history/braintrust.mjs";
+import { HELICONE } from "./history/helicone.mjs";
 import { LANGFUSE } from "./history/langfuse.mjs";
+import { LANGSMITH } from "./history/langsmith.mjs";
+import { OPIK } from "./history/opik.mjs";
+import { PHOENIX } from "./history/phoenix.mjs";
 import { packed, sendAll } from "./history/send.mjs";
 import * as say from "./intelligence-text.mjs";
 
 // `npx cortad intelligence`: Cortad Intelligence set up from this folder, one step per run. Each run
 // reads where things stand (Cortad's facts about this app, and this folder) and prints what is done
 // and the one next step, with the question for the person where one is theirs. It edits no code.
-export const SOURCES = [LANGFUSE];
+export const SOURCES = [LANGFUSE, LANGSMITH, HELICONE, BRAINTRUST, PHOENIX, OPIK];
 const ENV_FILES = [".env", ".env.local", ".env.development", ".env.development.local", ".dev.vars"];
 const WAIT_MS = 9 * 60_000;
 const POLL_MS = 5_000;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cortad/connector",
-  "version": "0.6.0-rc.1",
+  "version": "0.6.0-rc.2",
   "description": "Connects the AI app on your machine to Cortad for test conversations, and gives your coding agent the MCP and skill to run them.",
   "bin": {
     "cortad": "local.mjs"
@@ -73,10 +73,16 @@
     "lib/history/contract.mjs",
     "lib/history/send.mjs",
     "lib/history/langfuse.mjs",
-    "lib/intelligence-pair.mjs"
+    "lib/intelligence-pair.mjs",
+    "lib/history/langsmith.mjs",
+    "lib/history/helicone.mjs",
+    "lib/history/braintrust.mjs",
+    "lib/history/phoenix.mjs",
+    "lib/history/opik.mjs",
+    "lib/history/vendor.mjs"
   ],
   "scripts": {
-    "test": "node --test lib/*.test.mjs"
+    "test": "node --test lib/*.test.mjs lib/history/*.test.mjs"
   },
   "engines": {
     "node": ">=20"