@cortad/connector 0.6.2 → 0.6.3

package/lib/history/contract.mjs

@@ -1,6 +1,9 @@
 import { createHash } from "node:crypto";
+import messages from "../message-text.cjs";
 import { scrub } from "../redact.mjs";
 
+const { messageText } = messages;
+
 // Conversations from before, as any source hands them over, made into exactly what Cortad is sent:
 // the person's and the app's messages, oldest first, with the app's own ids hashed on this machine.
 // Newest conversations first and whole, from the last 30 days, until 2,500 messages as a plan counts
@@ -11,6 +14,20 @@ const DAY_MS = 86_400_000;
 const MAX_MESSAGES = 200;
 const MAX_TEXT = 8_000;
 const ROLES = { user: "user", human: "user", assistant: "assistant", ai: "assistant", bot: "assistant" };
+// What the app itself knew about the user for a conversation (a grade, a subject), as the export
+// carries it: the server takes 16 at most, each name a word or two and each value a short string.
+const MAX_ATTRIBUTES = 16;
+const MAX_VALUE = 200;
+const ATTRIBUTE_NAME = /^[\p{L}\p{N} _.-]{1,64}$/u;
+
+export function attributesOf(raw) {
+  if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
+  const kept = Object.entries(raw)
+    .filter(([name, value]) => ATTRIBUTE_NAME.test(name) && (typeof value === "string" || typeof value === "number" || typeof value === "boolean") && String(value).trim())
+    .slice(0, MAX_ATTRIBUTES)
+    .map(([name, value]) => [name, cut(String(value).trim(), MAX_VALUE)]);
+  return kept.length ? Object.fromEntries(kept) : null;
+}
 
 export const sha256 = (text) => createHash("sha256").update(String(text)).digest("hex");
 
@@ -30,11 +47,6 @@ const cut = (text, max) => {
   return text.slice(0, end);
 };
 
-const textOf = (content) => {
-  if (typeof content === "string") return content;
-  if (Array.isArray(content)) return content.map((part) => (typeof part === "string" ? part : part?.type === "text" ? part.text : "[image]")).filter(Boolean).join("\n");
-  return "";
-};
 
 // One conversation as the contract has it, or the reason it is left out.
 export function conversationOf(raw, count) {
@@ -47,7 +59,7 @@ export function conversationOf(raw, count) {
     if (!role) { count("not a person's or the app's message"); continue; }
     const at = timeOf(m.at ?? m.createdAt ?? m.created_at);
     if (!at) { count("a time with no zone"); continue; }
-    const text = textOf(m.text ?? m.content).trim();
+    const text = messageText(m.text ?? m.content).trim();
     if (!text) continue;
     if (text.length > MAX_TEXT) count("message cut at 8,000 characters");
     messages.push({ role, text: cut(text, MAX_TEXT), at });
@@ -56,15 +68,17 @@ export function conversationOf(raw, count) {
   messages.sort((a, b) => a.at - b.at);
   if (messages.length > MAX_MESSAGES) count("conversation kept to its latest 200 messages");
   const kept = messages.slice(-MAX_MESSAGES);
-  return { id, ...(raw.user ? { user: String(raw.user) } : {}), messages: kept, last: kept.at(-1).at };
+  const attributes = attributesOf(raw.attributes);
+  return { id, ...(raw.user ? { user: String(raw.user) } : {}), ...(attributes ? { attributes } : {}), messages: kept, last: kept.at(-1).at };
 }
 
-// The conversations to send, newest first, whole, from the window, until the plan's messages; and
-// why each one left out was left out.
+// The conversations to send, newest first, whole, from the window, until the messages asked for
+// (`cap`, never past the plan's); and why each one left out was left out.
 // `key`: the app's ingest key. Ids and users are hashed with it exactly as the SDK hashes them, so a
 // person brought in from history is the same person when they write to the live app, and a guessable
 // id (a first name, an email) cannot be read back from its hash without the key.
-export function selected(raws, { now = Date.now(), secrets = [], key }) {
+export function selected(raws, { now = Date.now(), secrets = [], key, cap = HISTORY_MESSAGES }) {
+  const most = Math.min(cap, HISTORY_MESSAGES);
   const subject = (kind, value) => sha256(`${key}:${kind}:${value}`);
   const left = new Map();
   const count = (why) => { left.set(why, (left.get(why) ?? 0) + 1); return null; };
@@ -74,15 +88,17 @@ export function selected(raws, { now = Date.now(), secrets = [], key }) {
   const recent = [...byId.values()].filter((c) => c.last.getTime() >= now - HISTORY_DAYS * DAY_MS || count("older than 30 days"));
   recent.sort((a, b) => b.last - a.last);
   const out = [];
-  let room = HISTORY_MESSAGES;
+  let room = most;
   let scrubbed = 0;
   for (const c of recent) {
     const replies = c.messages.filter((m) => m.role === "assistant").length;
-    if (replies > room) { count("past 2,500 messages"); continue; }
+    if (replies > room) { count(`past ${most.toLocaleString("en-US")} messages`); continue; }
     room -= replies;
     out.push({
       id: subject("session", c.id),
       ...(c.user ? { user: subject("user", c.user) } : {}),
+      // Scrubbed like the messages: a value is the app's own text and could hold a key.
+      ...(c.attributes ? { attributes: Object.fromEntries(Object.entries(c.attributes).map(([k, v]) => { const clean = scrub(v, secrets); scrubbed += clean.hits; return [k, clean.text]; })) } : {}),
       messages: c.messages.map((m) => {
         const clean = scrub(m.text, secrets);
         scrubbed += clean.hits;
@@ -90,7 +106,7 @@ export function selected(raws, { now = Date.now(), secrets = [], key }) {
       }),
     });
   }
-  return { conversations: out, messages: HISTORY_MESSAGES - room, total: out.reduce((n, c) => n + c.messages.length, 0), scrubbed, left: Object.fromEntries(left), users: new Set(recent.map((c) => c.user).filter(Boolean)).size };
+  return { conversations: out, messages: most - room, total: out.reduce((n, c) => n + c.messages.length, 0), scrubbed, left: Object.fromEntries(left), users: new Set(recent.map((c) => c.user).filter(Boolean)).size };
 }
 
 // The span of what will be sent, oldest message to newest.

package/lib/history/sources.mjs

@@ -0,0 +1,11 @@
+import { BRAINTRUST } from "./braintrust.mjs";
+import { HELICONE } from "./helicone.mjs";
+import { LANGFUSE } from "./langfuse.mjs";
+import { LANGSMITH } from "./langsmith.mjs";
+import { OPIK } from "./opik.mjs";
+import { PHOENIX } from "./phoenix.mjs";
+
+// Every tool history can be read from, and which of them an env names every key of. Names only:
+// a value is checked for being there, never read out or sent.
+export const SOURCES = [LANGFUSE, LANGSMITH, HELICONE, BRAINTRUST, PHOENIX, OPIK];
+export const sourcesIn = (env) => SOURCES.filter((s) => s.needs.every((n) => env[n]));

package/lib/intelligence-text.mjs

@@ -50,10 +50,10 @@ export function historyQuestion({ spec, sources, stores }) {
   ];
   return [
     "",
-    "Step 1 of 2: your users' conversations",
+    "Step 1 of 2: your users' messages",
     ...sources.map((s) => `Found ${s.name} (${s.names.join(" and ")} in your env).`),
     ...(store ? [`Found where this app keeps its conversations: ${storeName(store)}.`] : []),
-    `Ask the person, with your question tool: "Send our newest whole conversations from the last 30 days to Cortad, until our app's replies reach 2,500, so it can show us our users?"`,
+    `Ask the person, with your question tool: "Send our users' newest 2,500 messages from the last 30 days to Cortad, so it can show us our users?" Cortad counts your app's replies and takes each conversation whole.`,
     ...options.map((o, i) => `  ${i + 1}. ${o}`),
     ...sources.map((s) => `${s.name} is read from this computer at ${s.host} with the keys in your env; the keys go to ${s.name} only.`),
     "Before anything is sent, keys, card numbers, emails and phone numbers in the messages are replaced on this computer, and the app's conversation and user ids are hashed. What is sent goes to cortad.com, and a copy of exactly what was sent stays in ~/.cortad on this computer.",
@@ -99,39 +99,60 @@ export const historyNothing = (source) => [`${source} holds no conversations wit
 export const historySkipped = () => ["Skipped. Cortad starts with your live app's next deploy; history can still be sent later with the same command."];
 
 // Step 2: the code change, as lines for the agent to put in, and the one question that settles it.
-export function codeStep({ spec, stack, page, file, site, consent, host, settable }) {
-  const install = stack.kind === "python" ? `${stack.add} cortad==${SDK}` : `${stack.add} @cortad/sdk@${SDK}`;
+// An app in a language with no SDK of ours sends each conversation itself: one signed request, written
+// by the agent in the app's own language. The contract is the backend's POST /v1/conversations.
+const httpSteps = (address) => [
+  `1. No package to install: this app's language has no Cortad SDK, so the app sends each conversation in one HTTP request, written in its own language with its standard HTTP and crypto libraries.`,
+  `2. After each reply, from the server, never the browser: POST ${address}`,
+  `   Body (JSON): {"id": "<the app's own conversation id, the same on every turn>", "messages": [{"role": "user" or "assistant", "text": "<what was said>", "at": "<ISO 8601 time with its zone>"}, ...]}, every message so far, oldest first. A message that is not text (an image, a file, a tool call) goes in as a short marker such as "[image]".`,
+  `   Headers: content-type: application/json; x-cortad-key: the first 15 characters of CORTAD_INGEST_KEY; x-cortad-timestamp: now in milliseconds since 1970; x-cortad-nonce: 32 random hex characters, new on every request;`,
+  `   x-cortad-signature: "v1=" + hex(HMAC-SHA256(key = hex(SHA-256(CORTAD_INGEST_KEY)) as text, message = "v1." + timestamp + "." + nonce + "." + hex(SHA-256(the exact body bytes))))`,
+  `3. Send it only from the deployed app (skip it when CORTAD_ENV=development), off the reply's path so a slow send never holds the person's answer, and log a refusal's status without the key. 202 means it landed.`,
+];
+
+export function codeStep({ spec, stack, page, file, site, consent, host, settable, address = "https://cortad.com/api/v1/conversations" }) {
+  if (stack.kind === "other") return stepFrame({ spec, page, file, site, consent, host, settable, steps: httpSteps(address), closing: "Then deploy it: the first conversation on the live app shows in Cortad within a minute." });
+  const install = stack.kind === "python" ? `${stack.add} cortad==${SDK.python}` : `${stack.add} @cortad/sdk@${SDK.node}`;
   const init = stack.kind === "python"
     ? ["  from cortad import cortad", '  bl = cortad(key=os.environ["CORTAD_INGEST_KEY"], capture_content=True, only_run=True)', "  bl.install()"]
     : ['  import { cortad } from "@cortad/sdk";', "  const bl = cortad({ key: process.env.CORTAD_INGEST_KEY, captureContent: true, onlyRun: true });", "  bl.install();"];
   const wrap = stack.kind === "python"
     ? ["  with bl.run(conversation_id):", "      reply = <the call that writes the reply>"]
     : ["  const reply = await bl.run({ id: conversationId }, () => <the call that writes the reply>);"];
-  return [
-    "",
-    "Step 2 of 2: Cortad in your code",
-    "Make this change yourself; this command edits nothing. Show the person the diff.",
+  return stepFrame({ spec, page, file, site, consent, host, settable, steps: [
     `1. Install the SDK: ${install}`,
     "2. Where the app starts, once:",
     ...init,
     "3. Around the one model call that writes the reply the person reads, with the app's own conversation id, the same on every turn:",
     ...wrap,
     "   Routing, rewriting and title calls stay outside it. The SDK sends only from the deployed app; this computer's copy reads CORTAD_ENV=development and sends nothing but a hello.",
+  ] });
+}
+
+function stepFrame({ spec, page, file, site, consent, host, settable, steps, closing = "Then restart the app on this computer and send it one message, so Cortad sees the change work here." }) {
+  return [
+    "",
+    "Step 2 of 2: Cortad in your code",
+    "Make this change yourself; this command edits nothing. Show the person the diff.",
+    ...steps,
     ...(page ? [
       `4. In ${page.file ?? "the page every person loads"}, once: <script src="https://b.cortad.com/i/cortad.js" data-key="${page.key}"${consent ? " data-opt-out-by-default" : ""} async></script>`,
       "   This key is public by design: it only names which site may send what people did around each answer.",
       ...(consent ? [`   This app asks visitors for consent with ${consent}: where it records a yes, call window.cortad?.optIn().`] : []),
     ] : ["4. This app has no web page of its own, so there is no page tag."]),
-    "Then restart the app on this computer and send it one message, so Cortad sees the change work here.",
+    closing,
     "",
-    ...secondQuestion({ spec, host, settable, file, site }),
+    ...secondQuestion({ spec, host, settable, file, site, tagged: Boolean(page) }),
   ];
 }
-export const SDK = "0.3.3";
+// The SDK each kind of app installs, one place for each package: the Node one is 0.3.5 (0.3.3 and 0.3.4
+// crash an app that loads it with require), the Python one is at its own 0.3.4.
+export const SDK = { node: "0.3.5", python: "0.3.4" };
 
-function secondQuestion({ spec, host, settable, file, site }) {
+// The address is asked only of an app with a page tag: its key is what the address locks.
+function secondQuestion({ spec, host, settable, file, site, tagged }) {
   const h = HOSTS[host] ?? HOSTS.other;
-  const siteAsk = site ? [] : [`In the same form, ask: "Which address do people open your app at?" Then run npx -y ${spec} intelligence site <that address>, so the page key answers that site and no other.`];
+  const siteAsk = site || !tagged ? [] : [`In the same form, ask: "Which address do people open your app at?" Then run npx -y ${spec} intelligence site <that address>, so the page key answers that site and no other.`];
   if (settable) {
     return [
       `Ask the person: "Commit this on a new branch named cortad-intelligence, and add CORTAD_INGEST_KEY to ${h.name} production? Nothing live changes until you deploy."`,
@@ -156,11 +177,13 @@ export const siteLocked = (site) => [`The page key now answers ${site} and no ot
 export const workingHere = (at) => `Cortad started with your app on this computer at ${new Date(at).toLocaleTimeString("en-GB", { hour: "2-digit", minute: "2-digit" })}. The change works here.`;
 export const live = (connection) => `Your live app started with Cortad${connection.started?.release ? ` on version ${connection.started.release.slice(0, 7)}` : ""}.`;
 
-export function findings({ history, problems, where, done }) {
+// Said in messages, the number the person chose; in conversations where the server says only those.
+export function findings({ history, replies, problems, where, done }) {
   if (history.conversations === 0) return [];
-  const read = `${num(history.read)} of ${num(history.conversations)} conversations read`;
+  const [n, of, unit] = replies?.of ? [replies.read, replies.of, "messages"] : [history.read, history.conversations, "conversations"];
+  const read = `${num(n)} of ${num(of)} ${unit} read`;
   if (history.read === 0) return [`${read}. Reading has started.`];
-  const head = done ? `All ${num(history.conversations)} conversations read.` : `${read} so far.`;
+  const head = done ? `All ${num(of)} ${unit} read.` : `${read} so far.`;
   if (problems.length === 0) return [head, "No behavioral question failed in what was read.", `Every conversation is at ${where}.`];
   return [
     head,

package/lib/intelligence.mjs

@@ -7,20 +7,16 @@ import { parseEnv } from "./env-files.mjs";
 import { homeOf } from "./home.mjs";
 import { pair } from "./intelligence-pair.mjs";
 import { canSet, hostOf, HOSTS, setKey } from "./hosts.mjs";
-import { looksLikeDevelopment, selected, spanOf } from "./history/contract.mjs";
-import { BRAINTRUST } from "./history/braintrust.mjs";
-import { HELICONE } from "./history/helicone.mjs";
-import { LANGFUSE } from "./history/langfuse.mjs";
-import { LANGSMITH } from "./history/langsmith.mjs";
-import { OPIK } from "./history/opik.mjs";
-import { PHOENIX } from "./history/phoenix.mjs";
+import { HISTORY_MESSAGES, looksLikeDevelopment, selected, spanOf } from "./history/contract.mjs";
 import { packed, sendAll } from "./history/send.mjs";
+import { SOURCES, sourcesIn } from "./history/sources.mjs";
+import { changesIn, report } from "./setup-live.mjs";
 import * as say from "./intelligence-text.mjs";
 
 // `npx cortad intelligence`: Cortad Intelligence set up from this folder, one step per run. Each run
 // reads where things stand (Cortad's facts about this app, and this folder) and prints what is done
 // and the one next step, with the question for the person where one is theirs. It edits no code.
-export const SOURCES = [LANGFUSE, LANGSMITH, HELICONE, BRAINTRUST, PHOENIX, OPIK];
+export { SOURCES };
 const ENV_FILES = [".env", ".env.local", ".env.development", ".env.development.local", ".dev.vars"];
 const WAIT_MS = 9 * 60_000;
 const POLL_MS = 5_000;
@@ -44,7 +40,11 @@ export async function intelligence(args, ctx) {
   const flag = (name) => rest.includes(name);
   const value = (name) => { const i = rest.indexOf(name); return i === -1 ? null : rest[i + 1] ?? null; };
 
-  if (sub === "history") return history(ctx, api, { from: value("--from"), stdin: flag("--stdin"), skip: flag("--skip"), dry: flag("--dry-run"), anyway: flag("--anyway") });
+  if (sub === "history") {
+    const cap = value("--messages");
+    if (cap !== null && !capOk(cap)) return { text: say.lines(`--messages is a whole number from 1 to ${HISTORY_MESSAGES.toLocaleString("en-US")}: how many of your users' newest messages to bring in.`), code: 1 };
+    return history(ctx, api, { from: value("--from"), stdin: flag("--stdin"), skip: flag("--skip"), dry: flag("--dry-run"), anyway: flag("--anyway"), cap: cap === null ? HISTORY_MESSAGES : Number(cap) });
+  }
   if (sub === "key") return key(ctx, value("--to"));
   if (sub === "site") return site(ctx, api, rest[0]);
   return start(ctx, api, { wait: sub === "--wait" || flag("--wait") });
@@ -61,13 +61,16 @@ export async function intelligence(args, ctx) {
     const { connection, history: past, stores } = facts.data;
     if (past.state === "none") {
       const env = envHere();
-      const sources = SOURCES.filter((s) => s.needs.every((n) => env[n])).map((s) => ({ ...s, names: s.needs, host: hostName(s.hostOf(env)) }));
+      const sources = sourcesIn(env).map((s) => ({ ...s, names: s.needs, host: hostName(s.hostOf(env)) }));
       out.push(...say.historyQuestion({ spec, sources, stores: stores ?? [] }));
       return { text: say.lines(out) };
     }
     if (connection.started) out.push(say.live(connection));
     else if (connection.here) out.push(say.workingHere(connection.here.at));
-    if (!connection.started) out.push(...codeStepFor({ ...keys, site: keys.site ?? connection.site?.origin ?? null }));
+    if (!connection.started) {
+      out.push(...codeStepFor({ ...keys, site: keys.site ?? connection.site?.origin ?? null }));
+      await reportCode();
+    }
     const progress = await (wait ? waited(call) : call("/mcp/history"));
     if (progress.ok) out.push("", ...say.findings({ ...progress.data, done: progress.data.history.read >= progress.data.history.conversations }));
     if (progress.ok && progress.data.history.read < progress.data.history.conversations && !wait) out.push(say.next(spec, "--wait"));
@@ -96,14 +99,34 @@ export async function intelligence(args, ctx) {
     return { lines, file: file ?? join(homeOf(ctx.project), "keys.env"), site: answer.data.site ?? detected, page: answer.data.page ?? (file ? publicKeyIn(root, file) : null) };
   }
 
+  // Told to the agent just now, or already in the working tree: the card's code step reads it.
+  async function reportCode() {
+    const found = changesIn(root);
+    await live({ sdk: found.sdk ? "branch" : "busy", ...(pageFileOf(root) ? { tag: found.tag ? "branch" : "busy" } : { tag: "none" }) });
+  }
+
+  function live(body) {
+    return report({ apiBase: ctx.apiBase, token: token(), ...(ctx.fetchImpl ? { fetchImpl: ctx.fetchImpl } : {}) }, body);
+  }
+
   function codeStepFor(keys) {
     const pageFile = pageFileOf(root);
     const host = hostOf(root);
     const page = keys.page ? { file: pageFile, key: keys.page } : null;
-    return say.codeStep({ spec, stack: stackOf(root), page: pageFile ? page : null, file: keys.file, site: keys.site, consent: consentOf(root, pageFile), host, settable: canSet(root, host) });
+    return say.codeStep({ spec, stack: stackOf(root), page: pageFile ? page : null, file: keys.file, site: keys.site, consent: consentOf(root, pageFile), host, settable: canSet(root, host), address: `${ctx.apiBase}/v1/conversations` });
   }
 
+  // The send as the card sees it: under way from the first read, done once Cortad has taken it, and
+  // back to not started when it stops short.
   async function history(ctx_, call, opts) {
+    if (opts.skip || opts.dry) return send(ctx_, call, opts);
+    await live({ history: "busy" });
+    const out = await send(ctx_, call, opts);
+    await live({ history: out.code ? "next" : out.sent ? "done" : "busy" });
+    return out;
+  }
+
+  async function send(ctx_, call, opts) {
     if (opts.skip) {
       const skipped = await call("/mcp/history/skip", { method: "POST" });
       if (!skipped.ok) return refused(skipped);
@@ -137,7 +160,7 @@ export async function intelligence(args, ctx) {
     const label = SOURCES.find((s) => s.id === source)?.name ?? "your export";
     const key = ingestKeyHere();
     if (!key) return { text: say.lines(`This folder has no CORTAD_INGEST_KEY yet. Run npx -y ${spec} intelligence first.`), code: 1 };
-    const picked = selected(raws, { key, secrets: secretValues(ENV_FILES.map((f) => join(root, f)).filter(existsSync)).filter(credentialShaped) });
+    const picked = selected(raws, { key, cap: opts.cap, secrets: secretValues(ENV_FILES.map((f) => join(root, f)).filter(existsSync)).filter(credentialShaped) });
     if (picked.conversations.length === 0) return { text: say.lines(say.historyNothing(label)), code: 1 };
     const sends = packed(source, picked.conversations);
     const file = join(homeOf(ctx.project), "history.json");
@@ -149,7 +172,7 @@ export async function intelligence(args, ctx) {
     if (opts.dry) return { text: say.lines(say.historyDry({ picked, span, file })) };
     const result = await sendAll(sends, { api: ctx.apiBase, token: token() });
     if (result.refused) return { text: say.lines(`Cortad did not take the rest: ${result.refused}`, `${result.taken} conversations were taken before that.`), code: 1 };
-    return { text: say.lines(say.historySent({ result, picked, span, source: label, file }), say.next(spec)) };
+    return { text: say.lines(say.historySent({ result, picked, span, source: label, file }), say.next(spec)), sent: true };
   }
 
   // The ingest key as this folder holds it: in its env file, or in keys.env when a manager holds its settings.
@@ -198,6 +221,7 @@ export async function intelligence(args, ctx) {
   }
 }
 
+const capOk = (raw) => /^[1-9]\d{0,4}$/.test(raw) && Number(raw) <= HISTORY_MESSAGES;
 const hostName = (address) => { try { return new URL(address).host; } catch { return address; } };
 
 // Lines of JSON, one conversation each; a first line naming the source is optional. A line that is