cortad 0.1.6 → 0.1.8

package/local.mjs

@@ -22,7 +22,9 @@ import { openDoor } from "./lib/door.mjs";
 import { lockHolds, makeLock } from "./lib/lock.mjs";
 import { AS_HEADER, makeIdentities } from "./lib/mint.mjs";
 import { CAPTURED, makeCapture } from "./lib/replay.mjs";
-import { startPlan } from "./lib/start.mjs";
+import { sampleHere } from "./lib/sample.mjs";
+import { listingUrl } from "./lib/listing.mjs";
+import { installPlan, missingDependency, startPlan, workspaces } from "./lib/start.mjs";
 
 const argv = process.argv.slice(2);
 const flag = (name) => { const i = argv.indexOf(name); return i >= 0 ? argv[i + 1] : undefined; };
@@ -52,7 +54,12 @@ const api = `${origin.origin}/api`;
 
 const root = process.cwd();
 const MANIFEST = ["package.json", "pyproject.toml", "requirements.txt", "go.mod", "Cargo.toml", "docker-compose.yml", "docker-compose.yaml", "Gemfile", "mix.exs"];
-if (!MANIFEST.some((f) => existsSync(join(root, f)))) fail(`this folder has no package.json or pyproject.toml. Run it from your repository's root: ${root}`);
+// A repository whose root carries no manifest of its own is still a repository: crewai-examples
+// keeps one project per folder under crews/, and the whole of it was refused at the door. The root
+// is accepted when a project lives below it.
+if (!MANIFEST.some((f) => existsSync(join(root, f))) && !workspaces(root).length) {
+  fail(`no project here: this folder has no package.json or pyproject.toml, and neither does any folder in it. Run it from your repository's root: ${root}`);
+}
 
 // ---- what leaves the machine: the source git would commit, and nothing git is told to ignore
 const SKIP_DIR = /^(node_modules|\.git|dist|build|out|coverage|vendor|venv|\.venv|env|target|tmp|\.next|\.nuxt|\.turbo|\.cache|__pycache__|\.terraform|\.wrangler|\.svelte-kit|\.output|\.parcel-cache|\.idea|\.vscode|secrets?|\.secrets?)$/i;
@@ -136,6 +143,8 @@ function envOrigins(envFiles) {
     for (const line of text.split("\n")) {
       const m = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(line);
       if (!m || !/ORIGIN|URL|HOST|DOMAIN|FRONTEND|CLIENT|SITE|WEB/i.test(m[1])) continue;
+      // OPENAI_BASE_URL names where a key is sent, not a page a browser opens the app from.
+      if (/(?:BASE_URL|API_BASE|ENDPOINT)$/i.test(m[1]) && !/ORIGIN|FRONTEND|CLIENT|SITE|WEB|PUBLIC|APP/i.test(m[1])) continue;
       for (const v of m[2].replace(/^(['"])(.*)\1$/, "$2").split(",")) {
         try { const u = new URL(v.trim()); if (/^https?:$/.test(u.protocol)) out.add(u.origin); } catch { /* not an origin */ }
       }
@@ -175,6 +184,53 @@ function liftedLimits(envFiles, sources = []) {
   }
   return out;
 }
+
+// Every name your env files set, with its value, read here and used only here. An example file is
+// read first so a real one's value wins: a placeholder key asked for a model listing comes back
+// refused, and that would read as your own key being refused.
+function envValues(envFiles) {
+  const example = /\.(example|sample)$/;
+  const out = {};
+  for (const file of [...envFiles.filter((f) => example.test(f)), ...envFiles.filter((f) => !example.test(f))]) {
+    let text = "";
+    try { text = readFileSync(file, "utf8"); } catch { continue; }
+    for (const line of text.split("\n")) {
+      const m = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(line);
+      if (m) out[m[1]] = m[2].trim().replace(/^(['"])(.*)\1$/, "$2");
+    }
+  }
+  return out;
+}
+// Switch-shaped names and the word each one reads as. The name and the word travel, never the value.
+const SWITCH_NAME = /(?:_ENABLED|_DISABLED|_MODE|_ON|_OFF|_FLAG)$|^(?:ENABLE|USE|DISABLE|SKIP)_/;
+const switchStates = (values) => Object.fromEntries(Object.entries(values)
+  .filter(([name]) => SWITCH_NAME.test(name))
+  .map(([name, value]) => [name, value === "" ? "unset" : /^(?:1|true|yes|on)$/i.test(value) ? "on" : /^(?:0|false|no|off)$/i.test(value) ? "off" : "set"]));
+
+// What your app can actually reach: each provider key your env files set is asked that provider
+// for its own model listing, from this machine. Your key stays here; what goes back is the name it
+// is set under, the host, what the host answered, the model ids, and which switches are on.
+const MODEL_ID = /^[\w./:@-]{1,160}$/;
+async function inventoryOf(listings) {
+  const values = envValues(envFiles);
+  const providers = await Promise.all(Object.entries(listings)
+    .filter(([name]) => values[name])
+    .map(async ([name, canonical]) => {
+      const url = listingUrl(name, canonical, values);
+      let host = "";
+      try { host = new URL(url).host; } catch { return null; }
+      try {
+        const res = await fetch(url, { headers: { authorization: `Bearer ${values[name]}` }, signal: AbortSignal.timeout(20_000) });
+        const body = res.ok ? await res.json().catch(() => null) : null;
+        const models = (Array.isArray(body?.data) ? body.data : [])
+          .map((m) => String(m?.id ?? "")).filter((id) => MODEL_ID.test(id)).slice(0, 1500);
+        return { env: name, host, status: res.status, models };
+      } catch {
+        return { env: name, host, status: 0, models: [] };
+      }
+    }));
+  return { providers: providers.filter(Boolean), flags: switchStates(values) };
+}
 let secrets = [];
 let identities = null;
 let capture = null;
@@ -208,7 +264,8 @@ const work = join(tmpdir(), `cortad-${process.pid}`);
 mkdirSync(work, { recursive: true });
 const bootLog = join(work, "boot.log");
 writeFileSync(bootLog, "");
-const door = openDoor(root);
+// Opened once the server says whose session this is.
+let door = null;
 // Files nobody may read through this program: keys, and git's own internals.
 const SECRET_PATH = /(?:^|\/)(?:\.git|\.ssh|\.gnupg|\.aws|\.npmrc|\.netrc|id_(?:rsa|ed25519|ecdsa)[^/]*|[^/]*\.(?:pem|key|p12|pfx|jks|keystore))(?:\/|$)/;
 // The engine's paths, as this machine has them. Its scratch files live in this program's own
@@ -277,13 +334,39 @@ async function verb(job) {
       // working its plan, and by you. For those seconds nothing is listening. A turn that meets a
       // closed door is held until your app answers again and sent then, once, instead of being
       // counted against your app as a failure it never had.
+      // A door behind a guest session answers the first request with a redirect that mints the
+      // session and sends the browser back. A browser reaches it by loading a page first; the same
+      // request pushed down that chain arrives at a page route as a POST and reads as "wrong
+      // method", which is what your chat looked like from the outside. So the chain is walked the
+      // way a browser walks it, one GET with redirects followed, and the request is sent again with
+      // the session your app just handed out. The cookie is kept here and never leaves this machine.
+      const url = `http://127.0.0.1:${port}${path}`;
+      const yours = (to) => { try { const u = new URL(to, url); return u.hostname === "127.0.0.1" && u.port === String(port) ? u.href : null; } catch { return null; } };
+      let jar = "";
+      const sent = (at, over = {}) => fetch(at, {
+        ...init, ...over,
+        headers: { ...headers, ...(over.headers ?? {}), ...(jar ? { cookie: [headers.cookie, jar].filter(Boolean).join("; ") } : {}) },
+        signal: AbortSignal.timeout(170_000),
+      });
+      const keep = (res) => { const set = res.headers.getSetCookie?.() ?? []; if (set.length) jar = [jar, ...set.map((c) => c.split(";")[0])].filter(Boolean).join("; "); };
+      const sentBack = (res) => (res.status >= 300 && res.status < 400 ? yours(res.headers.get("location") ?? "") : null);
+      const warm = async () => {
+        let at = url;
+        for (let hop = 0; hop < 4 && at; hop++) {
+          const res = await sent(at, { method: "GET", body: undefined });
+          keep(res);
+          at = sentBack(res);
+        }
+      };
       const ask = async () => {
-        const res = await fetch(`http://127.0.0.1:${port}${path}`, { ...init, signal: AbortSignal.timeout(170_000) });
+        let res = await sent(url);
+        keep(res);
+        if (method !== "GET" && sentBack(res)) { await warm(); if (jar) res = await sent(url); }
         const buf = Buffer.from(await res.arrayBuffer());
         const LIMIT = 262_144;
         // A session cookie your app sets is its business: it is dropped here, and every other header masked.
-        const headers = Object.fromEntries([...res.headers].filter(([k]) => !/^set-cookie2?$/i.test(k)).map(([k, v]) => [k, mask(v)]));
-        return { status: res.status, headers, body: mask(buf.subarray(0, LIMIT).toString("utf8")), truncated: buf.length > LIMIT };
+        const said = Object.fromEntries([...res.headers].filter(([k]) => !/^set-cookie2?$/i.test(k)).map(([k, v]) => [k, mask(v)]));
+        return { status: res.status, headers: said, body: mask(buf.subarray(0, LIMIT).toString("utf8")), truncated: buf.length > LIMIT };
       };
       try { return await ask(); }
       catch (e) {
@@ -316,8 +399,14 @@ async function verb(job) {
     case "restore": return door.restore(String(b.checkpoint ?? ""));
     case "keep": return door.keep(typeof b.checkpoint === "string" && b.checkpoint ? b.checkpoint : undefined);
     case "restart": return restartApp();
+    case "inventory": return inventoryOf(b.probe && typeof b.probe === "object" ? b.probe : {});
+    // Your own pages, read here rather than in a world's shell: that shell is sealed away from
+    // every env file and every host but this one, so inside it no store of yours has an address.
+    case "sample": return { report: JSON.parse(mask(JSON.stringify(await sampleHere(b, envValues(envFiles), root)))) };
     case "lock": return { locked: Array.isArray(b.hosts) ? b.hosts.length : 0 };
-    case "usage": return { totals: [], rows: [] };
+    // What your app spent on its providers, from the hook inside it. An app this command did not
+    // start has no hook, and the empty answer says the meter is absent rather than that nothing was spent.
+    case "usage": return capture?.usage() ?? {};
     // A world is ended from this terminal, never from the cloud.
     case "destroy": return { ok: true };
     default: return { ok: true };
@@ -345,15 +434,23 @@ let child = null;
 async function startApp() {
   const wanted = Number(flag("--port"));
   if (wanted) {
-    if (!(await answers(wanted))) fail(`nothing is answering on port ${wanted}. Start your app first, then run this again.`);
+    if (!(await answers(wanted))) return { port: null, said: "", why: `nothing is answering on port ${wanted}, so there is nothing to ask.`, noStart: true };
     const took = await takeOver(wanted);
     if (took) return took;
     say("your app was already running, so its request limits stay as they are; if it answers 429, stop it and run this without --port and they are raised for the session");
     return { port: wanted, cmd: null };
   }
   const plan = startPlan({ root, typed: flag("--start"), onPath });
-  const cmd = plan?.cmd ?? await ask("How do you start your app? (for example: npm run dev) ");
-  if (!cmd) fail("tell me how your app starts: --start \"npm run dev\", or --port 3000 if it is already running.");
+  // A package is not a broken app, and asking its owner how it starts is the wrong question.
+  const cmd = plan?.cmd ?? (plan?.noServer ? null : await ask("How do you start your app? (for example: npm run dev) "));
+  // Your code is already here and already being read: quitting now would throw away what you have
+  // paid for. This stays connected, the read finishes, and the browser says what is missing.
+  if (!cmd) {
+    return { port: null, said: "", noStart: true,
+      why: plan?.noServer
+        ? "this repository has no server to run; connect the app that serves it. If it does have one, run this again with --start \"how you start it\"."
+        : "your code is being read, but I could not work out how your app starts, so nothing is running to ask. Run the command again with --start \"how you start it\", or start your app yourself and run it again with --port <the port it answers on>." };
+  }
   appDir = plan?.cwd ?? root;
   pinned = pinnedNode();
   if (plan?.within) say(`your app is in ${plan.within}, started there with: ${cmd}`);
@@ -362,7 +459,7 @@ async function startApp() {
   launched = { cmd, lifted };
   step(`starting your app: ${cmd}`);
   const up = await launch(180_000);
-  if (up.port) return { port: up.port, cmd, lifted: Object.keys(lifted) };
+  if (up.port) return { port: up.port, cmd: launched.cmd, lifted: Object.keys(lifted) };
   // Already running: a second start dies on the port the first one holds. The one that is running
   // is the app, so it is used as it stands rather than treated as a failure.
   if (up.exited !== null && /EADDRINUSE|address already in use|port.{0,40}(?:in use|already used|is taken|unavailable)/i.test(up.tail)) {
@@ -408,9 +505,9 @@ async function takeOver(port) {
   launched = { cmd: plan.cmd, lifted };
   step(`starting your app: ${plan.cmd}`);
   const up = await launch(180_000);
-  if (up.port) return { port: up.port, cmd: plan.cmd, lifted: names };
+  if (up.port) return { port: up.port, cmd: launched.cmd, lifted: names };
   if (up.tail) console.error(up.tail);
-  fail("your app did not come back after the restart. Start it yourself, then run this again with --port.");
+  return { port: null, said: up.tail, why: "your app did not come back after the restart." };
 }
 // The process listening on a port.
 async function listenerOn(port) {
@@ -448,7 +545,56 @@ function pinnedNode() {
 }
 let pinned = { major: null, bin: null };
 
+// Their app could not start because its dependencies are not on this machine. That is an install
+// nobody ran, not a broken app: it is installed once, with the manager the project locked, and the
+// app is started again. Every way the app is started comes through here, so it is fixed in one place.
 async function launch(waitMs) {
+  const up = await start(waitMs);
+  // Only an app that stopped: one still running has not failed to start, whatever it printed.
+  if (up.port || up.exited === null) return up;
+  const ok = await installOnce(up.tail);
+  // An install that failed is the reason their app cannot start, and it goes where the app's own
+  // output goes: the terminal, and the screen that is waiting for the app.
+  const said = (tail) => (installSaid ? `${installSaid}\n${tail}` : tail);
+  if (!ok) return installSaid ? { ...up, tail: said(up.tail) } : up;
+  const back = await start(waitMs);
+  return back.port ? back : { ...back, tail: said(back.tail) };
+}
+let installed = false;
+let installSaid = "";
+async function installOnce(said) {
+  if (installed) return false;
+  const name = missingDependency(said);
+  const cmd = name && installPlan(appDir, onPath);
+  if (!cmd) return false;
+  installed = true;
+  stepDone(`your app needs ${name}, which is not installed here`);
+  const started = Date.now();
+  const secondsSoFar = () => Math.round((Date.now() - started) / 1000);
+  // A line that rewrites itself in a terminal, and every tenth second where there is none.
+  const tick = () => { const n = secondsSoFar(); const line = `installing your app's dependencies, ${n} seconds so far`; if (process.stdout.isTTY) step(line); else if (n % 10 === 0) say(line); };
+  tick();
+  const ticking = setInterval(tick, 1000);
+  const done = await new Promise((r) => {
+    const child = spawn("/bin/sh", ["-c", cmd], { cwd: appDir, env: { ...process.env, CI: "1" }, stdio: ["ignore", "pipe", "pipe"] });
+    let tail = "";
+    const keep = (d) => { tail = (tail + d.toString()).slice(-4000); appendFileSync(bootLog, d.toString()); if (verbose) process.stdout.write(d.toString()); };
+    child.stdout.on("data", keep);
+    child.stderr.on("data", keep);
+    const timer = setTimeout(() => child.kill("SIGKILL"), 15 * 60_000);
+    child.on("close", (code) => { clearTimeout(timer); r({ ok: code === 0, tail }); });
+    child.on("error", (e) => { clearTimeout(timer); r({ ok: false, tail: String(e.message) }); });
+  });
+  clearInterval(ticking);
+  if (!done.ok) { installSaid = `installing your app's dependencies failed after ${secondsSoFar()} seconds:\n${done.tail.split("\n").filter(Boolean).slice(-12).join("\n")}`; lastSaid = done.tail; stepDone(`installing your app's dependencies failed after ${secondsSoFar()} seconds`); return false; }
+  stepDone(`installed your app's dependencies in ${secondsSoFar()} seconds`);
+  // A Python project that had no interpreter of its own has one now, and it is the one to start with.
+  const again = startPlan({ root, typed: flag("--start"), onPath });
+  if (again?.cmd && again.cmd !== launched.cmd) { launched = { ...launched, cmd: again.cmd }; say(`starting your app: ${again.cmd}`); }
+  return true;
+}
+
+async function start(waitMs) {
   const { cmd, lifted } = launched;
   child = spawn("/bin/sh", ["-c", cmd], { cwd: appDir, env: { ...process.env, ...lifted, ...(capture ? capture.env(process.env) : {}), FORCE_COLOR: "0", ...(pinned.bin ? { PATH: `${pinned.bin}:${process.env.PATH ?? ""}` } : {}) }, stdio: ["ignore", "pipe", "pipe"], detached: true });
   const mine = child;
@@ -544,13 +690,13 @@ if (explain) {
   console.log([
     `cortad --explain  (nothing is sent or started by this)`,
     ``,
-    `talks to        ${origin.origin}  and  localhost (your app's port only)`,
+    `talks to        ${origin.origin}, localhost (your app's port only), and your own model providers, to ask each which models your key can use`,
     `would send      ${files.length} source files, ${Math.round(bytes / 1024)} KB, once${listed ? " (what git would commit)" : ""}`,
     `not sent        anything git ignores, env files (${envFiles.length} here: ${envFiles.slice(0, 6).map(rel).join(", ") || "none"}), key files, data files, node_modules, .git`,
-    `env files       read here only: to hide their values in replies, and to sign in a test account`,
-    `would start     ${flag("--port") ? `nothing: uses your app on port ${flag("--port")}` : plan ? `${plan.cmd}   (in ${rel(plan.cwd)})` : "asks you how your app starts"}`,
+    `env files       values read here only: to hide them in replies, to sign in a test account, and to ask your providers what your keys reach. Variable names and whether a switch is on or off go up; no value does`,
+    `would start     ${flag("--port") ? `nothing: uses your app on port ${flag("--port")}` : plan?.cmd ? `${plan.cmd}   (in ${rel(plan.cwd)})` : plan?.noServer ? "nothing: this repository has no server to run" : "asks you how your app starts"}`,
     `would raise     ${flag("--port") ? "nothing: your app's own request limits stay as they are" : `${Object.keys(liftedLimits(envFiles, files.map((f) => join(root, f)))).join(", ") || "no request limits found"}   (for this session only)`}`,
-    `loads into app  lib/trace.cjs (Node) or lib/pyhook/sitecustomize.py (Python): records the one request during which your app calls a model`,
+    `loads into app  lib/trace.cjs (Node, Bun) or lib/pyhook/sitecustomize.py (Python): records the one request during which your app calls a model`,
     `agent edits     in your files, each with an undo kept in ~/.cortad/checkpoints; git is never touched`,
     `agent shell     confined by the OS: your project and toolchains only, writes to temp and build folders, localhost only`,
     ``,
@@ -577,6 +723,8 @@ const attach = await call("POST", "/local/attach", { code, name: basename(root),
 if (!attach.ok) fail(attach.data?.error ?? `could not sign in (${attach.status})`);
 box = attach.data.box;
 key = attach.data.key;
+// A server that names no journal gets one for this session alone: never another account's edits.
+door = openDoor(root, { owner: typeof attach.data.journal === "string" && attach.data.journal ? attach.data.journal : box });
 
 const list = join(work, "files.txt");
 writeFileSync(list, files.join("\n") + "\n");
@@ -592,6 +740,15 @@ const treeDigest = (() => {
   }
   return h.digest("hex");
 })();
+// The commit this tree is. Your .git is never uploaded -- nothing of your history leaves this
+// machine -- so the one sha the report needs is read here and sent as forty characters. A folder
+// that is not a checkout sends nothing, and the report says so rather than inventing one.
+const head = (() => {
+  try {
+    const sha = execFileSync("git", ["-C", root, "rev-parse", "HEAD"], { stdio: ["ignore", "pipe", "ignore"] }).toString().trim();
+    return /^[a-f0-9]{40}$/.test(sha) ? sha : "";
+  } catch { return ""; }
+})();
 const archive = join(work, "tree.tgz");
 step("connecting");
 await exec("tar", ["-czf", archive, "-C", root, "-T", list]);
@@ -602,7 +759,7 @@ const parts = Math.max(1, Math.ceil(bytes.length / PART));
 for (let off = 0; off < bytes.length; off += PART) {
   const last = off + PART >= bytes.length;
   step(parts > 1 ? `connecting, ${Math.floor(off / PART) + 1} of ${parts}` : "connecting");
-  const put = await call("PUT", `/local/${box}/tree?last=${last ? 1 : 0}${last ? `&digest=${treeDigest}` : ""}`, bytes.subarray(off, off + PART), { raw: true, timeoutMs: 120_000 });
+  const put = await call("PUT", `/local/${box}/tree?last=${last ? 1 : 0}${last ? `&digest=${treeDigest}${head ? `&head=${head}` : ""}` : ""}`, bytes.subarray(off, off + PART), { raw: true, timeoutMs: 120_000 });
   if (!put.ok) fail(put.data?.error ?? `upload failed (${put.status})`);
   if (last) resumed = put.data?.resumed === true;
 }
@@ -632,7 +789,7 @@ function sourceChanged() {
 }
 // `watching` says whether a message sent to this app can be seen arriving: only an app this command
 // started carries the hook, and only a runtime the hook exists for.
-const announce = () => call("POST", `/local/${box}/app`, { port: app.port, cmd: app.cmd, origins: envOrigins(envFiles), lifted: app.lifted ?? [], watching: Boolean(launched && capture?.alive()) });
+const announce = () => call("POST", `/local/${box}/app`, { port: app.port, cmd: app.cmd, origins: envOrigins(envFiles), lifted: app.lifted ?? [], watching: Boolean(launched && capture?.watching(app.port)), metered: Boolean(launched && capture?.watching(app.port)) });
 
 // Your app's life beside this connection. It is started; if it stops, or never comes up, this stays
 // and starts it again the moment you save a fix, and the browser is told each time it answers, so a
@@ -641,8 +798,8 @@ async function appLife() {
   for (let first = true; ; first = false) {
     const got = await startApp();
     if (got.port) { app = got; break; }
-    await call("POST", `/local/${box}/stopped`, { said: mask(got.said ?? "").slice(-2000) }).catch(() => {});
-    say(`${got.why} Fix it and save: it is started again by itself.`);
+    await call("POST", `/local/${box}/stopped`, { said: mask(got.said || got.why).slice(-2000) }).catch(() => {});
+    say(got.noStart ? got.why : `${got.why} Fix it and save: it is started again by itself.`);
     if (first) say("leave this open. Ctrl-C disconnects.");
     await sourceChanged();
     say("saw your change, starting your app again");
@@ -711,7 +868,7 @@ async function close(code = 0) {
   closing = true;
   await call("DELETE", `/local/${box}`).catch(() => {});
   if (child?.pid) await stopApp(child.pid);
-  const pending = door.pending();
+  const pending = door?.pending() ?? 0;
   if (pending) say(`${pending} agent edit${pending === 1 ? " is" : "s are"} waiting for you to keep or undo. Run the command again to review ${pending === 1 ? "it" : "them"} in the browser.`);
   await exec("rm", ["-rf", work]).catch(() => {});
   process.exit(code);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cortad",
-  "version": "0.1.6",
+  "version": "0.1.8",
   "description": "Connects the AI app on your machine to Cortad for test conversations. No dependencies.",
   "bin": {
     "cortad": "local.mjs"
@@ -14,6 +14,7 @@
     "lib/mint.mjs",
     "lib/pyhook/sitecustomize.py",
     "lib/replay.mjs",
+    "lib/sample.mjs",
     "lib/start.mjs",
     "lib/trace.cjs",
     "local.mjs"