cortad 0.3.1-rc.9 → 0.3.2

package/lib/proxy.mjs

@@ -0,0 +1,184 @@
+// For an app the hook cannot load into (Go, Ruby, Java, Rust, PHP, Deno, or a Node or Python app
+// that would not take it): what the hook sees from inside the process is seen from outside it, on
+// this machine. Two servers on 127.0.0.1. The front sits before the app's own port: the agent's
+// request and every conversation of a run go through it, and it writes the exchange rows the hook
+// writes. The model proxy is where the app's model settings point (OPENAI_BASE_URL and the others):
+// it hands each call to the provider the app's own settings name and writes the call rows the hook
+// writes. Both write into the same trace file, so the command reads them as it reads the hook's
+// (lib/replay.mjs). The line of code that made a call is not known here. Header values are never
+// written, and nothing goes anywhere the app would not have sent it.
+import { appendFileSync } from "node:fs";
+import { createServer, request } from "node:http";
+import { connect } from "node:net";
+import { once } from "node:events";
+import wire from "./wire.cjs";
+import tied from "./tied.cjs";
+
+const { MAX, REPLY_MAX, MODEL_HOST, scrubbed, turnOf, decoded, cookieNames, makeRecorder } = wire;
+
+// Where each SDK sends its calls when the app's settings name nowhere else.
+const DEFAULTS = { OPENAI_BASE_URL: "https://api.openai.com/v1", ANTHROPIC_BASE_URL: "https://api.anthropic.com", GOOGLE_GEMINI_BASE_URL: "https://generativelanguage.googleapis.com" };
+// Another setting of the app's that addresses a model: a URL on a model host, or a base URL named for one.
+const MODEL_SETTING = /OPENAI|ANTHROPIC|GEMINI|LLM|OLLAMA|LITELLM|VLLM/i;
+const ADDRESS_SETTING = /BASE|URL|ENDPOINT|HOST/i;
+const SECRET_SETTING = /KEY|TOKEN|SECRET|PASSWORD/i;
+// Said by one connection, never passed on. A provider's reply is passed on decoded, so its encoding goes too.
+const HOP = /^(?:host|connection|keep-alive|transfer-encoding|upgrade|te|trailer|proxy-.*|content-length|accept-encoding)$/i;
+const REPLY_HOP = /^(?:connection|keep-alive|transfer-encoding|content-length|content-encoding)$/i;
+
+// The person's words in a request, a form's fields as much as JSON's.
+const asked = (raw) => (/^[^=&\s]+=[^&]*(?:&[^=&\s]+=[^&]*)*$/.test(raw) ? [...new URLSearchParams(raw).values()].flatMap(tied.sentencesIn) : tied.sentencesIn(raw));
+// A request whose reply said something of its own, to words the person sent, with no model call begun
+// this long after it: an app that asks its model after the page answers (a job Rails or Celery runs)
+// does so well within it.
+// ponytail: a worker slower than this gets a request said unseen; the wait per app if one is seen.
+const UNSEEN_MS = 3000;
+
+const listen = async (server) => { server.listen(0, "127.0.0.1"); await once(server, "listening"); return server.address().port; };
+
+// `file`: the trace file the command reads. `target()`: where the app answers now, { host, port };
+// it moves when the app is started again on another port. `onSeen`: the first model call came through.
+// `onUnseen(door)`: a request the app answered in words with no model call begun, said once, as "POST /chat".
+export async function openProxy({ file, rulesFile, target, onSeen = () => {}, onUnseen = () => {} }) {
+  const row = (value) => { try { appendFileSync(file, JSON.stringify(scrubbed(value)) + "\n", { mode: 0o600 }); } catch { /* the command is gone */ } };
+  const rec = makeRecorder({ row, rulesFile });
+  // The requests in flight through the front that a model call can be made for.
+  const open = new Set();
+  // Model calls begun: a call is counted as it arrives, before the app's reply that it feeds can end.
+  let calls = 0, unseen = false;
+  const unseenIf = (method, path, sent, reply) => {
+    if (calls || unseen || !(reply.status >= 200 && reply.status < 300)) return;
+    const words = asked(sent);
+    if (!words.length || !tied.sentencesIn(reply.text).some((w) => !words.includes(w))) return;
+    setTimeout(() => { if (!calls && !unseen) { unseen = true; onUnseen(`${method} ${path.split("?")[0]}`); } }, UNSEEN_MS).unref();
+  };
+
+  const front = createServer((req, res) => {
+    const { "x-cortad-turn": _turn, ...headers } = req.headers;
+    const ctx = /^(?:HEAD|OPTIONS)$/.test(req.method) ? null : rec.requestCtx(req.method, req.url || "/", headers, turnOf(req.headers));
+    if (ctx && req.method !== "GET") open.add(ctx);
+    const finish = (reply) => {
+      if (!ctx || ctx.ended) return;
+      open.delete(ctx);
+      rec.ended(ctx, reply);
+      if (req.method !== "GET") unseenIf(req.method, ctx.path, Buffer.concat(ctx.chunks).toString("utf8"), reply);
+    };
+    const { host, port } = target();
+    // The Host the caller used goes on unchanged: an app that writes its own address from it (a Rails
+    // redirect to sign in) then sends the caller back here, where the request is seen.
+    const up = request({ host, port, method: req.method, path: req.url, headers });
+    if (ctx && req.method !== "GET") req.on("data", (c) => { if (ctx.size < MAX) { ctx.chunks.push(c); ctx.size += c.length; } });
+    req.pipe(up);
+    res.on("close", () => up.destroy());
+    // Nothing answered: the caller sees the connection close, as it would at the app's own port.
+    up.on("error", () => { finish({ status: 0, type: "", text: "", cookies: [], cut: true }); res.destroy(); });
+    up.on("response", (r) => {
+      const got = [];
+      let size = 0, writes = 0;
+      r.on("data", (c) => { writes += 1; if (size < MAX) { got.push(c); size += c.length; } });
+      const text = () => decoded(Buffer.concat(got), r.headers["content-encoding"]);
+      r.on("end", () => { if (ctx) rec.tied.replied(ctx, r.statusCode, text); });
+      r.on("error", () => res.destroy());
+      const done = () => finish({ status: r.statusCode, type: String(r.headers["content-type"] || ""), writes: r.headers["content-length"] ? 1 : writes, text: text(), cookies: cookieNames(r.headers["set-cookie"]), cut: !res.writableFinished });
+      res.once("finish", done);
+      res.once("close", done);
+      res.writeHead(r.statusCode, r.statusMessage, r.rawHeaders);
+      r.pipe(res);
+    });
+  });
+  // A socket the page opens (live reload, a chat over a WebSocket) is passed through as it is, unrecorded.
+  front.on("upgrade", (req, socket, head) => {
+    const { host, port } = target();
+    const up = connect({ host, port }, () => {
+      const lines = [];
+      for (let i = 0; i < req.rawHeaders.length; i += 2) lines.push(`${req.rawHeaders[i]}: ${req.rawHeaders[i + 1]}`);
+      up.write(`${req.method} ${req.url} HTTP/${req.httpVersion}\r\n${lines.join("\r\n")}\r\n\r\n`);
+      if (head?.length) up.write(head);
+      socket.pipe(up).pipe(socket);
+    });
+    const end = () => { socket.destroy(); up.destroy(); };
+    up.on("error", end);
+    socket.on("error", end);
+  });
+
+  // Each upstream the app's settings name, by its place: /u/<n>/<the SDK's own path>.
+  const slots = [];
+  const relay = async (req, res) => {
+    const m = /^\/u\/(\d+)([/?][^]*)?$/.exec(req.url || "");
+    const base = m && slots[Number(m[1])];
+    if (!base) { res.writeHead(404, { "content-type": "application/json" }); res.end('{"error":{"message":"cortad: no model setting points here"}}'); return; }
+    // A listing of models answers nobody; a call that sends a prompt does.
+    const metered = !/^(?:GET|HEAD)$/.test(req.method);
+    if (metered && ++calls === 1) onSeen();
+    const parts = [];
+    for await (const c of req) parts.push(c);
+    const body = Buffer.concat(parts);
+    // Read as chat when the AI SDK sent it in its own format; the provider still gets `body` as sent.
+    const sent = wire.asChat(body.toString("utf8"), wire.GATEWAY_MODEL.map((h) => req.headers[h]).find(Boolean));
+    const url = new URL(base.replace(/\/+$/, "") + (m[2] ?? ""));
+    // ponytail: a call is pinned by the person's words, else to the one request open. Two at once that
+    // open with the same or short words ("refund?") leave both calls unpinned and their exchanges
+    // unwritten, and a chat sent as a GET (EventSource) is never pinned; never pinned wrong. The
+    // request's own context, as the hook has it, is what tells them apart.
+    const ctx = metered ? rec.note(undefined, sent, open.size === 1 ? [...open][0] : undefined) : undefined;
+    const t0 = Date.now();
+    const meter = (status, type, text) => { if (metered) rec.meter({ host: url.host, path: url.pathname, sent, status, type, body: text, ctx, t0 }); };
+    // The app stopped reading: the provider stops too, so the app's tokens stop with it.
+    const gone = new AbortController();
+    res.on("close", () => { if (!res.writableFinished) gone.abort(); });
+    let got;
+    try {
+      got = await fetch(url, { method: req.method, headers: Object.fromEntries(Object.entries(req.headers).filter(([k]) => !HOP.test(k))), body: metered && body.length ? body : undefined, redirect: "manual", signal: gone.signal });
+    } catch (e) {
+      meter(0, "", "");
+      if (gone.signal.aborted) return;
+      res.writeHead(502, { "content-type": "application/json" });
+      res.end(JSON.stringify({ error: { message: `cortad could not reach ${url.host}: ${e.cause?.code ?? e.message}` } }));
+      return;
+    }
+    if (ctx) ctx.heard = true;
+    const headers = {};
+    for (const [k, v] of got.headers) if (!REPLY_HOP.test(k) && k !== "set-cookie") headers[k] = v;
+    const cookies = got.headers.getSetCookie();
+    res.writeHead(got.status, cookies.length ? { ...headers, "set-cookie": cookies } : headers);
+    const kept = [];
+    let size = 0;
+    try {
+      for await (const c of got.body ?? []) {
+        res.write(c);
+        if (size < REPLY_MAX) { kept.push(Buffer.from(c)); size += c.length; }
+      }
+    } catch { /* broken off: what arrived stands */ }
+    res.end();
+    meter(got.status, got.headers.get("content-type") ?? "", Buffer.concat(kept).toString("utf8"));
+  };
+  // An app that hangs up while it sends a call ends that call, never this command.
+  const model = createServer((req, res) => { relay(req, res).catch(() => res.destroy()); });
+
+  const frontPort = await listen(front);
+  const modelPort = await listen(model);
+  row({ hello: "proxy", pid: process.pid });
+  const via = (upstream) => {
+    if (!slots.includes(upstream)) slots.push(upstream);
+    return `http://127.0.0.1:${modelPort}/u/${slots.indexOf(upstream)}`;
+  };
+  return {
+    port: frontPort,
+    modelPort,
+    // The app's model settings, pointed here: each SDK's own, and any other of the app's that addresses a model.
+    env(base) {
+      const out = {};
+      for (const [name, fallback] of Object.entries(DEFAULTS)) out[name] = via(base[name] || fallback);
+      for (const [name, value] of Object.entries(base)) {
+        if (name in out || SECRET_SETTING.test(name) || typeof value !== "string") continue;
+        let u;
+        try { u = new URL(value); } catch { continue; }
+        if (/^https?:$/.test(u.protocol) && (MODEL_HOST.test(u.hostname) || (MODEL_SETTING.test(name) && ADDRESS_SETTING.test(name)))) out[name] = via(value);
+      }
+      return out;
+    },
+    // Whether a model call has begun yet.
+    seen: () => calls > 0,
+    close: () => { for (const s of [front, model]) { s.closeAllConnections(); s.close(); } },
+  };
+}

package/lib/pyhook/cortad_sites.py

@@ -0,0 +1,218 @@
+# Where in the app's own code each part of a reply was written, so a block the reply carried is placed
+# at the line that sent it, not at the model call that ran beside it. A framework's send is never that
+# line: for a streamed reply it is where the generator the framework iterates stopped to hand the chunk
+# over, or, when that generator only passes on what the one it loops over gave it, where that one
+# stopped. One entry per line and event name, with the first bytes written there.
+import dis
+import functools
+import gc
+import re
+import sys
+import types
+
+SITES_MAX, TEXT_MAX = 16, 300
+# ponytail: a reply's first thousand writes are traced; a leak later in a longer stream goes unplaced.
+WRITES_MAX = 1000
+# A framework puts a few frames between its send and the loop that iterates the reply, and a few
+# generators between that loop and the app's own; never dozens.
+FRAMES_MAX, DEPTH_MAX, WIDTH_MAX, NODES_MAX = 24, 8, 8, 256
+GENERATORS = (types.GeneratorType, types.AsyncGeneratorType)
+# What an object that holds the reply's iterator is made of: a response's attributes, a partial.
+HOLDERS = (dict, list, tuple, functools.partial)
+SSE_EVENT = re.compile(rb"^event:[ \t]*([\w.:-]{1,40})", re.M)
+JSON_EVENT = re.compile(rb'"(?:type|event)"\s*:\s*"([\w.:-]{1,40})"')
+
+
+def _event(chunk):
+    head = chunk[:400]
+    m = SSE_EVENT.search(head) or JSON_EVENT.search(head)
+    return m.group(1).decode("ascii") if m else None
+
+
+def _frame(gen):
+    return getattr(gen, "ag_frame", None) or getattr(gen, "gi_frame", None)
+
+
+def _live(o):
+    return isinstance(o, GENERATORS) and _frame(o) is not None
+
+
+def _iterator(o):
+    return hasattr(type(o), "__next__") or hasattr(type(o), "__anext__")
+
+
+# What a suspended generator holds, its locals and then its stack: the generator's own referents from
+# Python 3.11, its frame's before that.
+def _referents(gen):
+    return gc.get_referents(gen) if sys.version_info >= (3, 11) else gc.get_referents(_frame(gen))
+
+
+# The suspended generators a generator holds in its locals and on its stack.
+def _under(gen):
+    try:
+        return [g for g in _referents(gen) if _live(g)]
+    except Exception:
+        return []
+
+
+def _same(v, chunk):
+    if isinstance(v, (bytes, bytearray)):
+        return v == chunk
+    return isinstance(v, str) and len(v) <= len(chunk) and v.encode("utf-8", "replace") == chunk
+
+
+def _holds(frame, chunk):
+    return any(_same(v, chunk) for v in frame.f_locals.values())
+
+
+# A send wrapper: the frame was called with the message, where the one that wrote it built it.
+def _handed(frame, message):
+    code = frame.f_code
+    return any(frame.f_locals.get(name) is message for name in code.co_varnames[:code.co_argcount + code.co_kwonlyargcount])
+
+
+# The local the last loop before `lasti` in `code` gives each item to: the first store after its header.
+@functools.lru_cache(maxsize=512)
+def _loop_target(code, lasti):
+    target, header = None, False
+    for ins in dis.get_instructions(code):
+        if ins.offset >= lasti:
+            break
+        if ins.opname in ("FOR_ITER", "GET_ANEXT"):
+            target, header = None, True
+        elif header and ins.opname.startswith("STORE_FAST"):
+            target, header = ins.argval[0] if isinstance(ins.argval, tuple) else ins.argval, False
+    return target
+
+
+# The generator that handed `gen` the chunk it just yielded: the one it delegates to with `yield from`,
+# else, when it yielded the item of the innermost loop it stopped in as the item came, that loop's
+# iterator, which the loop keeps on the stack, the innermost last. A chunk it made from the item is its own.
+def _fed_by(gen, chunk):
+    inner, frame = getattr(gen, "gi_yieldfrom", None), _frame(gen)
+    if inner is None and _same(frame.f_locals.get(_loop_target(frame.f_code, frame.f_lasti)), chunk):
+        mine = {id(v) for v in frame.f_locals.values()}
+        inner = next((o for o in reversed(_referents(gen)) if id(o) not in mine and _iterator(o)), None)
+    return inner if _live(inner) else None
+
+
+def _followed(o):
+    return isinstance(o, HOLDERS) or (hasattr(o, "__dict__") and type(o).__module__ != "builtins" and not isinstance(o, (type, types.ModuleType)))
+
+
+# The live generators reachable from `roots` through the objects that hold them, nearest first.
+def _generators(roots):
+    seen, level, found = set(), list(roots), []
+    while level and len(seen) < NODES_MAX and not found:
+        deeper = []
+        for o in level:
+            if id(o) in seen:
+                continue
+            seen.add(id(o))
+            if isinstance(o, GENERATORS):
+                if _frame(o) is not None:
+                    found.append(o)
+            elif _followed(o):
+                deeper.extend(gc.get_referents(o))
+        level = deeper
+    return found
+
+
+class Sites:
+    # `own_at(frame)` is "file:line" for a frame of the app's own code, else None.
+    def __init__(self, own_at):
+        self.own_at = own_at
+
+    # The app's line that gave `chunk` out of the generator `gen` the framework iterates: the first of
+    # the app's generators under it, then down the generators that only passed the chunk on.
+    def _made_by(self, gen, chunk):
+        level = [gen]
+        for _ in range(DEPTH_MAX):
+            own = next((g for g in level if _frame(g) and self.own_at(_frame(g))), None)
+            if own is not None:
+                return self._handed_down(own, chunk)
+            level = [u for g in level for u in _under(g)][:WIDTH_MAX]
+            if not level:
+                return None
+        return None
+
+    # Down the generators that only handed the chunk on, to the one that made it where it stopped.
+    def _handed_down(self, gen, chunk):
+        at = self.own_at(_frame(gen))
+        for _ in range(DEPTH_MAX):
+            gen = _fed_by(gen, chunk)
+            if gen is None:
+                break
+            at = self.own_at(_frame(gen)) or at
+        return at
+
+    # An ASGI body message on its way out, from `frame`, the frame that called send, up to `stop`.
+    def sent(self, req, message, frame, stop):
+        chunk = bytes(message.get("body") or b"")
+        if self._room(req, chunk):
+            self._note(req, chunk, self._sender(req, message, chunk, frame, stop))
+
+    # A chunk a response is sending out of `iterator`, the reply it was made with, as it sends it.
+    def made(self, req, iterator, chunk):
+        chunk = bytes(chunk)
+        if self._room(req, chunk):
+            self._note(req, chunk, next(filter(None, (self._made_by(g, chunk) for g in _generators([iterator]))), None))
+
+    # The app's line that wrote the message: past the send wrappers that were handed it, the frame that
+    # sent it, when that is the app's, or an app frame above that holds the bytes. Else the reply's
+    # iterator, held by the framework frame that iterates it, which holds the chunk it was given, looked
+    # for once. An app frame above the sender that holds neither, such as a middleware's
+    # `await self.app(scope, receive, send)`, only called into what wrote it.
+    def _sender(self, req, message, chunk, frame, stop):
+        gen = req.get("iterated")
+        if gen:
+            return self._made_by(gen, chunk)
+        sender = True
+        for _ in range(FRAMES_MAX):
+            if frame is None or frame.f_code is stop:
+                break
+            if not _handed(frame, message):
+                at, held = self.own_at(frame), _holds(frame, chunk)
+                if at and (sender or held):
+                    return at
+                sender = False
+                at = self._first_made(req, list(frame.f_locals.values()), chunk) if gen is None and held else None
+                if at:
+                    return at
+            frame = frame.f_back
+        if gen is None:
+            req["iterated"] = False
+        return None
+
+    # The line the app made `chunk` at, in the first generator reachable from a frame's `values` that
+    # made it, which is kept as the reply's iterator.
+    def _first_made(self, req, values, chunk):
+        for g in _generators(values):
+            at = self._made_by(g, chunk)
+            if at:
+                req["iterated"] = g
+                return at
+        return None
+
+    # A WSGI reply's next item, out of the iterable the app returned.
+    def iterated(self, req, iterable, chunk):
+        chunk = bytes(chunk)
+        if not self._room(req, chunk):
+            return
+        gen = req.get("iterated")
+        if gen is None:
+            gen = req["iterated"] = next(iter(_generators([iterable])), False)
+        self._note(req, chunk, self._made_by(gen, chunk) if gen else None)
+
+    def _room(self, req, chunk):
+        req["traced"] = req.get("traced", 0) + 1
+        return bool(chunk) and req["traced"] <= WRITES_MAX and len(req.setdefault("sites", [])) < SITES_MAX
+
+    def _note(self, req, chunk, at):
+        if not at:
+            return
+        event = _event(chunk)
+        sites = req["sites"]
+        if any(s["at"] == at and s.get("event") == event for s in sites):
+            return
+        sites.append({"at": at, **({"event": event} if event else {}), "text": chunk[:TEXT_MAX].decode("utf-8", "replace")})