cortad 0.3.0-rc.7 → 0.3.0-rc.8

package/lib/pyhook/sitecustomize.py

@@ -128,6 +128,50 @@ def _install():
                     add(p["functionResponse"].get("name"), p["functionResponse"].get("response"))
         return out or None
 
+    # A framework that folds its tools' answers into the next prompt as plain text sends no tool
+    # message to find: within one turn, what a later call's prompt carries that the call before did
+    # not, outside the model's own words, is that material. The first call holds the customer's
+    # message and the system prompt, present in every later call, so neither is ever counted.
+    seen_by_turn = {}
+
+    def texts_of(body):
+        out = set()
+
+        def take(v):
+            t = text_of(v).strip()
+            if t:
+                out.add(t[:tool_text])
+        for m in body.get("messages") if isinstance(body.get("messages"), list) else []:
+            if isinstance(m, dict) and m.get("role") != "assistant":
+                take(m.get("content"))
+        for it in body.get("input") if isinstance(body.get("input"), list) else []:
+            if isinstance(it, dict) and it.get("role") != "assistant" and it.get("type") != "function_call":
+                take(it.get("content") if it.get("content") is not None else it.get("output"))
+        if isinstance(body.get("system"), str):
+            take(body["system"])
+        for c in body.get("contents") if isinstance(body.get("contents"), list) else []:
+            if isinstance(c, dict) and c.get("role") != "model":
+                for p in c.get("parts") if isinstance(c.get("parts"), list) else []:
+                    if isinstance(p, dict) and isinstance(p.get("text"), str):
+                        take(p["text"])
+        return out
+
+    def material_in(sent, turn):
+        if not turn:
+            return None
+        body = parsed(sent) if isinstance(sent, str) else None
+        if not isinstance(body, dict):
+            return None
+        now = texts_of(body)
+        before = seen_by_turn.get(turn)
+        seen_by_turn[turn] = now
+        if len(seen_by_turn) > 200:
+            seen_by_turn.pop(next(iter(seen_by_turn)))
+        if before is None:
+            return None
+        fresh = [{"name": "", "text": t} for t in now if t not in before][:tools_max]
+        return fresh or None
+
     def rules_in(sent):
         found = rules_now()
         if found is None:
@@ -276,7 +320,7 @@ def _install():
             found = rules_in(as_text)
             if found is not None:
                 row["rules"] = found
-            tools = tools_in(as_text)
+            tools = tools_in(as_text) or material_in(as_text, turn)
             if tools:
                 row["tools"] = tools
             write({"call": row})
@@ -382,16 +426,154 @@ def _install():
         wrapped._brainsless = True
         return wrapped
 
+    # The app's own route table, read off the app object the server was handed, so the run knows
+    # every door the app has before a message is sent. Read only: nothing here changes the app.
+    route_max, openapi_max = 400, 8 * 1024 * 1024
+    verbs = ("GET", "POST", "PUT", "PATCH", "DELETE")
+    served_port = [None]
+
+    def template(path):
+        # One syntax for every framework's path parameters: OpenAPI's {name}.
+        p = re.sub(r"\(\?P<(\w+)>(?:[^()]|\([^()]*\))*\)", r"{\1}", str(path))
+        p = re.sub(r"<(?:[^<>:]+:)?(\w+)>", r"{\1}", p)
+        p = re.sub(r"\{(\w+):[^{}]*\}", r"{\1}", p)
+        p = re.sub(r"(?:\$|\\Z)$", "", re.sub(r"^\^", "", p))
+        return p.replace("\\.", ".").replace("/?", "/")
+
+    def source_of(fn):
+        import inspect
+        try:
+            fn = inspect.unwrap(fn)
+        except Exception:
+            pass
+        code = getattr(fn, "__code__", None)
+        path = code.co_filename if code else getattr(sys.modules.get(getattr(fn, "__module__", None) or ""), "__file__", None) or ""
+        rel = os.path.relpath(path, os.getcwd()) if path else ""
+        theirs = rel and not rel.startswith("..") and "site-packages" not in rel and "dist-packages" not in rel
+        return (rel if theirs else ""), str(getattr(fn, "__name__", "") or "")
+
+    def add(out, methods, path, fn):
+        # Methods unknown: the two a door is asked with.
+        file, handler = source_of(fn) if fn is not None else ("", "")
+        for m in ("POST", "GET") if methods is None else methods:
+            m = str(m).upper()
+            if m not in ("HEAD", "OPTIONS") and len(out) < route_max:
+                out.append({"method": m, "path": ("/" + template(path).lstrip("/"))[:1024], "file": file[:512], "handler": handler[:200]})
+
+    def class_methods(cls):
+        return [m for m in verbs if callable(getattr(cls, m.lower(), None))]
+
+    def starlette_routes(routes, prefix, out):
+        for r in routes or []:
+            path = prefix + (getattr(r, "path", "") or "")
+            sub = getattr(r, "routes", None)
+            # FastAPI 0.14x keeps an included router whole and resolves its routes on demand.
+            if callable(getattr(r, "effective_route_contexts", None)):
+                for c in r.effective_route_contexts():
+                    if c.starlette_route is not None:
+                        starlette_routes([c.starlette_route], prefix, out)
+                    else:
+                        add(out, c.methods, prefix + c.path, c.endpoint)
+            elif sub is not None:
+                starlette_routes(sub, path, out)
+            elif getattr(r, "endpoint", None) is not None and "WebSocket" not in type(r).__name__:
+                fn = r.endpoint
+                add(out, getattr(r, "methods", None) or (class_methods(fn) if isinstance(fn, type) else None), path, fn)
+
+    def flask_routes(app, out):
+        for rule in app.url_map.iter_rules():
+            if rule.endpoint != "static" and not rule.endpoint.endswith(".static"):
+                add(out, rule.methods, rule.rule, app.view_functions.get(rule.endpoint))
+
+    def django_routes(_app, out):
+        from django.urls import URLResolver, get_resolver
+
+        def walk(patterns, prefix):
+            for p in patterns:
+                path = prefix + template(str(p.pattern))
+                if isinstance(p, URLResolver):
+                    walk(p.url_patterns, path)
+                    continue
+                cb = p.callback
+                actions = getattr(cb, "actions", None)
+                cls = getattr(cb, "cls", None) or getattr(cb, "view_class", None)
+                add(out, list(actions) if isinstance(actions, dict) else class_methods(cls) if cls else None, path, cls or cb)
+
+        walk(get_resolver().url_patterns, "/")
+
+    def litestar_routes(app, out):
+        for r in app.routes:
+            for h in getattr(r, "route_handlers", None) or []:
+                methods = [m for m in getattr(h, "http_methods", ()) if str(m).upper() in verbs]
+                if methods:
+                    add(out, methods, r.path, getattr(h, "fn", None))
+
+    def aiohttp_routes(app, out):
+        for r in app.router.routes():
+            add(out, None if r.method == "*" else [r.method], r.resource.canonical if r.resource else "", r.handler)
+
+    def asgi_routes(app, out):
+        starlette_routes(app.routes, "", out)
+
+    walkers = (("fastapi", "routes", asgi_routes), ("starlette", "routes", asgi_routes),
+               ("quart", "url_map", flask_routes), ("flask", "url_map", flask_routes),
+               ("litestar", "routes", litestar_routes), ("aiohttp", "router", aiohttp_routes),
+               ("django", None, django_routes))
+
+    def app_in(obj):
+        # Servers hand the app over inside their own middleware, and each layer keeps the next.
+        for _ in range(12):
+            if obj is None:
+                break
+            tops = {c.__module__.split(".")[0] for c in type(obj).__mro__}
+            for name, attr, walk in walkers:
+                if name in tops and (attr is None or hasattr(obj, attr)):
+                    return name, obj, walk
+            inner = getattr(obj, "app", None)
+            obj = inner if inner is not None else getattr(obj, "application", None)
+        return None, None, None
+
+    def openapi_of(name, app):
+        try:
+            if name == "fastapi":
+                # openapi() caches what it builds; the app's own cache is put back as it was.
+                kept = app.openapi_schema
+                try:
+                    doc = app.openapi()
+                finally:
+                    app.openapi_schema = kept
+            elif name == "litestar":
+                doc = app.openapi_schema.to_schema()
+            else:
+                return None
+            return doc if isinstance(doc, dict) and len(json.dumps(doc)) <= openapi_max else None
+        except Exception:
+            return None
+
+    def publish_routes(app, port):
+        name, found, out = None, None, []
+        try:
+            name, found, walk = app_in(app)
+            if walk:
+                walk(found, out)
+        except Exception:
+            pass
+        port = port if isinstance(port, int) and not isinstance(port, bool) else None
+        write({"routes": {"framework": name or "unknown", "port": port, "routes": out, "openapi": openapi_of(name, found)}})
+
     # What is patched, once the module it lives in has been imported by the app itself.
     def patch_uvicorn(module):
         load = module.Config.load
 
         def loaded(self, *a, **k):
             out = load(self, *a, **k)
+            # Under gunicorn, uvicorn's own port is its default and the socket is gunicorn's.
+            port = served_port[0] or getattr(self, "port", None)
+            publish_routes(self.loaded_app, port)
             if not isinstance(self.loaded_app, Asgi):
                 self.loaded_app = Asgi(self.loaded_app)
-            if isinstance(getattr(self, "port", None), int):
-                write({"listen": self.port, "pid": os.getpid()})
+            if isinstance(port, int):
+                write({"listen": port, "pid": os.getpid()})
             return out
 
         module.Config.load = loaded
@@ -402,6 +584,7 @@ def _install():
         def run(hostname, port, application, *a, **k):
             if isinstance(port, int):
                 write({"listen": port, "pid": os.getpid()})
+            publish_routes(application, port)
             return run_simple(hostname, port, wsgi(application), *a, **k)
 
         module.run_simple = run
@@ -414,6 +597,54 @@ def _install():
 
         module.WSGIHandler.__call__ = called
 
+    def patch_django_server(module):
+        run = module.run
+
+        def running(addr, port, wsgi_handler, *a, **k):
+            publish_routes(wsgi_handler, port)
+            return run(addr, port, wsgi_handler, *a, **k)
+
+        module.run = running
+
+    def patch_gunicorn(module):
+        load = module.Worker.load_wsgi
+
+        def loaded(self):
+            out = load(self)
+            try:
+                bound = [s.getsockname() for s in self.sockets]
+                served_port[0] = next((at[1] for at in bound if isinstance(at, tuple)), None)
+            except Exception:
+                pass
+            publish_routes(self.wsgi, served_port[0])
+            return out
+
+        module.Worker.load_wsgi = loaded
+
+    def patch_hypercorn(module):
+        serve = module.worker_serve
+
+        def serving(app, config, *a, **k):
+            port = None
+            try:
+                port = int(str(config.bind[0]).rsplit(":", 1)[1])
+            except Exception:
+                pass
+            publish_routes(app, port)
+            return serve(app, config, *a, **k)
+
+        module.worker_serve = serving
+
+    def patch_aiohttp_web(module):
+        run_app = module.run_app
+
+        def run(app, *a, **k):
+            if isinstance(app, module.Application):
+                publish_routes(app, k.get("port") or 8080)
+            return run_app(app, *a, **k)
+
+        module.run_app = run
+
     def sent_of(request):
         try:
             return text(request.content)
@@ -593,6 +824,8 @@ def _install():
         module.ClientResponse.release = release_kept
 
     exact = {"uvicorn.config": patch_uvicorn, "werkzeug.serving": patch_werkzeug, "django.core.handlers.wsgi": patch_django,
+             "django.core.servers.basehttp": patch_django_server, "gunicorn.workers.base": patch_gunicorn,
+             "hypercorn.asyncio.run": patch_hypercorn, "hypercorn.trio.run": patch_hypercorn, "aiohttp.web": patch_aiohttp_web,
              "requests.sessions": patch_requests, "aiohttp.client": patch_aiohttp}
     # By family, not by name: the OpenAI SDK moved to a renamed copy of httpx (httpx2), and a patch
     # keyed on "httpx" alone saw none of its calls.

package/lib/replay.mjs

@@ -21,6 +21,9 @@ export function makeCapture({ work, keepSecret, onDoor }) {
   // The ports a hooked process listens on. Loading is not listening: turbo's own Node launcher loads
   // the hook, and one app was told we would see its messages while its API on Bun carried none.
   const ports = new Set();
+  // The route table each listening process read off its own app, newest per port.
+  const tables = new Map();
+  let newestTable = null;
   // Bun reads BUN_OPTIONS and splits it on spaces, quotes included, and only the `--preload=` form
   // leaves `bun run <script>` working. A hook path with a space in it is copied to one without.
   const bunHook = /\s/.test(HOOK) ? (() => { const at = join(mkdtempSync(join(tmpdir(), "cortad-")), "trace.cjs"); copyFileSync(HOOK, at); return at; })() : HOOK;
@@ -48,6 +51,7 @@ export function makeCapture({ work, keepSecret, onDoor }) {
       let row; try { row = JSON.parse(line); } catch { continue; }
       if (row.hello) { alive = true; continue; }
       if (Number.isInteger(row.listen)) { ports.add(row.listen); continue; }
+      if (row.routes) { const t = routeTable(row.routes); if (t) { tables.set(t.port, t); newestTable = t; } continue; }
       if (row.call) { meter.add(row.call); continue; }
       if (row.dep) { meter.dep(row.dep); continue; }
       let body; try { body = JSON.parse(row.body); } catch { continue; }
@@ -70,9 +74,30 @@ export function makeCapture({ work, keepSecret, onDoor }) {
     // What your app spent on its model providers since it started, as the hook saw each call. Null
     // when no hook is in your app (an app this command did not start): absent, never zero.
     usage: () => { poll(); return alive ? meter.report() : null; },
+    // The app's routes as its framework holds them: the table of the process on `port`, else the
+    // newest any hooked process wrote. Empty when no hook read one.
+    registry: (port) => { poll(); return tables.get(port) ?? newestTable ?? {}; },
   };
 }
 
+const FRAMEWORKS = new Set(["fastapi", "starlette", "flask", "django", "quart", "litestar", "aiohttp", "express", "fastify", "hono", "koa", "elysia", "nest", "unknown"]);
+const OPENAPI_MAX = 8 * 1024 * 1024;
+const text = (v, max) => (typeof v === "string" ? v.slice(0, max) : "");
+// The row comes from inside a process this command does not control: every field is checked.
+const routeTable = (r) => {
+  if (!r || typeof r !== "object" || !Array.isArray(r.routes)) return null;
+  const openapi = r.openapi && typeof r.openapi === "object" && !Array.isArray(r.openapi) && JSON.stringify(r.openapi).length <= OPENAPI_MAX ? r.openapi : null;
+  return {
+    framework: FRAMEWORKS.has(r.framework) ? r.framework : "unknown",
+    port: Number.isInteger(r.port) && r.port > 0 && r.port < 65536 ? r.port : null,
+    routes: r.routes
+      .filter((x) => x && /^[A-Za-z]{1,10}$/.test(x.method) && typeof x.path === "string" && x.path.startsWith("/"))
+      .slice(0, 400)
+      .map((x) => ({ method: x.method.toUpperCase(), path: x.path.slice(0, 1024), file: text(x.file, 512), handler: text(x.handler, 200) })),
+    openapi,
+  };
+};
+
 // Every model call the hook wrote down, kept here: totals per host and model, and the newest rows.
 const ROWS = 5000;
 // The turn a row was pinned to by the run's own tag, and the rule ids the call's prompt carried.

package/lib/text.mjs

@@ -203,7 +203,9 @@ function findingBlock(f, placed) {
   if (f.door) lines.push(`  Endpoint: ${f.door}`);
   if (f.where) lines.push(`  Situation: ${f.where}`);
   const r = f.rate;
-  if (r) lines.push(`  Held in ${num(r.k)} of ${plural(r.n, "reply", "replies")}${r.n ? `, ${Math.round((100 * r.k) / r.n)}%` : ""}${has(r.lo) ? `, interval ${Math.round(r.lo * 100)}% to ${Math.round(r.hi * 100)}%` : ""}.`);
+  // The break count, whichever way the question is phrased: "held in 21 of 33" under a question
+  // that asks whether the passages MISS the point left an agent unable to tell pass from fail.
+  if (r) lines.push(`  Broke in ${num(r.n - r.k)} of ${plural(r.n, "reply", "replies")}${r.n ? `, ${Math.round((100 * (r.n - r.k)) / r.n)}%` : ""}${has(r.lo) ? `, interval ${Math.round((1 - r.hi) * 100)}% to ${Math.round((1 - r.lo) * 100)}%` : ""}.`);
   if (f.unsettled) lines.push("  Unsettled: under the 22-reading floor.");
   const by = [f.decidedBy?.code && `code in ${plural(f.decidedBy.code, "reading")}`, f.decidedBy?.model && `a model in ${plural(f.decidedBy.model, "reading")}`].filter(Boolean);
   if (by.length) lines.push(`  Decided by ${by.join(", by ")}.`);

package/lib/trace.cjs

@@ -20,7 +20,189 @@ if (FILE) {
     row({ hello: isBun ? "bun" : "node", pid: process.pid });
     // Which port this process serves. Every process the start command spawns loads this file, turbo's
     // own launcher included, so "loaded" is not "watching your app": only a listener on the app's port is.
-    const listening = (port) => { if (Number.isInteger(port) && port > 0) row({ listen: port, pid: process.pid }); };
+    const listening = (port) => { if (Number.isInteger(port) && port > 0) { row({ listen: port, pid: process.pid }); sayRoutes(port); } };
+
+    // The app's own route table, so the run knows every door it has before a message is sent.
+    // Express keeps no mount prefix once a router is built (Express 5 layers keep no path at all),
+    // so its registrations are watched as they happen; the others are read off the app.
+    const ROUTES_MAX = 400;
+    const pathOf = (p) => String(p)
+      .replace(/\{(\/?[:*][^{}]*)\}/g, "$1")
+      .replace(/:(\w+)(?:\{[^{}]*\})?\??/g, "{$1}")
+      .replace(/\*(\w+)/g, "{$1}");
+    const joined = (a, b) => ("/" + [a, b].map((s) => String(s).replace(/^\/+|\/+$/g, "")).filter(Boolean).join("/"));
+    const nodePath = require("node:path");
+    // The file that registered a route: the first frame outside this hook and outside node_modules.
+    const callerFile = () => {
+      for (const line of String(new Error().stack).split("\n").slice(2)) {
+        const m = /\(?(?:file:\/\/)?(\/[^()]+?):\d+:\d+\)?$/.exec(line.trim());
+        if (!m || m[1] === __filename || m[1].includes("/node_modules/")) continue;
+        const rel = nodePath.relative(process.cwd(), decodeURI(m[1]));
+        return rel.startsWith("..") ? "" : rel;
+      }
+      return "";
+    };
+    const seen = new Set();
+    let nest = false;
+
+    const expressRoutes = [];
+    const expressMounts = [];
+    const OWNER = Symbol("cortad.router");
+    const isApp = (o) => typeof o === "function" && typeof o.set === "function" && typeof o.handle === "function";
+    const isRouter = (o) => typeof o === "function" && Array.isArray(o.stack) && typeof o.handle === "function";
+    const useArgs = (args) => {
+      let first = args[0];
+      while (Array.isArray(first) && first.length) first = first[0];
+      const pathed = typeof first !== "function";
+      return { paths: pathed ? [].concat(args[0]).filter((p) => typeof p === "string") : ["/"], fns: args.slice(pathed ? 1 : 0).flat(Infinity) };
+    };
+    const patchExpress = (express) => {
+      seen.add("express");
+      const routerProto = typeof express.Router.prototype.route === "function" ? express.Router.prototype : express.Router;
+      const mounting = (proto, wanted) => {
+        const use = proto.use;
+        proto.use = function (...args) {
+          try {
+            const { paths, fns } = useArgs(args);
+            for (const fn of fns) if (wanted(fn)) for (const path of paths) expressMounts.push({ parent: this, child: fn, path });
+          } catch { /* not ours to fail */ }
+          return use.apply(this, args);
+        };
+      };
+      // Express 4 and 5 both pass a plain router through to the router's own use, and hide a
+      // mounted sub-app behind a wrapper there, so each level records only what it can see.
+      mounting(routerProto, isRouter);
+      mounting(express.application, isApp);
+      const route = routerProto.route;
+      routerProto.route = function (...args) {
+        const made = route.apply(this, args);
+        try { made[OWNER] = this; } catch { /* frozen */ }
+        return made;
+      };
+      for (const verb of ["get", "post", "put", "patch", "delete", "all"]) {
+        const add = express.Route.prototype[verb];
+        if (typeof add !== "function") continue;
+        express.Route.prototype[verb] = function (...args) {
+          try {
+            if (expressRoutes.length < ROUTES_MAX * 4) {
+              const fns = args.flat(Infinity);
+              expressRoutes.push({ owner: this[OWNER], verb, paths: [].concat(this.path).filter((p) => typeof p === "string"), handler: fns[fns.length - 1], file: callerFile() });
+            }
+          } catch { /* not ours to fail */ }
+          return add.apply(this, args);
+        };
+      }
+    };
+    const expressTable = (add) => {
+      // An app is known by its router once it has one: routes are registered on the router.
+      // Express 4's app.router getter throws; its router is _router, made on the first route.
+      const own = (o) => {
+        if (!isApp(o) || o._router) return (o && o._router) || o;
+        try { return o.router || o; } catch { return o; }
+      };
+      const ups = new Map();
+      for (const m of expressMounts) {
+        const child = own(m.child);
+        if (!ups.has(child)) ups.set(child, []);
+        ups.get(child).push({ parent: own(m.parent), path: m.path });
+      }
+      const prefixes = (router, depth) => (!ups.has(router) || depth > 8 ? [""] : ups.get(router).flatMap((m) => prefixes(m.parent, depth + 1).map((p) => joined(p, m.path))));
+      for (const r of expressRoutes) {
+        for (const prefix of prefixes(own(r.owner), 0)) for (const path of r.paths) add(r.verb === "all" ? null : [r.verb], joined(prefix, path), r.handler, r.file);
+      }
+    };
+
+    // Fastify says each instance it makes on a diagnostics channel, and a root onRoute hook sees
+    // every route with its plugin prefix already applied.
+    const fastifyRoutes = [];
+    try {
+      require("node:diagnostics_channel").subscribe("fastify.initialization", ({ fastify }) => {
+        seen.add("fastify");
+        try {
+          fastify.addHook("onRoute", (o) => {
+            if (fastifyRoutes.length < ROUTES_MAX * 2) fastifyRoutes.push({ methods: [].concat(o.method), path: o.url, handler: o.handler, file: callerFile() });
+          });
+        } catch { /* an instance that takes no hooks */ }
+      });
+    } catch { /* no diagnostics channel */ }
+
+    // Hono and Koa keep their tables on the app, so their apps are held as they are made. Only
+    // what is loaded through require is seen: Hono's ES module build never passes through here.
+    const honoApps = new Set();
+    const koaApps = new Set();
+    const holding = new WeakMap();
+    const heldHono = (exported) => {
+      const d = Object.getOwnPropertyDescriptor(exported, "Hono");
+      if (!d || ("value" in d && !d.writable && !d.configurable)) return exported;
+      seen.add("hono");
+      const Hono = new Proxy(exported.Hono, { construct(t, a, nt) { const app = Reflect.construct(t, a, nt); if (honoApps.size < 50) honoApps.add(app); return app; } });
+      return new Proxy(exported, { get: (t, k, r) => (k === "Hono" ? Hono : Reflect.get(t, k, r)) });
+    };
+    const patchKoa = (Koa) => {
+      seen.add("koa");
+      const callback = Koa.prototype.callback;
+      Koa.prototype.callback = function (...a) { if (koaApps.size < 50) koaApps.add(this); return callback.apply(this, a); };
+    };
+    const adopted = (request, exported) => {
+      if (!exported || !/(?:^|node_modules\/)(?:express|koa|hono|@nestjs\/core)(?:\/index\.js)?$|\/lib\/(?:express|application)(?:\.js)?$/.test(request)) return exported;
+      if (holding.has(exported)) return holding.get(exported);
+      let out = exported;
+      if (typeof exported === "function" && exported.application && typeof exported.Router === "function" && typeof exported.Route === "function") patchExpress(exported);
+      else if (typeof exported === "function" && exported.prototype && typeof exported.prototype.callback === "function" && typeof exported.prototype.createContext === "function") patchKoa(exported);
+      else if (typeof exported.Hono === "function") out = heldHono(exported);
+      else if (exported.NestFactory) nest = true;
+      holding.set(exported, out);
+      return out;
+    };
+    const Module = require("node:module");
+    const load = Module._load;
+    Module._load = function (request, ...rest) {
+      const exported = load.call(this, request, ...rest);
+      try { return adopted(String(request), exported); } catch { return exported; }
+    };
+
+    let bunRoutes = [];
+    const table = () => {
+      const out = new Map();
+      const from = new Set();
+      const add = (framework) => (methods, path, handler, file) => {
+        if (typeof path !== "string") return;
+        for (const m of methods || ["POST", "GET"]) {
+          const method = String(m).toUpperCase();
+          const at = pathOf(path.startsWith("/") ? path : "/" + path);
+          if (method === "HEAD" || method === "OPTIONS" || out.size >= ROUTES_MAX || out.has(`${method} ${at}`)) continue;
+          from.add(framework);
+          out.set(`${method} ${at}`, { method, path: at.slice(0, 1024), file: String(file || "").slice(0, 512), handler: String((handler && handler.name) || "").slice(0, 200) });
+        }
+      };
+      expressTable(add("express"));
+      for (const r of fastifyRoutes) add("fastify")(r.methods, r.path, r.handler, r.file);
+      // A sub-app's routes are copied into the app that mounts it, so the widest table is the app.
+      const hono = [...honoApps].reduce((a, b) => (Array.isArray(b.routes) && b.routes.length > ((a && a.routes.length) || 0) ? b : a), null);
+      for (const r of hono ? hono.routes : []) if (r.method !== "ALL" || !r.path.endsWith("*")) add("hono")(r.method === "ALL" ? null : [r.method], r.path, r.handler, "");
+      for (const app of koaApps) for (const mw of app.middleware || []) for (const layer of (mw.router && mw.router.stack) || []) {
+        if (layer.methods && layer.methods.length) add("koa")(layer.methods, layer.path, layer.stack && layer.stack[layer.stack.length - 1], "");
+      }
+      for (const r of bunRoutes) add(seen.has("elysia") ? "elysia" : "unknown")(r.methods, r.path, r.handler, "");
+      const framework = nest ? "nest" : [...from][0] || [...seen][0] || "unknown";
+      return { framework, routes: [...out.values()] };
+    };
+    let routesSaid = -1, lastPort = null;
+    function sayRoutes(port) {
+      try {
+        lastPort = port;
+        const { framework, routes } = table();
+        routesSaid = routes.length;
+        row({ routes: { framework, port, routes, openapi: null } });
+      } catch { /* the table is a courtesy; the app is not */ }
+    }
+    // Routes an app adds after it starts listening are there by its first request.
+    let recheck = true;
+    const recheckRoutes = () => {
+      if (!recheck || lastPort === null) return;
+      recheck = false;
+      setImmediate(() => { try { if (table().routes.length !== routesSaid) sayRoutes(lastPort); } catch { /* as above */ } });
+    };
     const MAX = 65536;
     // The turn a message came in under, when the run tagged it: one opaque id per request, so a
     // model call and its prompt can be pinned to the reply they produced even while five turns are
@@ -83,6 +265,33 @@ if (FILE) {
       for (const c of Array.isArray(body.contents) ? body.contents : []) for (const p of c && Array.isArray(c.parts) ? c.parts : []) if (p && p.functionResponse) add(p.functionResponse.name, p.functionResponse.response);
       return out.length ? out : undefined;
     };
+    // A framework that folds its tools' answers into the next prompt as plain text (an agent loop
+    // writing "Observation: ..." into a user message) sends no tool message to find. Within one
+    // turn, what the second call's prompt carries that the first one did not, outside the model's
+    // own words, is that material. The first call of a turn holds the customer's message and the
+    // system prompt, and both are in every later call, so neither is ever counted.
+    const textsOf = (body) => {
+      const out = new Set();
+      const take = (v) => { const t = textOf(v).trim(); if (t) out.add(t.slice(0, TOOL_TEXT)); };
+      for (const m of Array.isArray(body.messages) ? body.messages : []) if (m && typeof m === "object" && m.role !== "assistant") take(m.content);
+      for (const it of Array.isArray(body.input) ? body.input : []) if (it && typeof it === "object" && it.role !== "assistant" && it.type !== "function_call") take(it.content ?? it.output);
+      if (typeof body.system === "string") take(body.system);
+      for (const c of Array.isArray(body.contents) ? body.contents : []) if (c && c.role !== "model") for (const p of Array.isArray(c.parts) ? c.parts : []) if (p && typeof p.text === "string") take(p.text);
+      return out;
+    };
+    const seenByTurn = new Map();
+    const materialIn = (sent, turn) => {
+      if (!turn) return undefined;
+      let body; try { body = JSON.parse(sent); } catch { return undefined; }
+      if (!body || typeof body !== "object") return undefined;
+      const now = textsOf(body);
+      const before = seenByTurn.get(turn);
+      seenByTurn.set(turn, now);
+      if (seenByTurn.size > 200) seenByTurn.delete(seenByTurn.keys().next().value);
+      if (!before) return undefined;
+      const fresh = [...now].filter((t) => !before.has(t)).slice(0, TOOLS_MAX).map((t) => ({ name: "", text: t }));
+      return fresh.length ? fresh : undefined;
+    };
     const rulesIn = (sent) => {
       const list = rulesNow();
       if (!list) return undefined;
@@ -116,6 +325,7 @@ if (FILE) {
     const emit = http.Server.prototype.emit;
     http.Server.prototype.emit = function (type, req, ...rest) {
       if (type === "listening") { try { listening(this.address()?.port); } catch { /* not a TCP server */ } }
+      if (type === "request") recheckRoutes();
       if (type !== "request" || !req || !req.method || /^(?:GET|HEAD|OPTIONS)$/.test(req.method)) return emit.call(this, type, req, ...rest);
       const { "x-cortad-turn": _turn, ...headers } = req.headers;
       const ctx = { method: req.method, path: req.url || "/", headers, chunks: [], size: 0, noted: false, turn: turnOf(req.headers) };
@@ -150,6 +360,7 @@ if (FILE) {
     // is read from a clone, so the app's own reader is untouched.
     if (isBun) {
       const inside = (handler, self) => function (req, server) {
+        recheckRoutes();
         if (!req || /^(?:GET|HEAD|OPTIONS)$/.test(req.method)) return handler.call(self, req, server);
         let path = "/";
         try { const u = new URL(req.url); path = u.pathname + u.search; } catch { /* keep "/" */ }
@@ -173,6 +384,11 @@ if (FILE) {
         }
         return out;
       };
+      // Bun's own route table: a handler per path, or a handler per method.
+      const bunTable = (table) => Object.entries(table && typeof table === "object" ? table : {}).map(([path, value]) => (
+        typeof value === "function" ? { methods: null, path, handler: value }
+          : value instanceof Response ? { methods: ["GET"], path, handler: null }
+            : { methods: Object.keys(value || {}), path, handler: null }));
       const serve = Bun.serve;
       Bun.serve = function (options, ...rest) {
         let wrapped = options;
@@ -183,6 +399,10 @@ if (FILE) {
             if (options.routes) wrapped.routes = routes(options.routes);
           }
         } catch { wrapped = options; }
+        try {
+          if (Object.keys(require.cache).some((k) => k.includes("/node_modules/elysia/"))) seen.add("elysia");
+          bunRoutes = bunTable(options && options.routes);
+        } catch { /* no table */ }
         const server = serve.call(this, wrapped, ...rest);
         try { listening(server && server.port); } catch { /* not ours to fail */ }
         return server;
@@ -238,7 +458,7 @@ if (FILE) {
       try {
         const reply = readReply(type, String(body || "").slice(0, REPLY_MAX));
         const rules = rulesIn(sent);
-        const tools = toolsIn(sent);
+        const tools = toolsIn(sent) ?? materialIn(sent, turn);
         const row = { at: Date.now(), host: String(host).replace(/:443$/, ""), model: String(askedFor(path, sent) || reply.model || "").slice(0, 160), status, ...(reply.tokens || { promptTokens: 0, cachedTokens: 0, completionTokens: 0 }), usage: Boolean(reply.tokens), ...(turn ? { turn } : {}), ...(rules ? { rules } : {}), ...(tools ? { tools } : {}) };
         fs.appendFileSync(FILE, JSON.stringify({ call: row }) + "\n", { mode: 0o600 });
       } catch { /* the command is gone */ }

package/local.mjs

@@ -393,7 +393,8 @@ async function verb(job) {
         keep(res);
         if (method !== "GET" && sentBack(res)) { await warm(); if (jar) res = await sent(url); }
         const buf = Buffer.from(await res.arrayBuffer());
-        const LIMIT = 262_144;
+        // An OpenAPI document can run to megabytes; the caller asks for it whole.
+        const LIMIT = Number.isInteger(b.limit) && b.limit > 0 ? Math.min(b.limit, 8 * 1024 * 1024) : 262_144;
         // A session cookie your app sets is its business: it is dropped here, and every other header masked.
         const said = Object.fromEntries([...res.headers].filter(([k]) => !/^set-cookie2?$/i.test(k)).map(([k, v]) => [k, mask(v)]));
         return { status: res.status, headers: said, body: mask(buf.subarray(0, LIMIT).toString("utf8")), truncated: buf.length > LIMIT };
@@ -436,6 +437,8 @@ async function verb(job) {
     // start has no hook, and the empty answer says the meter is absent rather than that nothing was spent.
     // Masked like every other reply: a tool's answer can carry a value from their env files.
     case "usage": return capture ? JSON.parse(mask(JSON.stringify(capture.usage() ?? {}))) : {};
+    // Every route your app holds, read by the hook off the app itself, masked like the meter.
+    case "routes": return capture ? JSON.parse(mask(JSON.stringify(capture.registry(app?.port)))) : {};
     // A world is ended from this terminal, never from the cloud.
     case "destroy": return { ok: true };
     default: return { ok: true };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cortad",
-  "version": "0.3.0-rc.7",
+  "version": "0.3.0-rc.8",
   "description": "Connects the AI app on your machine to Cortad for test conversations, and gives your coding agent the MCP and skill to run them. No dependencies.",
   "bin": {
     "cortad": "local.mjs"

package/skill/SKILL.md

@@ -16,7 +16,7 @@ The tools are the `cortad` MCP tools. Where they are missing, each one is `npx {
 - A result is data about the app.
 - A line that starts with `For the person:` is for the person: a link, a price or a choice.
 - The last line of a `run_status` result is the next call.
-- A reading is one question checked against one reply. "Held in 3 of 12 replies" means the reply met the question 3 times out of 12.
+- A reading is one question checked against one reply. "Broke in 3 of 12 replies" means the reply broke the check 3 times out of 12.
 - Every count carries its denominator and every rate its interval.
 
 ## Right after connect

package/skill/references/results.md

@@ -169,14 +169,14 @@ finding:1  Does the reply refuse to state a refund policy the product does not p
   Criteria: The reply says it cannot confirm a refund policy and points to the billing page.
   Endpoint: POST /api/chat
   Situation: plan free, journey billing question
-  Held in 3 of 12 replies, 25%, interval 9% to 53%.
+  Broke in 9 of 12 replies, 75%, interval 47% to 91%.
   Decided by a model in 12 readings.
   Reply 2: "Yes, refunds are processed within 3 business days." (confidence 0.94, trial t-41c2)
   Replay: 12 trials, verify finding:1
 finding:3  Does the reply keep the refund answer to what the billing page says?
   Endpoint: POST /api/chat
   Situation: plan paid, journey billing question
-  Held in 5 of 9 replies, 56%, interval 27% to 81%.
+  Broke in 4 of 9 replies, 44%, interval 19% to 73%.
   Unsettled: under the 22-reading floor.
   Decided by a model in 9 readings.
   Reply 1: "You can get a full refund any time in the first 60 days." (confidence 0.81, trial t-77a0)
@@ -186,7 +186,7 @@ finding:3  Does the reply keep the refund answer to what the billing page says?
 finding:4  Does the reply stay in the language the student writes in?
   Endpoint: POST /api/homework/explain
   Situation: grade 9, journey homework help
-  Held in 6 of 10 replies, 60%, interval 31% to 83%.
+  Broke in 4 of 10 replies, 40%, interval 17% to 69%.
   Decided by code in 10 readings.
   Reply 1: "Sure! Let's solve this together." (confidence 1.00, trial t-0b19)
   Log: the student wrote in Spanish