cortad 0.3.2 → 0.3.4

package/lib/identities.mjs

@@ -34,6 +34,10 @@ const STATUSES = new Set(["minted", "sealed", "absent", "refused", "unreachable"
 // so USER_USERNAME pairs with USER_PASSWORD and TUDUDI_USER_EMAIL with TUDUDI_USER_PASSWORD.
 const USER_TOKEN = /_(?:USERNAME|USER|EMAIL|LOGIN)$/;
 const SECRET_TOKEN = /_(?:PASSWORD|PASS|SECRET)$/;
+// A pair that opens a store or a mail server is never the app's own account: PG_DB_USER with
+// PG_DB_PASSWORD signs in to Postgres, and offered to a sign-in form it sent a database password
+// to the app's login route.
+const SERVICE_PREFIX = /(?:^|_)(?:DB|DATABASE|SQL|PG|PSQL|POSTGRES(?:QL)?|MYSQL|MARIADB|MSSQL|MONGO(?:DB)?|REDIS|VALKEY|ELASTIC(?:SEARCH)?|OPENSEARCH|ES|QDRANT|NEO4J|CLICKHOUSE|RABBIT(?:MQ)?|AMQP|KAFKA|SMTP|MAIL|EMAIL|S3|MINIO|FTP|SFTP|LDAP|PROXY)(?:_|$)/;
 const CRED_USER_FIELD = /^(?:e-?mail|user(?:_?name)?|login|account|identifier)$/i;
 const CRED_PASS_FIELD = /^(?:password|passcode|pass|pwd)$/i;
 const SIGN_IN_ROUTE = /(?:^|\/)(?:login|signin|sign-in|sign_in|session|token|authenticate)(?:\/|$)/i;
@@ -586,7 +590,7 @@ export function credentialPairs(names) {
   const seen = new Set();
   for (const u of users) {
     const s = secrets.find((x) => x.prefix === u.prefix);
-    if (s && !seen.has(u.prefix)) { seen.add(u.prefix); pairs.push({ user: u.name, secret: s.name, prefix: u.prefix }); }
+    if (s && !seen.has(u.prefix) && !SERVICE_PREFIX.test(u.prefix)) { seen.add(u.prefix); pairs.push({ user: u.name, secret: s.name, prefix: u.prefix }); }
   }
   return pairs;
 }

package/lib/keeping.mjs

@@ -5,7 +5,7 @@
 import { createHash } from "node:crypto";
 import { localAddresses, makeBacking, tcpOpen } from "./backing.mjs";
 import { keepData } from "./data.mjs";
-import { bypassedOf, isStorePort, onThisMachine, storeId, storeKeeper, storesOf, unnamedLines } from "./stores.mjs";
+import { bypassedOf, isStorePort, onThisMachine, storeId, storeKeeper, storesOf, unnamedHeld } from "./stores.mjs";
 
 // A line that opens with a setting's name keeps its case: "nEXT_PUBLIC_SUPABASE_URL" was printed.
 const lower = (line) => (/^[A-Z][A-Z0-9_]/.test(line) ? line : line.charAt(0).toLowerCase() + line.slice(1));
@@ -34,17 +34,26 @@ function blockedRows(blocked) {
 // folder and its source files. launch(plan): a service of the repository started the way the app
 // is. connections(): what the hook saw the app reach. hold(): the run's requests wait from here until
 // the caller has the app on the copies made, since until then it still writes into the originals.
-export function makeKeeping({ root, work, ledgerFile, onPath, say, settings, launch, connections, hold = () => {} }) {
+// `typed`: the person gave the start command. It is run as it is, with the settings and the stores
+// that command gives the app: a legal research API typed ENV=staging and was started on copies of
+// the test databases its default settings file names, where its library is empty.
+const AS_TYPED = "Your app is started with your own command as it is, on the databases that command gives it. Nothing is copied: a run writes into them once you say yes to each on the card.";
+export function makeKeeping({ root, work, ledgerFile, onPath, say, settings, launch, connections, hold = () => {}, typed = false }) {
   let pending = null;
   let now = null;
   let backing = null;
   let backedUp = false;
   let seen = 0;
-  const told = new Set();
   const tell = (lines) => { for (const line of lines) say(lower(line)); };
   const held = (blocked) => tell(blocked.map((b) => `Run waits for your yes on the card in the browser: ${lower(b.why)}`));
 
   async function keepAll(started) {
+    if (typed) {
+      const none = async () => ({ env: {}, risks: [], copies: [], blocked: [], pass: [] });
+      now = { env: {}, risks: [], notes: [AS_TYPED], blocked: [], pass: [], stores: [], keeper: { keep: none, drop: async () => {} }, values: {}, started };
+      tell(now.notes);
+      return now;
+    }
     const { values, from, appDir, sources } = settings();
     const disk = keepData({ values, from, appDir, root, work, sources, started });
     const stores = storesOf(disk.values);
@@ -61,6 +70,7 @@ export function makeKeeping({ root, work, ledgerFile, onPath, say, settings, lau
 
   // Brings up what is wanted, copies each store that now answers, and says both.
   async function bringUp(wants) {
+    if (typed) return { changed: false, came: false, copied: false };
     const k = await once(true);
     hold();
     const up = await backing.up(wants);
@@ -136,14 +146,11 @@ export function makeKeeping({ root, work, ledgerFile, onPath, say, settings, lau
       const k = await pending;
       const live = [];
       for (const c of conns) if (!onThisMachine(c.host) || (await tcpOpen(c.host, c.port))) live.push(c);
-      const fresh = unnamedLines(live, k.stores, told);
-      k.risks.push(...fresh);
-      tell(fresh);
-      const bypassed = bypassedOf(live, k.stores, k.blocked);
+      const bypassed = [...bypassedOf(live, k.stores, k.blocked), ...unnamedHeld(live, k.stores, k.blocked)];
       k.blocked.push(...bypassed);
       held(bypassed);
-      const got = await bringUp(reached(appPort));
-      return { changed: fresh.length > 0 || bypassed.length > 0 || got.changed, copied: got.copied };
+      const got = typed ? { changed: false, copied: false } : await bringUp(reached(appPort));
+      return { changed: bypassed.length > 0 || got.changed, copied: got.copied };
     },
     // An app attached to instead of started keeps its own connections: its copies are of no use.
     async forget() {

package/lib/pyhook/cortad_tied.py

@@ -26,6 +26,23 @@ RECENT_KEPT, RECENT_MS = 32, 300000
 # turn (a chat UI shows the message with one event, then answers it with the next).
 BEFORE_MS = 60000
 PROMPTS_KEPT, PROMPT_HEAD = 64, 500
+# How much of a sentence the model wrote a fetched reply has to carry to be the answer.
+CARRY = 60
+# tied.cjs UNDER_WAY: a job's own word for work still going on.
+UNDER_WAY = re.compile(r"^(?:queued|pending|waiting|scheduled|submitted|accepted|created|starting|started|running|processing|generating|working|thinking|searching|retrieving|streaming|in[_ -]?progress|not[_ -]?(?:started|ready))$", re.I)
+
+
+def _under_way(v, depth=0):
+    if depth > 4:
+        return False
+    if isinstance(v, list):
+        return bool(v) and _under_way(v[-1], depth + 1)
+    if not isinstance(v, dict):
+        return False
+    state = v.get("status", v.get("state"))
+    if isinstance(state, str) and UNDER_WAY.match(state):
+        return True
+    return any(isinstance(x, (list, dict)) and _under_way(x, depth + 1) for x in v.values())
 # The ask the last call pinned in this handler's context was made for. A queue runs each handler in a
 # context of its own, so its later calls, which carry none of the person's words, are told apart by
 # it when several asks are open at once. Only a tiebreak: it never outranks what the rules decide.
@@ -68,10 +85,23 @@ def _address_ids(req):
     return {v for v in path.split("/") + [v for _, v in parse_qsl(query)] if ID.match(v)}
 
 
+# tied.cjs saidSentences: a JSON answer the clip cut short is read by its strings.
+def _said_sentences(text):
+    text = text.decode("utf-8", "replace") if isinstance(text, (bytes, bytearray)) else str(text or "")
+    v = _json(text)
+    whole = [s for s in (_scalars(v, []) if v is not None else [text]) if _sentence(s)]
+    if len(whole) != 1 or not re.match(r"\s*[\[{]", whole[0]):
+        return whole
+    return [m for m in re.findall(r'"((?:[^"\\]|\\.){8,})"', whole[0]) if _sentence(m)]
+
+
 class Tied:
     # `body_of(req)` and `reply_of(req)` read what a request carried and what the app answered.
-    def __init__(self, norm, body_of, reply_of):
+    # `said_of(req)`: what the model calls made for an ask answered later have written, one entry a
+    # call and none before one has, None where the hook hears none (tied.cjs saidOf).
+    def __init__(self, norm, body_of, reply_of, said_of=None):
         self.norm, self.body_of, self.reply_of = norm, body_of, reply_of
+        self.said_of = said_of or (lambda req: None)
         self.open = {}
         self.recent = collections.deque(maxlen=RECENT_KEPT)
         # The heads of the prompts this process sent: one arriving at a server in the same process (an
@@ -127,7 +157,7 @@ class Tied:
         now = int(time.time() * 1000)
         for t in reversed(list(self.recent)) if mine else []:
             # A request answered later is fetched afterwards only by a read of it, never by a new message.
-            if t["at"] <= req["at"] and now - t["at"] <= RECENT_MS and (t.get("pinned") or not pinned_only) and (not t.get("settled") or req["method"] == "GET"):
+            if t["at"] <= req["at"] and now - t.get("call_at", t["at"]) <= RECENT_MS and (t.get("pinned") or not pinned_only) and (not t.get("settled") or req["method"] == "GET"):
                 shared = mine & self._ids(t)
                 if shared:
                     return t, shared
@@ -146,7 +176,7 @@ class Tied:
     def closed(self, req):
         self.open.pop(req["id"], None)
         asker = req.get("asker")
-        if asker and not asker.get("written") and not asker.get("calls_open"):
+        if asker and not asker.get("written") and not asker.get("calls_open") and self._carries(req, asker):
             since = max([self.answered[i] for i in req["tie"] if self.answered.get(i, asker["at"]) < asker["at"]], default=0)
             before = [p for p in list(self.recent) if p is not asker and not p.get("settled") and since < p["at"] < asker["at"] and asker["at"] - p["at"] <= BEFORE_MS and self._ids(p) & req["tie"]]
             for s in before + [asker]:
@@ -161,6 +191,18 @@ class Tied:
             self.recent.append(req)
         return None
 
+    # tied.cjs carries: a page polling for an answer several calls build comes back many times
+    # between two of them, on a document with no answer in it yet.
+    def _carries(self, fetch, asker):
+        said = self.said_of(asker)
+        if said is None:
+            return True
+        raw = self.reply_of(fetch)
+        if _under_way(_json(raw)):
+            return False
+        reply = self.norm(raw)
+        return any(self.norm(w)[:CARRY] in reply for one in (said if isinstance(said, (list, tuple)) else [said]) for w in _said_sentences(one))
+
     def _answered(self, tie, at):
         for i in tie:
             self.answered.pop(i, None)
@@ -230,6 +272,7 @@ class Tied:
             asker["late"] = True
         asker["pinned"] = True
         asker["calls_open"] = asker.get("calls_open", 0) + 1
+        asker["call_at"] = int(time.time() * 1000)
         _HELD.set(asker)
         return asker
 
@@ -260,6 +303,7 @@ class Tied:
         if req.get("late"):
             req["pinned"] = True
             req["calls_open"] = req.get("calls_open", 0) + 1
+            req["call_at"] = int(time.time() * 1000)
         else:
             req["waiting"] = req.get("waiting", 0) + 1
         return req
@@ -285,7 +329,7 @@ class Tied:
     # request tied to it came to fetch the answer. It stays where a fetch that comes later can still
     # find it and complete the exchange on a second request, but no call is pinned to it by its words.
     def settled(self, req):
-        if not req or not req.get("late") or not req.get("ended") or req.get("calls_open") or req.get("waiting") or req.get("written") or req.get("settled") or req.get("fetched"):
+        if not req or not req.get("late") or not req.get("ended") or req.get("calls_open") or req.get("waiting") or req.get("written") or req.get("settled") or (req.get("fetched") and self.said_of(req) is None):
             return False
         req["settled"] = req["answered"] = True
         return True

package/lib/pyhook/sitecustomize.py

@@ -1548,7 +1548,7 @@ def _install():
 
     try:
         from cortad_tied import Tied
-        tied = Tied(norm, body_of, reply_of)
+        tied = Tied(norm, body_of, reply_of, lambda req: req.get("said_all", []) if req.get("late") else None)
     except Exception:
         tied = None
 
@@ -1749,6 +1749,7 @@ def _install():
                 if req and req.get("late"):
                     if not row.get("embedding"):
                         req["said"] = (row["status"], reply_text(events)[:model_words], now)
+                        req["said_all"] = (req.get("said_all", []) + [req["said"][1]])[-12:]
                     settle(req)
             row.update(inside_of(req))
             if req and (req.get("kept") or req.get("turn")):

package/lib/stores.mjs

@@ -21,7 +21,7 @@ export const PORT_OF = { postgres: 5432, mysql: 3306, mongo: 27017, redis: 6379,
 const ENGINE_AT = { ...Object.fromEntries(Object.entries(PORT_OF).filter(([e]) => e !== "chroma" && e !== "weaviate").map(([e, p]) => [p, e])), 6334: "qdrant" };
 export const isStorePort = (port) => Boolean(ENGINE_AT[port]);
 // Stores reached through their maker's SDK, which a setting rarely names: the hook sees the host.
-const HOSTED = [["supabase", /\.supabase\.(?:co|in|com)$/], ["mongo", /\.mongodb\.net$/], ["redis", /\.upstash\.io$|\.redis-cloud\.com$|\.redislabs\.com$/], ["postgres", /\.neon\.tech$/], ["pinecone", /\.pinecone\.io$/], ["qdrant", /\.qdrant\.io$/], ["firestore", /^firestore\.googleapis\.com$/]];
+const HOSTED = [["supabase", /\.supabase\.(?:co|in|com)$/], ["mongo", /\.mongodb\.net$/], ["elasticsearch", /\.elastic-cloud\.com$|\.found\.io$|\.es\.amazonaws\.com$/], ["redis", /\.upstash\.io$|\.redis-cloud\.com$|\.redislabs\.com$/], ["postgres", /\.neon\.tech$/], ["pinecone", /\.pinecone\.io$/], ["qdrant", /\.qdrant\.io$/], ["firestore", /^firestore\.googleapis\.com$/]];
 // How each store we copy is copied and dropped: on its own server, under the name given.
 const COPY = {
   postgres: { copy: (s, clone) => clonePg(s.conn, s.db, clone), drop: (s, clone) => dropPg(s.conn, clone) },
@@ -39,6 +39,10 @@ const WORD_ENGINE = [["postgres", /^(?:PG|POSTGRES|PSQL)|POSTGRES/], ["mysql", /
 const SECRETISH = /(?:^|_)(?:KEY|TOKEN|SECRET|PASSWORD|PASS|PWD)$/;
 // Every engine's word in a setting's name, so a setting that carries one is that engine's own.
 const ENGINE_WORD = new RegExp([...Object.keys(ENGINE), ...[...BY_NAME, ...WORD_ENGINE].map(([, re]) => re.source)].join("|"), "i");
+// A settings file's values without the ones that name a store. An app the person started with
+// their own command is handed the rest: which databases it uses is that command's to say.
+const STORE_SETTING = new RegExp(`${ENGINE_WORD.source}|(?:^|_)(?:DB|DATABASE)(?:_|$)`, "i");
+export const withoutStoreSettings = (values) => Object.fromEntries(Object.entries(values).filter(([name]) => !STORE_SETTING.test(name)));
 // Stores that hold no collections: a bare COLLECTION setting beside one of them is still Qdrant's.
 const NO_COLLECTIONS = new Set(["redis", "mysql", "sqlserver", "neo4j", "meilisearch", "pinecone"]);
 const LOOPBACK = /^(?:localhost|127(?:\.\d+){3}|::1|\[::1\]|0\.0\.0\.0|host\.docker\.internal)?$/i;
@@ -329,19 +333,19 @@ export function storeKeeper({ ledgerFile, values = {}, dirs = [], fetchFn, local
 }
 
 // The stores the app reached that no setting names, as the hook saw each connection: a store written
-// into a config file rather than the environment, or one its maker's SDK finds by itself. One
-// sentence per store, said once.
-export function unnamedLines(connections, stores, told) {
-  const lines = [];
+// into a config file rather than the environment, one its maker's SDK finds by itself, or the ones an
+// app started with the person's own command was given. No copy stands in for any of them, so each
+// holds Run for the person's yes, once. `blocked`: the rows already holding it.
+export function unnamedHeld(connections, stores, blocked) {
+  const rows = [];
   for (const { host, port } of connections) {
     const local = onThisMachine(host);
-    const engine = local ? ENGINE_AT[port] : HOSTED.find(([, re]) => re.test(String(host).toLowerCase()))?.[0];
-    const key = local ? `localhost:${port}` : `${engine}:${String(host).toLowerCase()}`;
-    if (!engine || told.has(key) || stores.some((s) => (local ? `${hostKey(s.host)}:${s.port}` === key : s.engine === engine && !onThisMachine(s.host)))) continue;
-    told.add(key);
-    lines.push(local
-      ? `Your app connects to a ${ENGINE[engine]} server on this machine that none of its settings names, so it cannot be pointed at a copy: what test conversations create there stays there.`
-      : `Your app reaches ${ENGINE[engine]} on a server off this machine, and a copy is only made on this machine: what test conversations create there stays there.`);
+    const engine = local ? ENGINE_AT[port] : HOSTED.find(([, re]) => re.test(String(host).toLowerCase()))?.[0] ?? ENGINE_AT[port];
+    if (!engine || stores.some((s) => (local ? `${hostKey(s.host)}:${s.port}` === `localhost:${port}` : s.engine === engine && !onThisMachine(s.host)))) continue;
+    const id = createHash("sha256").update(`${engine}|${local ? "localhost" : String(host).toLowerCase()}|${port}`).digest("hex").slice(0, 16);
+    if ([...blocked, ...rows].some((b) => b.id === id)) continue;
+    // Named by its engine where a setting's name would stand: the API takes a name, never an address.
+    rows.push({ id, names: [ENGINE[engine].replace(/\W+/g, "_")], why: `Your app reaches ${/^[AEIOU]/.test(ENGINE[engine]) ? "an" : "a"} ${ENGINE[engine]} server ${local ? "on" : "off"} this machine that it was not given a copy of, so the run would write what its test conversations create into it as it is. Start your app on a copy of it, then run the connect command again.` });
   }
-  return lines;
+  return rows;
 }

package/lib/tied.cjs

@@ -39,9 +39,32 @@ const sentence = (v) => v.length >= 8 && /\s|[^\x00-\x7F]/.test(v);
 // The sentences a body or a reply carries, read as the person's words are: an "ok" or an id says nothing.
 const sentencesIn = (text) => { const v = json(text); return (v === undefined ? [String(text || "")] : scalars(v)).filter(sentence); };
 const PROMPTS_KEPT = 64, PROMPT_HEAD = 500;
+// How much of a sentence the model wrote a fetched reply has to carry to be the answer.
+const CARRY = 60;
+// A job's own word for work still going on, as its status or state says it: at the top of a document,
+// in an object inside it, or on the newest entry of a list in it. The backend reads the same words
+// (customer/endpoint.ts).
+const UNDER_WAY = /^(?:queued|pending|waiting|scheduled|submitted|accepted|created|starting|started|running|processing|generating|working|thinking|searching|retrieving|streaming|in[_ -]?progress|not[_ -]?(?:started|ready))$/i;
+const underWay = (v, depth = 0) => {
+  if (!v || typeof v !== "object" || depth > 4) return false;
+  if (Array.isArray(v)) return underWay(v.at(-1), depth + 1);
+  const state = v.status ?? v.state;
+  if (typeof state === "string" && UNDER_WAY.test(state)) return true;
+  return Object.values(v).some((x) => x && typeof x === "object" && underWay(x, depth + 1));
+};
+// The sentences a model wrote, read as the person's words are; a JSON answer the clip cut short is
+// read by its strings.
+const saidSentences = (text) => {
+  const whole = sentencesIn(text);
+  if (whole.length !== 1 || !/^\s*[[{]/.test(whole[0])) return whole;
+  return [...whole[0].matchAll(/"((?:[^"\\]|\\.){8,})"/g)].map((m) => m[1]).filter(sentence);
+};
 
 // `bodyOf(req)` and `replyOf(req)` read what a request carried and what the app answered.
-function makeTied({ norm, bodyOf, replyOf }) {
+// `saidOf(req)`: what the model calls made for an ask answered later have written, one entry a call
+// and none before one has, undefined where the hook hears none. The answer a person reads is not
+// always the last call's: a second look that changes nothing leaves the first reading standing.
+function makeTied({ norm, bodyOf, replyOf, saidOf = () => undefined }) {
   const open = new Set();
   const recent = [];
   // The heads of the prompts this process sent: one arriving at a server in the same process (an
@@ -68,6 +91,17 @@ function makeTied({ norm, bodyOf, replyOf }) {
     return first[1] >= 0 && first[1] !== second[1] ? first[0] : null;
   };
   const drop = (req) => { const i = recent.indexOf(req); if (i >= 0) recent.splice(i, 1); };
+  // Whether a fetch carries the ask's answer. A page polling for an answer several calls build comes
+  // back many times between two of them, on a document with no answer in it yet, or with a first
+  // reading in it while the document itself still says running.
+  const carries = (fetch, asker) => {
+    const said = saidOf(asker);
+    if (said === undefined) return true;
+    const raw = replyOf(fetch);
+    if (underWay(json(raw))) return false;
+    const reply = norm(raw);
+    return [].concat(said).flatMap(saidSentences).some((w) => reply.includes(norm(w).slice(0, CARRY)));
+  };
   // Asks reached through one session are one client's turns, the newest the one being answered: a
   // client holds a second stream open on its session (a heartbeat) that points at an older ask.
   const newestOf = (found) => {
@@ -89,7 +123,7 @@ function makeTied({ norm, bodyOf, replyOf }) {
     for (let i = mine.size ? recent.length - 1 : -1; i >= 0; i--) {
       const t = recent[i];
       // A request answered later is fetched afterwards only by a read of it, never by a new message.
-      if (t.at > req.at || now - t.at > RECENT_MS || (pinnedOnly && !t.pinned) || (t.settled && req.method !== "GET")) continue;
+      if (t.at > req.at || now - (t.callAt ?? t.at) > RECENT_MS || (pinnedOnly && !t.pinned) || (t.settled && req.method !== "GET")) continue;
       const tie = [...mine].filter((v) => idsOf(t).has(v));
       if (tie.length) return { asker: t, tie: new Set(tie) };
     }
@@ -107,7 +141,7 @@ function makeTied({ norm, bodyOf, replyOf }) {
     closed(req) {
       open.delete(req);
       const { asker } = req;
-      if (asker && !asker.written && !asker.callsOpen) {
+      if (asker && !asker.written && !asker.callsOpen && carries(req, asker)) {
         const since = Math.max(0, ...[...req.tie].map((i) => answered.get(i) ?? asker.at).filter((at) => at < asker.at));
         const before = recent.filter((p) => p !== asker && !p.settled && since < p.at && p.at < asker.at && asker.at - p.at <= BEFORE_MS && meets(idsOf(p), req.tie));
         for (const s of [...before, asker]) drop(s);
@@ -140,7 +174,7 @@ function makeTied({ norm, bodyOf, replyOf }) {
     inside(req, sent) {
       if (req.finished && (!req.noted || req.late) && !says(req, norm(sent))) return pinned(sent, req);
       if (!req.noted && req.finished) req.late = true;
-      if (req.late) { req.pinned = true; req.callsOpen = (req.callsOpen || 0) + 1; }
+      if (req.late) { req.pinned = true; req.callsOpen = (req.callsOpen || 0) + 1; req.callAt = Date.now(); }
       else req.waiting = (req.waiting || 0) + 1;
       return req;
     },
@@ -166,7 +200,7 @@ function makeTied({ norm, bodyOf, replyOf }) {
     // request tied to it came to fetch the answer. It stays where a fetch that comes later can still
     // find it and complete the exchange on a second request, but no call is pinned to it by its words.
     settled(req) {
-      if (!req || !req.late || !req.ended || req.callsOpen || req.waiting || req.written || req.settled || req.fetched) return false;
+      if (!req || !req.late || !req.ended || req.callsOpen || req.waiting || req.written || req.settled || (req.fetched && saidOf(req) === undefined)) return false;
       req.settled = true;
       return true;
     },
@@ -210,6 +244,7 @@ function makeTied({ norm, bodyOf, replyOf }) {
     if (!asker.noted && asker.finished) asker.late = true;
     asker.pinned = true;
     asker.callsOpen = (asker.callsOpen || 0) + 1;
+    asker.callAt = Date.now();
     return asker;
   }
 }

package/lib/wire.cjs

@@ -761,14 +761,14 @@ function makeRecorder({ row, rulesFile, onTurn = () => {}, onEnded = () => {} })
   // of the person's is kept: their newest is the sign-in and the proof a run takes, and the command
   // keeps only the newest few per door.
   // ponytail: the trace file grows by one row per request of the person's; a cap per route if a person's own load test fills it.
-  const EXCHANGES_PER_ROUTE = 4, CANARY_PER_ROUTE = 30, ROUTES_KEPT = 1000, SENT_MAX = 4, MODEL_WORDS = 4000;
+  const EXCHANGES_PER_ROUTE = 4, CANARY_PER_ROUTE = 30, ROUTES_KEPT = 1000, SENT_MAX = 4, MODEL_WORDS = 4000, SAID_KEPT = 12;
   const slotOf = (turn) => (!turn ? { cls: "", max: Infinity } : /^canary:/.test(turn) ? { cls: "canary ", max: CANARY_PER_ROUTE } : { cls: "ours ", max: EXCHANGES_PER_ROUTE });
   const perRoute = new Map();
   // One route whatever id its path carries: a conversation in the path is a new path every trial.
   const routeKey = (method, path) => `${method} ${path.split("?")[0].split("/").map((s) => (/^\d+$/.test(s) || (s.length >= 8 && /\d/.test(s)) ? "{id}" : s)).join("/")}`;
   let lastExchange = 0;
   const { makeTied, sentencesIn } = require("./tied.cjs");
-  const tied = makeTied({ norm, bodyOf, replyOf: (ctx) => (ctx.reply ? ctx.reply.text.slice(0, MAX) : "") });
+  const tied = makeTied({ norm, bodyOf, replyOf: (ctx) => (ctx.reply ? ctx.reply.text.slice(0, MAX) : ""), saidOf: (ctx) => (ctx.late ? ctx.saidAll || [] : undefined) });
   const requestCtx = (method, path, headers, turn) => {
     if (turn) onTurn();
     const ctx = { id: `${process.pid}.${++lastExchange}`, at: Date.now(), method, path, headers, chunks: [], size: 0, noted: false, kept: false, sent: [], turn, sites: [], traced: 0 };
@@ -866,7 +866,10 @@ function makeRecorder({ row, rulesFile, onTurn = () => {}, onEnded = () => {} })
       const told = asked && typeof asked === "object" && !EMBEDDING.test(path) ? toldOf(asked, ctx ? new Set([...heard, ...heardIn(ctx)]) : heard) : {};
       tied.done(ctx);
       if (ctx && ctx.late) {
-        if (!EMBEDDING.test(path)) ctx.said = { status, words: replyText(reply.events).slice(0, MODEL_WORDS), at: Date.now() };
+        if (!EMBEDDING.test(path)) {
+          ctx.said = { status, words: replyText(reply.events).slice(0, MODEL_WORDS), at: Date.now() };
+          ctx.saidAll = [...(ctx.saidAll || []), ctx.said.words].slice(-SAID_KEPT);
+        }
         settle(ctx);
       }
       row({ call: { at: Date.now(), ms: Date.now() - t0, host: String(host).replace(/:443$/, ""), model: String(askedFor(path, sent) || reply.model || "").slice(0, 160), ...answeredBy(askedFor(path, sent), reply.model), status, ...(reply.tokens || { promptTokens: 0, cachedTokens: 0, completionTokens: 0 }), usage: Boolean(reply.tokens), ...(EMBEDDING.test(path) ? { embedding: true } : {}), ...(ctx ? { ex: ctx.id } : {}), ...(ctx && (ctx.kept || ctx.turn) ? { reply: replyText(reply.events).slice(0, MODEL_WORDS) } : {}), ...(turn ? { turn } : {}), ...(rules ? { rules } : {}), ...(tools ? { tools } : {}), ...(called ? { called } : {}), ...(passages ? { passages } : {}), ...(caller ? { caller } : {}), ...(reply.limit ? { limit: { cap: capOf(asked), reasoning: reply.reasoning } } : {}), ...told } });

package/local.mjs

@@ -12,7 +12,7 @@
 // on with the key this connect leaves in ~/.cortad. Nothing here touches git. Your environment
 // never leaves this machine. Ctrl-C ends everything.
 
-import { storesOf } from "./lib/stores.mjs";
+import { storesOf, withoutStoreSettings } from "./lib/stores.mjs";
 import { notAnApp, withoutStores } from "./lib/app-port.mjs";
 import { holdsKeys, secretEnvValues } from "./lib/keys.mjs";
 import { spawn, execFile, execFileSync } from "node:child_process";
@@ -133,15 +133,10 @@ if (viaToken) await holdProject();
 
 // ---- what leaves the machine: the source git would commit, and nothing git is told to ignore
 const SKIP_DIR = /^(node_modules|\.git|dist|build|out|coverage|vendor|venv|\.venv|env|target|tmp|\.next|\.nuxt|\.turbo|\.cache|__pycache__|\.terraform|\.wrangler|\.svelte-kit|\.output|\.parcel-cache|\.idea|\.vscode|secrets?|\.secrets?)$/i;
-// Env files by any name (scripts/books.env is one), keys, and data rather than code.
-const SKIP_FILE = /^\.env(\..*)?$|^\.envrc$|\.env$|\.(pem|key|p12|pfx|jks|keystore|sqlite|sqlite3|db|log|lock|map|zip|tar|gz|tgz|7z|rar|png|jpe?g|gif|webp|ico|svg|mp3|mp4|wav|mov|pdf|woff2?|ttf|otf|eot|bin|exe|dll|so|dylib|wasm|onnx|pt|pth|safetensors|parquet|csv|tsv|jsonl|ndjson|xlsx?|numbers|DS_Store)$/i;
-const MAX_FILE = 1_000_000;
-// A JSON or YAML file this large is a dataset (exports, catalogues, fixtures), not code or a prompt,
-// and datasets are where projects keep what they would never paste into a chat.
-const MAX_DATA = 200_000;
-const DATA_FILE = /\.(json|ya?ml)$/i;
-const MANIFEST_FILE = /^(package|tsconfig|composer|app|manifest)\.json$/i;
-const MAX_TOTAL = 80_000_000;
+// Env files by any name (scripts/books.env is one), keys, credentials, media and compiled output.
+const SKIP_FILE = /^\.env(\..*)?$|^\.envrc$|\.env$|\.(pem|key|p12|pfx|jks|keystore|sqlite|sqlite3|db|log|lock|map|zip|tar|gz|tgz|7z|rar|png|jpe?g|gif|webp|ico|svg|mp3|mp4|wav|mov|pdf|woff2?|ttf|otf|eot|bin|exe|dll|so|dylib|wasm|onnx|pt|pth|safetensors|numbers|DS_Store)$/i;
+const MAX_FILE = 25_000_000;
+const MAX_TOTAL = 256_000_000;
 const ENV_FILE = /^\.env(\.(local|staging|stage|development|dev|test|example|sample))?$/;
 
 function walk(dir, depth, out, envs, total) {
@@ -156,7 +151,6 @@ function walk(dir, depth, out, envs, total) {
     let size;
     try { size = statSync(full).size; } catch { continue; }
     if (size > MAX_FILE || total + size > MAX_TOTAL) continue;
-    if (size > MAX_DATA && DATA_FILE.test(e.name) && !MANIFEST_FILE.test(e.name)) continue;
     if (!shareable(relative(root, full), false)) continue;
     out.push(relative(root, full));
     total += size;
@@ -918,7 +912,7 @@ async function freed(cmd, lifted) {
 // was copied, started, or left where trials write into it is told in the terminal here and to the
 // run with the app (announce), so status says it beside Run and the report keeps it.
 const keeping = makeKeeping({
-  root, work, ledgerFile: join(homeOf(project), "made.json"), onPath, say, hold,
+  root, work, ledgerFile: join(homeOf(project), "made.json"), onPath, say, hold, typed: Boolean(flag("--start")),
   launch: (plan) => sidecar(plan.cmd, plan.cwd),
   connections: () => capture?.connections() ?? [],
   settings: () => {
@@ -947,7 +941,10 @@ async function start(waitMs, tries = 3) {
   loaded = note;
   const copies = await keeping.env(true);
   await allowOutbound();
-  const base = { ...envExports(envFiles), ...process.env, ...lifted, ...copies };
+  // A start command the person typed says which databases the app uses: the store settings of their
+  // settings files are left for the app to load, since handed over here they outrank the
+  // environment that command chose, and no copy is pointed at.
+  const base = flag("--start") ? { ...withoutStoreSettings(envExports(envFiles)), ...process.env, ...lifted } : { ...envExports(envFiles), ...process.env, ...lifted, ...copies };
   child = spawnTied(cmd, { cwd: appDir, env: { ...base, ...(proxy ? proxy.env(base) : capture ? capture.env(process.env) : {}), FORCE_COLOR: "0", ...(pinned.bin ? { PATH: `${pinned.bin}:${process.env.PATH ?? ""}` } : {}) } });
   const mine = child;
   become("starting", { app: mine.pid ?? null });
@@ -1150,8 +1147,8 @@ if (explain) {
     `talks to        ${origin.origin}, localhost (your app's port only), and your own model providers, to ask each which models your key can use`,
     `would send      ${files.length} source files, ${Math.round(bytes / 1024)} KB, once${listed ? " (what git would commit)" : ""}`,
     ...(kept.length ? [`kept here       ${kept.length} file${kept.length === 1 ? " that holds" : "s that hold"} keys: ${kept.slice(0, 6).join(", ")}${kept.length > 6 ? ", ..." : ""}`] : []),
-    ...(left.length ? [`also not sent   ${left.length} tracked data, media or lock file${left.length === 1 ? "" : "s"}: ${left.slice(0, 6).join(", ")}${left.length > 6 ? ", ..." : ""}`] : []),
-    `not sent        anything git ignores, env files (${envFiles.length} here: ${envFiles.slice(0, 6).map(rel).join(", ") || "none"}), key files, data files, node_modules, .git`,
+    ...(left.length ? [`also not sent   ${left.length} tracked media or lock file${left.length === 1 ? "" : "s"}: ${left.slice(0, 6).join(", ")}${left.length > 6 ? ", ..." : ""}`] : []),
+    `not sent        anything git ignores, env files (${envFiles.length} here: ${envFiles.slice(0, 6).map(rel).join(", ") || "none"}), key files, node_modules, .git`,
     `env files       values read here only: to hide them in replies, to sign in a test account, and to ask your providers what your keys reach. Variable names and whether a switch is on or off go up; no value does`,
     `would start     ${flag("--port") ? `nothing: uses your app on port ${flag("--port")}` : plan?.cmd ? `${plan.cmd}   (in ${rel(plan.cwd)})` : plan?.noServer ? "nothing: this repository has no server to run" : "asks you how your app starts"}`,
     `would raise     ${flag("--port") ? "nothing: your app's own request limits stay as they are" : `${Object.keys(liftedLimits(envFiles, files.map((f) => join(root, f)))).join(", ") || "no request limits found"}   (only after your yes, for this session only)`}`,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cortad",
-  "version": "0.3.2",
+  "version": "0.3.4",
   "description": "Connects the AI app on your machine to Cortad for test conversations, and gives your coding agent the MCP and skill to run them. No dependencies.",
   "bin": {
     "cortad": "local.mjs"