cortad 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/door.mjs +5 -2
- package/lib/identities.mjs +15 -3
- package/lib/mint.mjs +18 -2
- package/lib/pyhook/sitecustomize.py +211 -16
- package/lib/replay.mjs +72 -6
- package/lib/sample.mjs +218 -0
- package/lib/start.mjs +69 -14
- package/lib/trace.cjs +184 -12
- package/local.mjs +179 -22
- package/package.json +2 -1
package/lib/door.mjs
CHANGED
|
@@ -14,9 +14,12 @@ const ENV_FILE = /^\.env(\..*)?$/;
|
|
|
14
14
|
const KEY_FILE = /^(?:id_(?:rsa|ed25519|ecdsa).*|.*\.(?:pem|key|p12|pfx|jks|keystore)|\.npmrc|\.netrc|\.pypirc)$/;
|
|
15
15
|
const CLOSED_DIR = new Set([".git", "node_modules", ".ssh", ".aws", ".gnupg"]);
|
|
16
16
|
|
|
17
|
-
|
|
17
|
+
// `owner` names whose checkpoints these are: the account and connection the server attached this
|
|
18
|
+
// folder to. Keyed by the folder alone, a second account on the same folder was shown the first
|
|
19
|
+
// account's edits as its own.
|
|
20
|
+
export function openDoor(root, { store = join(homedir(), ".cortad", "checkpoints"), owner = "" } = {}) {
|
|
18
21
|
const realRoot = realpathSync(root);
|
|
19
|
-
const dir = join(store, sha(realRoot).slice(0, 16));
|
|
22
|
+
const dir = join(store, sha(`${owner}\0${realRoot}`).slice(0, 16));
|
|
20
23
|
const blobs = join(dir, "blobs");
|
|
21
24
|
const journalFile = join(dir, "journal.json");
|
|
22
25
|
mkdirSync(blobs, { recursive: true });
|
package/lib/identities.mjs
CHANGED
|
@@ -40,10 +40,12 @@ export function identityCurlArg(as) {
|
|
|
40
40
|
|
|
41
41
|
// Every string the recipes carry reaches the program as base64 JSON, never as source, and only
|
|
42
42
|
// after the charset gate: a route or a role is read from a repository.
|
|
43
|
-
export function identityScript(recipes, target) {
|
|
43
|
+
export function identityScript(recipes, target, accounts = {}) {
|
|
44
44
|
const safe = recipes.filter(safeRecipe).slice(0, 12);
|
|
45
45
|
const headers = Object.fromEntries(Object.entries(target.headers ?? {}).filter(([k, v]) => HEADER.test(k) && typeof v === "string" && v.length <= 300 && !/[\r\n]/.test(v)));
|
|
46
|
-
const
|
|
46
|
+
const kept = Object.fromEntries(Object.entries(accounts).filter(([role, a]) => ROLE.test(role) && a
|
|
47
|
+
&& ["username", "email", "password", "name"].every((f) => typeof a[f] === "string" && a[f].length <= 120 && !/[\r\n]/.test(a[f]))));
|
|
48
|
+
const plan = { envPath: /^[\w./-]+$/.test(target.envPath) ? target.envPath : "/workspace/repo/.env", base: BASE.test(target.base) ? target.base : "http://127.0.0.1:0", headers, recipes: safe, accounts: kept };
|
|
47
49
|
return PROGRAM
|
|
48
50
|
.replace("__PLAN__", Buffer.from(JSON.stringify(plan), "utf8").toString("base64"))
|
|
49
51
|
.replace("__OUT__", IDENTITY_OUT)
|
|
@@ -68,7 +70,10 @@ PLAN = json.loads(base64.b64decode("__PLAN__").decode("utf-8"))
|
|
|
68
70
|
OUT = "__OUT__"
|
|
69
71
|
FILE = "__FILE__"
|
|
70
72
|
TTL = 6 * 3600
|
|
71
|
-
|
|
73
|
+
# Digits only: an account column is an integer in Strapi, Django and Rails, and a word here makes
|
|
74
|
+
# the database itself raise. Nonerds crashed its whole process on "cortad-test-user". Digits are a
|
|
75
|
+
# valid string id too, so this one value fits both kinds of account table.
|
|
76
|
+
TEST = "9000001"
|
|
72
77
|
# A dev server compiles its auth route on the first request: better-chatbot's sign-in timed out at 10s
|
|
73
78
|
# and the caller was filed unreachable. The whole program stays inside the step's two minutes.
|
|
74
79
|
DEADLINE = time.time() + 100
|
|
@@ -96,7 +101,11 @@ def slug(role):
|
|
|
96
101
|
return re.sub(r"[^a-z0-9-]", "-", role)
|
|
97
102
|
|
|
98
103
|
|
|
104
|
+
# One account per role, made once and signed into on every later run: a fresh account per run left
|
|
105
|
+
# 1,600 active test logins in one app's own account store.
|
|
99
106
|
def gen(role):
|
|
107
|
+
if role in PLAN.get("accounts", {}):
|
|
108
|
+
return PLAN["accounts"][role]
|
|
100
109
|
tag = secrets.token_hex(4)
|
|
101
110
|
return {
|
|
102
111
|
"username": "cortad-%s-%s" % (slug(role), tag),
|
|
@@ -206,6 +215,9 @@ def walk(steps, recipe, env, g, role_value):
|
|
|
206
215
|
return {"status": "unreachable", "note": text}
|
|
207
216
|
if status >= 400 and unset:
|
|
208
217
|
return {"status": "absent", "note": unset}
|
|
218
|
+
# The account this role was given on an earlier run is still there: go on and sign in.
|
|
219
|
+
if status in (400, 409, 422) and step is not steps[-1] and re.search(r"exist|taken|already|duplicate|in use|registered", text or "", re.I):
|
|
220
|
+
continue
|
|
209
221
|
if status >= 400:
|
|
210
222
|
said = re.sub(r"\s+", " ", re.sub(r"<[^>]+>", "", text or "")).strip()[:160]
|
|
211
223
|
return {"status": "refused", "note": "HTTP %d at %s%s" % (status, step["route"], (": " + said) if said else "")}
|
package/lib/mint.mjs
CHANGED
|
@@ -5,8 +5,9 @@
|
|
|
5
5
|
// a role arrives carrying a marker; the real header is put in its place here, so neither a secret
|
|
6
6
|
// nor a token ever leaves this machine.
|
|
7
7
|
import { execFile } from "node:child_process";
|
|
8
|
-
import { createSign } from "node:crypto";
|
|
8
|
+
import { createHmac, createSign, randomBytes } from "node:crypto";
|
|
9
9
|
import { existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
10
|
+
import { homedir } from "node:os";
|
|
10
11
|
import { basename, dirname, join } from "node:path";
|
|
11
12
|
import { promisify } from "node:util";
|
|
12
13
|
import { identityScript, parseIdentityReport } from "./identities.mjs";
|
|
@@ -17,6 +18,18 @@ const ROLE = /^[a-z][a-z0-9_-]{0,30}(?::[a-z][a-z0-9_-]{0,30})?$/;
|
|
|
17
18
|
const slug = (role) => role.replace(/[^a-z0-9-]/g, "-");
|
|
18
19
|
const TEST_UID = "brainsless-test-user";
|
|
19
20
|
|
|
21
|
+
// The account each role signs in as, the same on every run from this folder: derived from a key that
|
|
22
|
+
// never leaves this machine, so no server holds a password to any of their accounts.
|
|
23
|
+
export function accountsFor(root, roles, keyFile = join(homedir(), ".cortad", "identity.key")) {
|
|
24
|
+
if (!existsSync(keyFile)) { mkdirSync(dirname(keyFile), { recursive: true, mode: 0o700 }); writeFileSync(keyFile, randomBytes(32), { mode: 0o600 }); }
|
|
25
|
+
const key = readFileSync(keyFile);
|
|
26
|
+
const mac = (what) => createHmac("sha256", key).update(`${what}\0${root}`).digest("base64url");
|
|
27
|
+
return Object.fromEntries(roles.filter((role) => ROLE.test(role)).map((role) => {
|
|
28
|
+
const username = `cortad-${slug(role)}-${mac(`name\0${role}`).replace(/[^a-z0-9]/gi, "").slice(0, 8).toLowerCase()}`;
|
|
29
|
+
return [role, { username, email: `${username}@example.invalid`, password: `${mac(`pw\0${role}`).slice(0, 24)}A1!`, name: `Cortad test ${role}` }];
|
|
30
|
+
}));
|
|
31
|
+
}
|
|
32
|
+
|
|
20
33
|
// Who this machine signs in as, whatever a recipe asks: an account made for the session, or a test
|
|
21
34
|
// id that belongs to nobody. Never a person who already has an account, never an admin, and never a
|
|
22
35
|
// key the app does not already hand to every browser. The recipes come from the cloud; these rules
|
|
@@ -130,7 +143,10 @@ export function makeIdentities({ root, work, envFiles, sourceFiles = () => [], s
|
|
|
130
143
|
const refused = new Map();
|
|
131
144
|
for (const r of recipes) { const why = refusal(r, merged, (value) => inSource(root, sourceFiles(), value)); if (why && typeof r?.role === "string") refused.set(r.role, why); }
|
|
132
145
|
const forProgram = recipes.filter((r) => !refused.has(r?.role) && (r?.kind !== "firebase" || emulated));
|
|
133
|
-
|
|
146
|
+
// The program signs in once per role value ("staff" as sales, support, marketing), each named
|
|
147
|
+
// role:value; an account made for "staff" alone was never the one it asked for.
|
|
148
|
+
const accounts = accountsFor(root, forProgram.flatMap((r) => (Array.isArray(r.roles) && r.roles.length ? r.roles.map((v) => `${r.role}:${v}`) : [r.role])));
|
|
149
|
+
const script = identityScript(forProgram, { envPath: envs().path, base: `http://127.0.0.1:${port}`, headers: body?.headers ?? {} }, accounts).split("/tmp/bl-identit").join(join(dir, "bl-identit"));
|
|
134
150
|
const file = join(dir, "mint.py");
|
|
135
151
|
writeFileSync(file, script, { mode: 0o600 });
|
|
136
152
|
const ran = forProgram.length ? await exec("python3", [file], { cwd: root, timeout: 115_000, maxBuffer: 1 << 20 }).then(() => true, (e) => e?.code !== "ENOENT") : true;
|
|
@@ -22,7 +22,7 @@ def _install():
|
|
|
22
22
|
ctx = contextvars.ContextVar("cortad_request", default=None)
|
|
23
23
|
limit = 65536
|
|
24
24
|
said = set()
|
|
25
|
-
model_host = re.compile(r"(?:^|\.)(?:openai\.com|anthropic\.com|fireworks\.ai|openrouter\.ai|groq\.com|mistral\.ai|together\.xyz|together\.ai|deepseek\.com|cohere\.ai|cohere\.com|perplexity\.ai|x\.ai|openai\.azure\.com|cognitiveservices\.azure\.com|replicate\.com|huggingface\.co|cerebras\.ai|deepinfra\.com|novita\.ai|moonshot\.cn|dashscope\.aliyuncs\.com|bigmodel\.cn)$", re.I)
|
|
25
|
+
model_host = re.compile(r"(?:^|\.)(?:openai\.com|anthropic\.com|fireworks\.ai|openrouter\.ai|groq\.com|mistral\.ai|together\.xyz|together\.ai|deepseek\.com|cohere\.ai|cohere\.com|perplexity\.ai|x\.ai|openai\.azure\.com|cognitiveservices\.azure\.com|replicate\.com|huggingface\.co|cerebras\.ai|deepinfra\.com|novita\.ai|moonshot\.cn|dashscope\.aliyuncs\.com|bigmodel\.cn|ai-gateway\.vercel\.sh|gateway\.ai\.cloudflare\.com|helicone\.ai|portkey\.ai)$", re.I)
|
|
26
26
|
model_path = re.compile(r"/(?:chat/completions|completions|responses|messages|embeddings)$|:(?:generateContent|streamGenerateContent)|/invoke(?:-with-response-stream)?$|/api/(?:chat|generate)$", re.I)
|
|
27
27
|
|
|
28
28
|
def write(row):
|
|
@@ -70,6 +70,88 @@ def _install():
|
|
|
70
70
|
write({"at": int(time.time() * 1000), "method": req["method"], "path": req["path"], "headers": req["headers"],
|
|
71
71
|
"body": b"".join(req["chunks"]).decode("utf-8", "replace")[:limit], "sent": text(body)})
|
|
72
72
|
|
|
73
|
+
# The meter: one row per model call your app makes, with the provider's own token counts and
|
|
74
|
+
# never the reply's text.
|
|
75
|
+
reply_max = 8 * 1024 * 1024
|
|
76
|
+
|
|
77
|
+
def number(v):
|
|
78
|
+
return v if isinstance(v, int) and not isinstance(v, bool) and v > 0 else 0
|
|
79
|
+
|
|
80
|
+
def tokens_of(u):
|
|
81
|
+
if not isinstance(u, dict):
|
|
82
|
+
return None
|
|
83
|
+
if isinstance(u.get("tokens"), dict):
|
|
84
|
+
return tokens_of(u["tokens"])
|
|
85
|
+
if "prompt_tokens" in u:
|
|
86
|
+
return {"promptTokens": number(u.get("prompt_tokens")), "cachedTokens": number((u.get("prompt_tokens_details") or {}).get("cached_tokens")), "completionTokens": number(u.get("completion_tokens"))}
|
|
87
|
+
if "input_tokens" in u and isinstance(u.get("input_tokens_details"), dict):
|
|
88
|
+
return {"promptTokens": number(u.get("input_tokens")), "cachedTokens": number(u["input_tokens_details"].get("cached_tokens")), "completionTokens": number(u.get("output_tokens"))}
|
|
89
|
+
if "input_tokens" in u:
|
|
90
|
+
read = number(u.get("cache_read_input_tokens"))
|
|
91
|
+
return {"promptTokens": number(u.get("input_tokens")) + read + number(u.get("cache_creation_input_tokens")), "cachedTokens": read, "completionTokens": number(u.get("output_tokens"))}
|
|
92
|
+
if "inputTokens" in u:
|
|
93
|
+
read = number(u.get("cacheReadInputTokens"))
|
|
94
|
+
return {"promptTokens": number(u.get("inputTokens")) + read + number(u.get("cacheWriteInputTokens")), "cachedTokens": read, "completionTokens": number(u.get("outputTokens"))}
|
|
95
|
+
if "promptTokenCount" in u:
|
|
96
|
+
return {"promptTokens": number(u.get("promptTokenCount")), "cachedTokens": number(u.get("cachedContentTokenCount")), "completionTokens": number(u.get("candidatesTokenCount"))}
|
|
97
|
+
return None
|
|
98
|
+
|
|
99
|
+
def parsed(raw):
|
|
100
|
+
try:
|
|
101
|
+
return json.loads(raw)
|
|
102
|
+
except (TypeError, ValueError):
|
|
103
|
+
return None
|
|
104
|
+
|
|
105
|
+
def read_reply(kind, raw):
|
|
106
|
+
if "event-stream" in (kind or "").lower():
|
|
107
|
+
events = [parsed(line[5:].strip()) for line in raw.split("\n") if line.startswith("data:")]
|
|
108
|
+
else:
|
|
109
|
+
body = parsed(raw)
|
|
110
|
+
events = body if isinstance(body, list) else [body]
|
|
111
|
+
usage, model = {}, None
|
|
112
|
+
for e in events:
|
|
113
|
+
if not isinstance(e, dict):
|
|
114
|
+
continue
|
|
115
|
+
msg = e["message"] if isinstance(e.get("message"), dict) else {}
|
|
116
|
+
resp = e["response"] if isinstance(e.get("response"), dict) else {}
|
|
117
|
+
part = e.get("usage") or msg.get("usage") or resp.get("usage") or e.get("usageMetadata")
|
|
118
|
+
if isinstance(part, dict):
|
|
119
|
+
usage.update(part)
|
|
120
|
+
model = e.get("model") or msg.get("model") or resp.get("model") or e.get("modelVersion") or model
|
|
121
|
+
return tokens_of(usage or None), model
|
|
122
|
+
|
|
123
|
+
def asked_for(url, sent):
|
|
124
|
+
body = parsed(sent) if isinstance(sent, str) else None
|
|
125
|
+
if isinstance(body, dict) and isinstance(body.get("model"), str):
|
|
126
|
+
return body["model"]
|
|
127
|
+
m = re.search(r"/models/([^/:]+):|/model/([^/]+)/(?:invoke|converse)", urlsplit(str(url)).path or "")
|
|
128
|
+
return (m.group(1) or m.group(2)) if m else ""
|
|
129
|
+
|
|
130
|
+
def unpacked(raw, encoding):
|
|
131
|
+
import zlib
|
|
132
|
+
try:
|
|
133
|
+
e = (encoding or "").lower()
|
|
134
|
+
if e == "gzip":
|
|
135
|
+
raw = zlib.decompress(raw, 16 + zlib.MAX_WBITS)
|
|
136
|
+
elif e == "deflate":
|
|
137
|
+
raw = zlib.decompress(raw)
|
|
138
|
+
elif e == "br":
|
|
139
|
+
return ""
|
|
140
|
+
return raw.decode("utf-8", "replace")
|
|
141
|
+
except Exception:
|
|
142
|
+
return ""
|
|
143
|
+
|
|
144
|
+
def meter(url, sent, status, kind, raw):
|
|
145
|
+
try:
|
|
146
|
+
tokens, model = read_reply(kind, (raw or "")[:reply_max])
|
|
147
|
+
parts = urlsplit(str(url))
|
|
148
|
+
row = {"at": int(time.time() * 1000), "host": parts.netloc.replace(":443", ""), "model": str(asked_for(url, sent) or model or "")[:160],
|
|
149
|
+
"status": int(status or 0), "usage": tokens is not None}
|
|
150
|
+
row.update(tokens or {"promptTokens": 0, "cachedTokens": 0, "completionTokens": 0})
|
|
151
|
+
write({"call": row})
|
|
152
|
+
except Exception:
|
|
153
|
+
pass
|
|
154
|
+
|
|
73
155
|
def started(method, path, headers):
|
|
74
156
|
return {"method": method, "path": path, "headers": headers, "chunks": [], "size": 0, "noted": False}
|
|
75
157
|
|
|
@@ -175,6 +257,8 @@ def _install():
|
|
|
175
257
|
out = load(self, *a, **k)
|
|
176
258
|
if not isinstance(self.loaded_app, Asgi):
|
|
177
259
|
self.loaded_app = Asgi(self.loaded_app)
|
|
260
|
+
if isinstance(getattr(self, "port", None), int):
|
|
261
|
+
write({"listen": self.port, "pid": os.getpid()})
|
|
178
262
|
return out
|
|
179
263
|
|
|
180
264
|
module.Config.load = loaded
|
|
@@ -183,6 +267,8 @@ def _install():
|
|
|
183
267
|
run_simple = module.run_simple
|
|
184
268
|
|
|
185
269
|
def run(hostname, port, application, *a, **k):
|
|
270
|
+
if isinstance(port, int):
|
|
271
|
+
write({"listen": port, "pid": os.getpid()})
|
|
186
272
|
return run_simple(hostname, port, wsgi(application), *a, **k)
|
|
187
273
|
|
|
188
274
|
module.run_simple = run
|
|
@@ -195,30 +281,107 @@ def _install():
|
|
|
195
281
|
|
|
196
282
|
module.WSGIHandler.__call__ = called
|
|
197
283
|
|
|
284
|
+
def sent_of(request):
|
|
285
|
+
try:
|
|
286
|
+
return text(request.content)
|
|
287
|
+
except Exception:
|
|
288
|
+
return ""
|
|
289
|
+
|
|
198
290
|
def patch_httpx(module):
|
|
291
|
+
# A streamed reply is read by your app after send returns: its raw bytes are kept as they go
|
|
292
|
+
# past and the row is written when the stream closes. A read reply is metered at once.
|
|
293
|
+
class Kept:
|
|
294
|
+
def __init__(self, stream, done):
|
|
295
|
+
self.stream, self.done, self.got, self.size = stream, done, [], 0
|
|
296
|
+
|
|
297
|
+
def keep(self, chunk):
|
|
298
|
+
if self.size < reply_max:
|
|
299
|
+
self.got.append(bytes(chunk))
|
|
300
|
+
self.size += len(chunk)
|
|
301
|
+
|
|
302
|
+
def finish(self):
|
|
303
|
+
done, self.done = self.done, None
|
|
304
|
+
if done:
|
|
305
|
+
done(b"".join(self.got))
|
|
306
|
+
|
|
307
|
+
class SyncKept(module.SyncByteStream, Kept):
|
|
308
|
+
def __iter__(self):
|
|
309
|
+
for chunk in self.stream:
|
|
310
|
+
self.keep(chunk)
|
|
311
|
+
yield chunk
|
|
312
|
+
|
|
313
|
+
def close(self):
|
|
314
|
+
try:
|
|
315
|
+
self.stream.close()
|
|
316
|
+
finally:
|
|
317
|
+
self.finish()
|
|
318
|
+
|
|
319
|
+
class AsyncKept(module.AsyncByteStream, Kept):
|
|
320
|
+
async def __aiter__(self):
|
|
321
|
+
async for chunk in self.stream:
|
|
322
|
+
self.keep(chunk)
|
|
323
|
+
yield chunk
|
|
324
|
+
|
|
325
|
+
async def aclose(self):
|
|
326
|
+
try:
|
|
327
|
+
await self.stream.aclose()
|
|
328
|
+
finally:
|
|
329
|
+
self.finish()
|
|
330
|
+
|
|
331
|
+
def watch(request, response, sent):
|
|
332
|
+
if not is_model_call(request.url):
|
|
333
|
+
return response
|
|
334
|
+
kind = response.headers.get("content-type", "")
|
|
335
|
+
if getattr(response, "_content", None) is not None:
|
|
336
|
+
meter(request.url, sent, response.status_code, kind, response.content.decode("utf-8", "replace"))
|
|
337
|
+
return response
|
|
338
|
+
done = lambda raw: meter(request.url, sent, response.status_code, kind, unpacked(raw, response.headers.get("content-encoding")))
|
|
339
|
+
stream = response.stream
|
|
340
|
+
if hasattr(stream, "__aiter__") and hasattr(stream, "aclose"):
|
|
341
|
+
response.stream = AsyncKept(stream, done)
|
|
342
|
+
elif hasattr(stream, "__iter__"):
|
|
343
|
+
response.stream = SyncKept(stream, done)
|
|
344
|
+
else:
|
|
345
|
+
meter(request.url, sent, response.status_code, kind, "")
|
|
346
|
+
return response
|
|
347
|
+
|
|
199
348
|
for cls in (module.Client, module.AsyncClient):
|
|
200
349
|
send = cls.send
|
|
201
350
|
if cls is module.Client:
|
|
202
351
|
def sync_send(self, request, *a, _send=send, **k):
|
|
352
|
+
sent = sent_of(request)
|
|
353
|
+
try:
|
|
354
|
+
note(request.url, sent)
|
|
355
|
+
except Exception:
|
|
356
|
+
pass
|
|
357
|
+
try:
|
|
358
|
+
response = _send(self, request, *a, **k)
|
|
359
|
+
except Exception:
|
|
360
|
+
if is_model_call(request.url):
|
|
361
|
+
meter(request.url, sent, 0, "", "")
|
|
362
|
+
raise
|
|
203
363
|
try:
|
|
204
|
-
|
|
364
|
+
return watch(request, response, sent)
|
|
205
365
|
except Exception:
|
|
206
|
-
|
|
207
|
-
note(request.url, None)
|
|
208
|
-
except Exception:
|
|
209
|
-
pass
|
|
210
|
-
return _send(self, request, *a, **k)
|
|
366
|
+
return response
|
|
211
367
|
cls.send = sync_send
|
|
212
368
|
else:
|
|
213
369
|
async def async_send(self, request, *a, _send=send, **k):
|
|
370
|
+
sent = sent_of(request)
|
|
371
|
+
try:
|
|
372
|
+
note(request.url, sent)
|
|
373
|
+
except Exception:
|
|
374
|
+
pass
|
|
375
|
+
try:
|
|
376
|
+
response = await _send(self, request, *a, **k)
|
|
377
|
+
except Exception:
|
|
378
|
+
if is_model_call(request.url):
|
|
379
|
+
meter(request.url, sent, 0, "", "")
|
|
380
|
+
raise
|
|
214
381
|
try:
|
|
215
|
-
|
|
382
|
+
return watch(request, response, sent)
|
|
216
383
|
except Exception:
|
|
217
|
-
|
|
218
|
-
note(request.url, None)
|
|
219
|
-
except Exception:
|
|
220
|
-
pass
|
|
221
|
-
return await _send(self, request, *a, **k)
|
|
384
|
+
return response
|
|
222
385
|
cls.send = async_send
|
|
223
386
|
|
|
224
387
|
def patch_requests(module):
|
|
@@ -229,7 +392,15 @@ def _install():
|
|
|
229
392
|
note(request.url, request.body)
|
|
230
393
|
except Exception:
|
|
231
394
|
pass
|
|
232
|
-
|
|
395
|
+
response = send(self, request, *a, **k)
|
|
396
|
+
try:
|
|
397
|
+
if is_model_call(request.url):
|
|
398
|
+
# A streamed reply is counted as a call whose counts were not read.
|
|
399
|
+
raw = response.content.decode("utf-8", "replace") if getattr(response, "_content_consumed", False) else ""
|
|
400
|
+
meter(request.url, text(request.body), response.status_code, response.headers.get("content-type", ""), raw)
|
|
401
|
+
except Exception:
|
|
402
|
+
pass
|
|
403
|
+
return response
|
|
233
404
|
|
|
234
405
|
module.Session.send = sent
|
|
235
406
|
|
|
@@ -237,13 +408,37 @@ def _install():
|
|
|
237
408
|
request = module.ClientSession._request
|
|
238
409
|
|
|
239
410
|
async def requested(self, method, str_or_url, *a, **k):
|
|
411
|
+
body = k.get("json") if k.get("json") is not None else k.get("data")
|
|
240
412
|
try:
|
|
241
|
-
note(str_or_url,
|
|
413
|
+
note(str_or_url, body)
|
|
242
414
|
except Exception:
|
|
243
415
|
pass
|
|
244
|
-
|
|
416
|
+
response = await request(self, method, str_or_url, *a, **k)
|
|
417
|
+
if is_model_call(str_or_url):
|
|
418
|
+
response._cortad = (str_or_url, text(body))
|
|
419
|
+
return response
|
|
420
|
+
|
|
421
|
+
# Metered when your app reads the reply, or, for one it streams, when the reply is let go.
|
|
422
|
+
read, release = module.ClientResponse.read, module.ClientResponse.release
|
|
423
|
+
|
|
424
|
+
async def read_kept(self, *a, **k):
|
|
425
|
+
raw = await read(self, *a, **k)
|
|
426
|
+
call = getattr(self, "_cortad", None)
|
|
427
|
+
if call:
|
|
428
|
+
self._cortad = None
|
|
429
|
+
meter(call[0], call[1], self.status, self.headers.get("content-type", ""), (raw or b"").decode("utf-8", "replace"))
|
|
430
|
+
return raw
|
|
431
|
+
|
|
432
|
+
def release_kept(self, *a, **k):
|
|
433
|
+
call = getattr(self, "_cortad", None)
|
|
434
|
+
if call:
|
|
435
|
+
self._cortad = None
|
|
436
|
+
meter(call[0], call[1], self.status, self.headers.get("content-type", ""), "")
|
|
437
|
+
return release(self, *a, **k)
|
|
245
438
|
|
|
246
439
|
module.ClientSession._request = requested
|
|
440
|
+
module.ClientResponse.read = read_kept
|
|
441
|
+
module.ClientResponse.release = release_kept
|
|
247
442
|
|
|
248
443
|
exact = {"uvicorn.config": patch_uvicorn, "werkzeug.serving": patch_werkzeug, "django.core.handlers.wsgi": patch_django,
|
|
249
444
|
"requests.sessions": patch_requests, "aiohttp.client": patch_aiohttp}
|
package/lib/replay.mjs
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
// The other half of lib/trace.cjs: the command's side. It hands the hook to the app it starts,
|
|
2
2
|
// reads what the hook wrote, keeps the sign-in that request carried on this machine, and tells the
|
|
3
3
|
// cloud only what it needs to ask again: the route, the method and the body.
|
|
4
|
-
import { readFileSync, statSync } from "node:fs";
|
|
4
|
+
import { copyFileSync, mkdtempSync, readFileSync, statSync } from "node:fs";
|
|
5
|
+
import { tmpdir } from "node:os";
|
|
5
6
|
import { dirname, join } from "node:path";
|
|
6
7
|
import { fileURLToPath } from "node:url";
|
|
7
8
|
|
|
@@ -17,12 +18,19 @@ export function makeCapture({ work, keepSecret, onDoor }) {
|
|
|
17
18
|
let read = 0;
|
|
18
19
|
let held = null;
|
|
19
20
|
let alive = false;
|
|
20
|
-
//
|
|
21
|
-
//
|
|
22
|
-
|
|
21
|
+
// The ports a hooked process listens on. Loading is not listening: turbo's own Node launcher loads
|
|
22
|
+
// the hook, and databuddy was told we would see its messages while its API on Bun carried none.
|
|
23
|
+
const ports = new Set();
|
|
24
|
+
// Bun reads BUN_OPTIONS and splits it on spaces, quotes included, and only the `--preload=` form
|
|
25
|
+
// leaves `bun run <script>` working. A hook path with a space in it is copied to one without.
|
|
26
|
+
const bunHook = /\s/.test(HOOK) ? (() => { const at = join(mkdtempSync(join(tmpdir(), "cortad-")), "trace.cjs"); copyFileSync(HOOK, at); return at; })() : HOOK;
|
|
27
|
+
// The hooks are handed to whatever is started: a Node app loads the first, a Bun app the same file
|
|
28
|
+
// through its own variable, a Python app the second, and each ignores the others' variables.
|
|
29
|
+
// ponytail: Node, Bun and Python. Go, Ruby, Java and PHP apps are asked for their route on the screen.
|
|
23
30
|
const env = (base) => ({
|
|
24
31
|
CORTAD_TRACE_FILE: file,
|
|
25
32
|
NODE_OPTIONS: `${base.NODE_OPTIONS ?? ""} --require ${JSON.stringify(HOOK)}`.trim(),
|
|
33
|
+
BUN_OPTIONS: `${base.BUN_OPTIONS ?? ""} --preload=${bunHook}`.trim(),
|
|
26
34
|
PYTHONPATH: [PYHOOK, base.PYTHONPATH].filter(Boolean).join(":"),
|
|
27
35
|
});
|
|
28
36
|
|
|
@@ -30,11 +38,15 @@ export function makeCapture({ work, keepSecret, onDoor }) {
|
|
|
30
38
|
let size = 0;
|
|
31
39
|
try { size = statSync(file).size; } catch { return; }
|
|
32
40
|
if (size <= read) return;
|
|
33
|
-
|
|
41
|
+
// Bytes, not characters: a reply in Arabic put the next read in the middle of a line.
|
|
42
|
+
const fresh = readFileSync(file).subarray(read, size).toString("utf8");
|
|
34
43
|
read = size;
|
|
35
44
|
for (const line of fresh.split("\n").filter(Boolean)) {
|
|
36
45
|
let row; try { row = JSON.parse(line); } catch { continue; }
|
|
37
46
|
if (row.hello) { alive = true; continue; }
|
|
47
|
+
if (Number.isInteger(row.listen)) { ports.add(row.listen); continue; }
|
|
48
|
+
if (row.call) { meter.add(row.call); continue; }
|
|
49
|
+
if (row.dep) { meter.dep(row.dep); continue; }
|
|
38
50
|
let body; try { body = JSON.parse(row.body); } catch { continue; }
|
|
39
51
|
if (!body || typeof body !== "object" || typeof row.path !== "string" || !row.path.startsWith("/")) continue;
|
|
40
52
|
const headers = Object.fromEntries(Object.entries(row.headers ?? {}).filter(([k, v]) => !HOP.test(k) && typeof v === "string"));
|
|
@@ -45,5 +57,59 @@ export function makeCapture({ work, keepSecret, onDoor }) {
|
|
|
45
57
|
}
|
|
46
58
|
const timer = setInterval(poll, 700);
|
|
47
59
|
timer.unref();
|
|
48
|
-
return {
|
|
60
|
+
return {
|
|
61
|
+
env,
|
|
62
|
+
headers: () => held?.headers ?? null,
|
|
63
|
+
// A message sent to the app on `port` can be seen arriving. A hook that has not said where it
|
|
64
|
+
// listens (Django under runserver) is taken at its hello, as before.
|
|
65
|
+
watching: (port) => { poll(); return ports.has(port) || (alive && ports.size === 0); },
|
|
66
|
+
alive: () => { poll(); return alive; },
|
|
67
|
+
// What your app spent on its model providers since it started, as the hook saw each call. Null
|
|
68
|
+
// when no hook is in your app (an app this command did not start): absent, never zero.
|
|
69
|
+
usage: () => { poll(); return alive ? meter.report() : null; },
|
|
70
|
+
};
|
|
49
71
|
}
|
|
72
|
+
|
|
73
|
+
// Every model call the hook wrote down, kept here: totals per host and model, and the newest rows.
|
|
74
|
+
const ROWS = 5000;
|
|
75
|
+
const meter = (() => {
|
|
76
|
+
const rows = [];
|
|
77
|
+
const deps = [];
|
|
78
|
+
const count = (v) => (Number.isInteger(v) && v > 0 ? v : 0);
|
|
79
|
+
return {
|
|
80
|
+
add(call) {
|
|
81
|
+
if (!call || typeof call.host !== "string" || !call.host) return;
|
|
82
|
+
rows.push({
|
|
83
|
+
at: count(call.at), host: call.host.slice(0, 253), model: String(call.model ?? "").slice(0, 160), status: count(call.status),
|
|
84
|
+
promptTokens: count(call.promptTokens), cachedTokens: count(call.cachedTokens), completionTokens: count(call.completionTokens),
|
|
85
|
+
usage: call.usage === true,
|
|
86
|
+
});
|
|
87
|
+
if (rows.length > ROWS) rows.splice(0, rows.length - ROWS);
|
|
88
|
+
},
|
|
89
|
+
// A service their settings name: only its setting, host and status travel, never a byte of it.
|
|
90
|
+
dep(d) {
|
|
91
|
+
if (!d || typeof d.env !== "string" || !/^[A-Z_][A-Z0-9_]*$/.test(d.env) || typeof d.host !== "string") return;
|
|
92
|
+
deps.push({ at: count(d.at), env: d.env, host: d.host.slice(0, 253), status: count(d.status), ...(typeof d.code === "string" ? { code: d.code.slice(0, 40) } : {}) });
|
|
93
|
+
if (deps.length > ROWS) deps.splice(0, deps.length - ROWS);
|
|
94
|
+
},
|
|
95
|
+
report() {
|
|
96
|
+
const totals = new Map();
|
|
97
|
+
for (const r of rows) {
|
|
98
|
+
const key = `${r.host}|${r.model}`;
|
|
99
|
+
const t = totals.get(key) ?? { host: r.host, model: r.model, calls: 0, promptTokens: 0, cachedTokens: 0, completionTokens: 0, unmetered: 0 };
|
|
100
|
+
t.calls += 1;
|
|
101
|
+
t.promptTokens += r.promptTokens;
|
|
102
|
+
t.cachedTokens += r.cachedTokens;
|
|
103
|
+
t.completionTokens += r.completionTokens;
|
|
104
|
+
// An answered call the provider sent no counts for: it was spent, and what it cost is not known.
|
|
105
|
+
if (!r.usage && r.status > 0 && r.status < 400) t.unmetered += 1;
|
|
106
|
+
totals.set(key, t);
|
|
107
|
+
}
|
|
108
|
+
return {
|
|
109
|
+
totals: [...totals.values()].sort((a, b) => b.calls - a.calls),
|
|
110
|
+
rows: rows.slice(-200).reverse().map(({ at, host, model, status }) => ({ at, host, model, status })),
|
|
111
|
+
deps: deps.slice(-400).reverse(),
|
|
112
|
+
};
|
|
113
|
+
},
|
|
114
|
+
};
|
|
115
|
+
})();
|