@tokenzero/cli 0.1.1 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/bin/tokenzero.mjs +185 -37
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -13,7 +13,7 @@ Audits are **static heuristics**: they read code, not traffic, and report estima
|
|
|
13
13
|
|
|
14
14
|
## Connecting to the gateway
|
|
15
15
|
|
|
16
|
-
These commands
|
|
16
|
+
These commands talk to the hosted gateway at `https://tokenzero.sh`. To use another deployment, pass `--url` or set `TOKENZERO_URL`.
|
|
17
17
|
|
|
18
18
|
```sh
|
|
19
19
|
npx -y @tokenzero/cli init # audit, create a workspace, write base URLs to .env (asks first)
|
package/bin/tokenzero.mjs
CHANGED
|
@@ -118,7 +118,7 @@ function priceFor(model) {
|
|
|
118
118
|
const key = normalizeModel(model);
|
|
119
119
|
return key ? { key, ...PRICES[key] } : null;
|
|
120
120
|
}
|
|
121
|
-
function costOf(model, u) {
|
|
121
|
+
function costOf(model, u, factor = 1) {
|
|
122
122
|
const p = priceFor(model);
|
|
123
123
|
if (!p) return null;
|
|
124
124
|
const tier = p.longCtx && p.longCtxAt && u.input > p.longCtxAt ? { ...p, ...p.longCtx } : p;
|
|
@@ -126,7 +126,7 @@ function costOf(model, u) {
|
|
|
126
126
|
const written = u.cacheWrite ?? 0;
|
|
127
127
|
const fresh = Math.max(0, u.input - cached - written);
|
|
128
128
|
const M = 1e6;
|
|
129
|
-
return fresh * tier.input / M + cached * (tier.cachedInput ?? tier.input) / M + written * (tier.cacheWrite ?? tier.input) / M + u.output * tier.output / M;
|
|
129
|
+
return factor * (fresh * tier.input / M + cached * (tier.cachedInput ?? tier.input) / M + written * (tier.cacheWrite ?? tier.input) / M + u.output * tier.output / M);
|
|
130
130
|
}
|
|
131
131
|
|
|
132
132
|
// cli/src/tokens-lite.ts
|
|
@@ -203,7 +203,9 @@ function lex(src, path3) {
|
|
|
203
203
|
j++;
|
|
204
204
|
}
|
|
205
205
|
const end = Math.min(j, src.length);
|
|
206
|
-
|
|
206
|
+
const rep = (py ? /^\s*\*\s*(\d{1,6})\b/ : /^\.repeat\(\s*(\d{1,6})\s*\)/).exec(src.slice(end + 1, end + 40));
|
|
207
|
+
const body = src.slice(i + 1, end);
|
|
208
|
+
literals.push({ text: rep ? body.repeat(Math.max(1, Math.min(Number(rep[1]), Math.floor(2e5 / Math.max(1, body.length))))) : body, line });
|
|
207
209
|
blank(i + 1, end);
|
|
208
210
|
advance(i, end + 1);
|
|
209
211
|
i = end + 1;
|
|
@@ -222,6 +224,7 @@ function promptLike(t) {
|
|
|
222
224
|
const words = (t.match(/[A-Za-z]{2,}/g) ?? []).length;
|
|
223
225
|
return tags < 4 && cssRules < 3 && codeChars / t.length < 0.03 && words / (t.length / 6) > 0.5;
|
|
224
226
|
}
|
|
227
|
+
var REPEAT_TASK = /\b(classify|classification|categori[sz]e|sentiment|label (this|the|each)|is (this|it) spam|yes or no|true or false)\b/i;
|
|
225
228
|
var SIMPLE_TASK = /\b(classify|classification|categori[sz]e|sentiment|extract (the|all|every|any)|return (only |just )?(valid |a )?json|respond (only )?(with|in) (valid )?json|output (only )?(valid )?json|translate (this|the following|into)|summari[sz]e (this|the following|in \d+)|label (this|the|each)|is (this|it) spam|yes or no|true or false)\b/i;
|
|
226
229
|
var LLM_LIB_EXACT = /* @__PURE__ */ new Set(["openai", "anthropic", "@anthropic-ai/sdk", "ai", "litellm", "openai-agents", "@openai/agents", "instructor", "dspy", "dspy-ai", "guidance", "semantic-kernel", "haystack-ai", "autogen", "pyautogen", "autogen-agentchat", "crewai", "google-genai", "@google/genai", "mistralai", "@mistralai/mistralai", "cohere", "cohere-ai", "llama-index", "llama_index", "llamaindex", "langchain", "langgraph", "@langchain/core", "groq-sdk", "together-ai", "ollama"]);
|
|
227
230
|
var LLM_LIB_FAMILY = /^(@langchain\/[\w-]+|langchain[-_][\w-]+|langgraph[-_][\w-]+|llama[-_]index[-_][\w-]+|@ai-sdk\/[\w-]+|openai[-_]agents[-_][\w-]+)$/i;
|
|
@@ -254,6 +257,98 @@ function detectRepoKind(manifests2, files) {
|
|
|
254
257
|
return { kind: "unknown", reason: "no manifest signal" };
|
|
255
258
|
}
|
|
256
259
|
var DOWN = { high: "medium", medium: "low", low: "info", info: "info" };
|
|
260
|
+
var OPEN = "([{";
|
|
261
|
+
var CLOSE = ")]}";
|
|
262
|
+
function matchClose(m, from, limit = 4e4) {
|
|
263
|
+
let depth = 0;
|
|
264
|
+
for (let k = from; k < Math.min(m.length, from + limit); k++) {
|
|
265
|
+
const c2 = m[k];
|
|
266
|
+
if (OPEN.includes(c2)) depth++;
|
|
267
|
+
else if (CLOSE.includes(c2) && --depth === 0) return k;
|
|
268
|
+
}
|
|
269
|
+
return Math.min(m.length, from + limit) - 1;
|
|
270
|
+
}
|
|
271
|
+
function makeLocator(masked) {
|
|
272
|
+
const text = masked.join("\n");
|
|
273
|
+
const starts = [];
|
|
274
|
+
let o = 0;
|
|
275
|
+
for (const l of masked) {
|
|
276
|
+
starts.push(o);
|
|
277
|
+
o += l.length + 1;
|
|
278
|
+
}
|
|
279
|
+
const lineOf = (off) => {
|
|
280
|
+
let lo = 0, hi = starts.length - 1;
|
|
281
|
+
while (lo < hi) {
|
|
282
|
+
const mid = lo + hi + 1 >> 1;
|
|
283
|
+
if (starts[mid] <= off) lo = mid;
|
|
284
|
+
else hi = mid - 1;
|
|
285
|
+
}
|
|
286
|
+
return lo;
|
|
287
|
+
};
|
|
288
|
+
return { text, starts, lineOf };
|
|
289
|
+
}
|
|
290
|
+
var ASSIGN = (name) => new RegExp(`^\\s*(?:export\\s+)?(?:const|let|var|final|static|private|public|readonly|\\s)*\\s*(?:self\\.|this\\.)?${name}\\s*(?::[^=\\n]+)?=(?![=>])`);
|
|
291
|
+
var KEYWORDS = /* @__PURE__ */ new Set(["self", "this", "await", "async", "return", "model", "messages", "role", "content", "system", "user", "true", "false", "True", "False", "None", "null", "undefined", "new", "max_tokens", "temperature", "client", "resp", "response"]);
|
|
292
|
+
function assignmentScope(loc, masked, line) {
|
|
293
|
+
const eq = masked[line].search(/=(?![=>])/);
|
|
294
|
+
let end = line;
|
|
295
|
+
let k = loc.starts[line] + eq + 1;
|
|
296
|
+
for (; ; ) {
|
|
297
|
+
const lineEnd = loc.starts[end] + masked[end].length;
|
|
298
|
+
let opened = -1;
|
|
299
|
+
for (let q = k; q < lineEnd; q++) if (OPEN.includes(loc.text[q])) {
|
|
300
|
+
opened = q;
|
|
301
|
+
break;
|
|
302
|
+
}
|
|
303
|
+
if (opened >= 0) {
|
|
304
|
+
const close = matchClose(loc.text, opened, 2e4);
|
|
305
|
+
end = loc.lineOf(close);
|
|
306
|
+
k = close + 1;
|
|
307
|
+
if (k < loc.starts[end] + masked[end].length) continue;
|
|
308
|
+
}
|
|
309
|
+
const tail = masked[end].trimEnd();
|
|
310
|
+
if (end + 1 < masked.length && (/[+\\,(]$/.test(tail) || /^\s*[+.]/.test(masked[end + 1] ?? "")) && end - line < 60) {
|
|
311
|
+
end++;
|
|
312
|
+
k = loc.starts[end];
|
|
313
|
+
continue;
|
|
314
|
+
}
|
|
315
|
+
return { start: line, end };
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
function identifiersIn(masked, sc) {
|
|
319
|
+
const ids = /* @__PURE__ */ new Set();
|
|
320
|
+
for (let l = sc.start; l <= sc.end; l++) for (const m of (masked[l] ?? "").matchAll(/\b([A-Za-z_]\w{1,60})\b/g)) if (!KEYWORDS.has(m[1])) ids.add(m[1]);
|
|
321
|
+
return ids;
|
|
322
|
+
}
|
|
323
|
+
function resolveScopes(loc, masked, from, depth = 2) {
|
|
324
|
+
const out2 = [], seen = /* @__PURE__ */ new Set();
|
|
325
|
+
let frontier = [from];
|
|
326
|
+
for (let d = 0; d < depth && frontier.length; d++) {
|
|
327
|
+
const next = [];
|
|
328
|
+
for (const sc of frontier) for (const id of identifiersIn(masked, sc)) {
|
|
329
|
+
if (seen.has(id)) continue;
|
|
330
|
+
seen.add(id);
|
|
331
|
+
const re = ASSIGN(id);
|
|
332
|
+
let found = -1;
|
|
333
|
+
for (let l = sc.start; l >= 0; l--) if (re.test(masked[l])) {
|
|
334
|
+
found = l;
|
|
335
|
+
break;
|
|
336
|
+
}
|
|
337
|
+
if (found < 0) {
|
|
338
|
+
for (let l = sc.end + 1; l < masked.length; l++) if (re.test(masked[l])) {
|
|
339
|
+
found = l;
|
|
340
|
+
break;
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
if (found < 0 || found >= from.start && found <= from.end) continue;
|
|
344
|
+
const a = assignmentScope(loc, masked, found);
|
|
345
|
+
out2.push(a);
|
|
346
|
+
next.push(a);
|
|
347
|
+
}
|
|
348
|
+
frontier = next;
|
|
349
|
+
}
|
|
350
|
+
return out2;
|
|
351
|
+
}
|
|
257
352
|
function analyzeFiles(files, repo = null, manifests2 = []) {
|
|
258
353
|
const findings = [];
|
|
259
354
|
const callSites = [];
|
|
@@ -265,7 +360,7 @@ function analyzeFiles(files, repo = null, manifests2 = []) {
|
|
|
265
360
|
const { kind, reason } = detectRepoKind(manifests2, sources);
|
|
266
361
|
if (kind === "library") notes.push(`Library/SDK layout detected (${reason}): call sites are most likely pass-through wrappers where the caller chooses the model and prompt, so findings are downgraded one level and the estimate is discounted.`);
|
|
267
362
|
const promptLibrary = [];
|
|
268
|
-
const repoCaches = sources.some((f) => /cache_control|cacheControl|prompt_cache_key|promptCacheKey|cachedContent|cached_content/.test(f.content));
|
|
363
|
+
const repoCaches = sources.some((f) => /cache_control|cacheControl|prompt_cache_key|promptCacheKey|prompt_cache_options|promptCacheOptions|prompt_cache_breakpoint|promptCacheBreakpoint|prompt_cache_retention|promptCacheRetention|cachedContent|cached_content/.test(f.content));
|
|
269
364
|
const PROMPT_FILE = /(^|\/)[^/]*(prompt|instruction|template|persona)s?[^/]*\.[a-z]+$|(^|\/)prompts?\//i;
|
|
270
365
|
const PROMPT_NAME = /\b[\w.]*(prompt|system|instruction|template|persona)\w*\s*(:\s*[\w[\]|. ]+)?\s*[:=]\s*(f|r|rf|fr|b)?$/i;
|
|
271
366
|
for (const f of sources) {
|
|
@@ -278,12 +373,19 @@ function analyzeFiles(files, repo = null, manifests2 = []) {
|
|
|
278
373
|
const lead = (masked[l.line - 1] ?? "").slice(0, Math.max(0, (masked[l.line - 1] ?? "").search(/["'`]|$/))).trimEnd();
|
|
279
374
|
if (PROMPT_FILE.test(f.path) || PROMPT_NAME.test(lead) || PROMPT_NAME.test((masked[l.line - 2] ?? "").trimEnd())) promptLibrary.push({ file: f.path, line: l.line, tokens });
|
|
280
375
|
}
|
|
281
|
-
const hasCacheControl = /cache_control|cacheControl|prompt_cache_key|promptCacheKey|cachedContent|cached_content/.test(src);
|
|
376
|
+
const hasCacheControl = /cache_control|cacheControl|prompt_cache_key|promptCacheKey|prompt_cache_options|promptCacheOptions|prompt_cache_breakpoint|promptCacheBreakpoint|prompt_cache_retention|promptCacheRetention|cachedContent|cached_content/.test(src);
|
|
282
377
|
const illustrative = EXAMPLE_PATH.test(f.path);
|
|
283
378
|
let repeatFlagged = false;
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
379
|
+
const loc = makeLocator(masked);
|
|
380
|
+
const callLines = [];
|
|
381
|
+
masked.forEach((code2, i) => {
|
|
382
|
+
const hit = CALL_RE.find(([re]) => re.test(code2));
|
|
383
|
+
if (!hit) return;
|
|
384
|
+
const m = hit[0].exec(code2);
|
|
385
|
+
const close = matchClose(loc.text, loc.starts[i] + m.index + m[0].length - 1);
|
|
386
|
+
callLines.push({ line: i, sdk: hit[1], scope: { start: i, end: Math.max(i, loc.lineOf(close)) } });
|
|
387
|
+
});
|
|
388
|
+
callLines.forEach(({ line: i, sdk, scope }, n) => {
|
|
287
389
|
const local = [];
|
|
288
390
|
const push = (fd) => {
|
|
289
391
|
let sev = fd.severity;
|
|
@@ -291,16 +393,36 @@ function analyzeFiles(files, repo = null, manifests2 = []) {
|
|
|
291
393
|
if (illustrative) sev = DOWN[sev];
|
|
292
394
|
local.push({ ...fd, severity: sev });
|
|
293
395
|
};
|
|
294
|
-
const
|
|
295
|
-
|
|
396
|
+
const refs = resolveScopes(loc, masked, scope);
|
|
397
|
+
if (sdk.startsWith("langchain")) {
|
|
398
|
+
const assigned = /^\s*(?:const|let|var|self\.|this\.)?\s*([A-Za-z_]\w*)\s*(?::[^=\n]+)?=/.exec(masked[i])?.[1];
|
|
399
|
+
if (assigned) for (let l = 0; l < masked.length; l++) {
|
|
400
|
+
const inv = new RegExp(`\\b${assigned}\\.(invoke|ainvoke|stream|astream|batch|abatch|call|predict|generate)\\s*\\(`).exec(masked[l]);
|
|
401
|
+
if (!inv) continue;
|
|
402
|
+
const close = matchClose(loc.text, loc.starts[l] + inv.index + inv[0].length - 1);
|
|
403
|
+
const sc = { start: l, end: Math.max(l, loc.lineOf(close)) };
|
|
404
|
+
refs.push(sc, ...resolveScopes(loc, masked, sc));
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
const inScopes = (l, scs) => scs.some((sc) => l.line - 1 >= sc.start && l.line - 1 <= sc.end);
|
|
408
|
+
let near = literals.filter((l) => inScopes(l, [scope, ...refs]));
|
|
409
|
+
if (!near.some((l) => l.text.trim().length > 20)) {
|
|
410
|
+
const prevEnd = n > 0 ? callLines[n - 1].scope.end : -1;
|
|
411
|
+
near = near.concat(literals.filter((l) => l.line - 1 > prevEnd && l.line - 1 >= i - 40 && l.line - 1 < i));
|
|
412
|
+
}
|
|
413
|
+
const spanText = lines.slice(scope.start, scope.end + 1).join("\n");
|
|
414
|
+
const nextStart = n + 1 < callLines.length ? callLines[n + 1].line : lines.length;
|
|
415
|
+
const fallbackWindow = lines.slice(i, Math.min(nextStart, i + 30)).join("\n");
|
|
416
|
+
const modelFrom = (t) => [...t.matchAll(MODEL_RE)].map((m) => m[1])[0] ?? null;
|
|
417
|
+
const model = modelFrom(spanText) ?? modelFrom(refs.map((r) => lines.slice(r.start, r.end + 1).join("\n")).join("\n")) ?? modelFrom(fallbackWindow);
|
|
296
418
|
callSites.push({ file: f.path, line: i + 1, model, sdk });
|
|
297
419
|
if (model) models[model] = (models[model] ?? 0) + 1;
|
|
298
420
|
const key = normalizeModel(model);
|
|
299
421
|
const price = key ? priceFor(key) : null;
|
|
300
|
-
const near = literals.filter((l) => l.line >= i + 1 - 40 && l.line <= i + 1 + 40);
|
|
301
422
|
const prompts = near.filter((l) => promptLike(l.text));
|
|
302
423
|
const codeCtx = masked.slice(Math.max(0, i - 25), i + 25).join("\n");
|
|
303
424
|
const simpleTask = near.some((l) => SIMPLE_TASK.test(l.text));
|
|
425
|
+
const repeatTask = near.some((l) => REPEAT_TASK.test(l.text));
|
|
304
426
|
const isAnthropic = sdk.includes("anthropic") || (model?.startsWith("claude") ?? false);
|
|
305
427
|
const bigTok = prompts.reduce((a, l) => a + countTokens(l.text), 0);
|
|
306
428
|
if (bigTok >= 300 && !hasCacheControl) {
|
|
@@ -311,24 +433,26 @@ function analyzeFiles(files, repo = null, manifests2 = []) {
|
|
|
311
433
|
severity: isAnthropic ? bigTok >= 1500 ? "high" : cacheable ? "medium" : "low" : cacheable ? "medium" : "low",
|
|
312
434
|
file: f.path,
|
|
313
435
|
line: prompts[0].line,
|
|
314
|
-
title: `~${bigTok.toLocaleString()} static prompt tokens re-billed at full price on every call`,
|
|
315
|
-
detail: isAnthropic ? `This call sends a large fixed prompt but never marks a cache breakpoint, so Anthropic bills all ${bigTok} tokens at the full input rate each time instead of the cache-read rate (${readRate}% cheaper).` : `A fixed
|
|
436
|
+
title: isAnthropic ? `~${bigTok.toLocaleString()} static prompt tokens re-billed at full price on every call` : `~${bigTok.toLocaleString()} static prompt tokens sent on every call; only a stable prefix gets the cache discount`,
|
|
437
|
+
detail: (isAnthropic ? `This call sends a large fixed prompt but never marks a cache breakpoint, so Anthropic bills all ${bigTok} tokens at the full input rate each time instead of the cache-read rate (${readRate}% cheaper).` : `A fixed block of ~${bigTok} tokens goes out with every call. ${model ? model : "This provider"} discounts cached input ${readRate}%, but only for a byte-identical prefix of at least ${min} tokens that is routed to the same cache; anything volatile (dates, ids, user text) placed before or inside it, or traffic spread across cache shards, means paying full price.`) + (prompts[0].line !== i + 1 ? ` Sent by the call at ${f.path}:${i + 1}.` : ""),
|
|
316
438
|
fix: isAnthropic ? 'Route through TokenZero (breakpoints are injected automatically) or add `cache_control: {type: "ephemeral"}` to the last system block and the last tool.' : "Route through TokenZero to get a stable `prompt_cache_key`, or move dynamic content below the static prefix and set `prompt_cache_key` yourself.",
|
|
317
439
|
// OpenAI/Gemini already cache stable prefixes automatically; the recoverable part
|
|
318
440
|
// is prefix hygiene and cache routing, not the whole cache discount.
|
|
319
441
|
est_savings_pct: !cacheable ? 0 : isAnthropic ? Math.min(85, Math.round(bigTok / (bigTok + 400) * readRate)) : 15
|
|
320
442
|
});
|
|
321
443
|
}
|
|
322
|
-
|
|
323
|
-
|
|
444
|
+
const cascadeTo = key ? CASCADE_TARGET[key] : void 0;
|
|
445
|
+
const ratio = cascadeTo ? costOf(cascadeTo, { input: 1500, output: 200 }) / costOf(key, { input: 1500, output: 200 }) : 1;
|
|
446
|
+
if (key && price && cascadeTo && simpleTask && (price.tier === "frontier" || ratio <= 0.5)) {
|
|
447
|
+
const to = cascadeTo;
|
|
324
448
|
const a = costOf(key, { input: 1500, output: 200 }), b = costOf(to, { input: 1500, output: 200 });
|
|
325
449
|
push({
|
|
326
|
-
severity: "high",
|
|
450
|
+
severity: price.tier === "frontier" || ratio <= 0.2 ? "high" : "medium",
|
|
327
451
|
file: f.path,
|
|
328
452
|
line: i + 1,
|
|
329
453
|
title: `${key} used for a task whose prompt reads like extraction/classification`,
|
|
330
|
-
detail:
|
|
331
|
-
fix: `Opt into TokenZero cascade routing (\`X-TokenZero-Cascade: auto\`) so this call is routed to ${to} only when the classifier is confident
|
|
454
|
+
detail: `${price.tier === "frontier" ? "Frontier" : "Full-size"} pricing ($${price.input}/M in, $${price.output}/M out) on a constrained task. ${to} costs ~${Math.round((1 - b / a) * 100)}% less per call at this shape.`,
|
|
455
|
+
fix: `Opt into TokenZero cascade routing (\`X-TokenZero-Cascade: auto\`) so this call is routed to ${to} only when the classifier is confident (an opt-in shadow parity monitor is coming; nothing is promised), or switch the model here behind an eval.`,
|
|
332
456
|
est_savings_pct: Math.round((1 - b / a) * 100)
|
|
333
457
|
});
|
|
334
458
|
}
|
|
@@ -339,15 +463,15 @@ function analyzeFiles(files, repo = null, manifests2 = []) {
|
|
|
339
463
|
file: f.path,
|
|
340
464
|
line: i + 1,
|
|
341
465
|
title: "LLM call inside a loop",
|
|
342
|
-
detail:
|
|
343
|
-
fix:
|
|
466
|
+
detail: 'Per-item calls repeat the full prompt prefix for every element. If items are independent and latency-tolerant, the provider Batch API is 50% cheaper, and OpenAI Flex processing (`service_tier: "flex"`) gives the same 50% on ordinary calls without restructuring; if the prefix is shared, caching pays off immediately.',
|
|
467
|
+
fix: 'Batch where possible, or set `service_tier: "flex"` on OpenAI calls that can wait; otherwise keep the shared prefix cache-formatted (TokenZero does this automatically).',
|
|
344
468
|
est_savings_pct: 25
|
|
345
469
|
});
|
|
346
470
|
}
|
|
347
|
-
if (
|
|
471
|
+
if (repeatTask && !repeatFlagged && !/seed["']?\s*[:=]|temperature["']?\s*[:=]\s*0\b|X-TokenZero-Cache/i.test(spanText)) {
|
|
348
472
|
repeatFlagged = true;
|
|
349
473
|
push({
|
|
350
|
-
severity: "
|
|
474
|
+
severity: "low",
|
|
351
475
|
file: f.path,
|
|
352
476
|
line: i + 1,
|
|
353
477
|
title: "Repeat-prone task with no response-cache opt-in",
|
|
@@ -356,7 +480,7 @@ function analyzeFiles(files, repo = null, manifests2 = []) {
|
|
|
356
480
|
est_savings_pct: 10
|
|
357
481
|
});
|
|
358
482
|
}
|
|
359
|
-
if (isAnthropic && /\btools\s*[:=]/.test(masked.slice(
|
|
483
|
+
if (isAnthropic && /\btools\s*[:=]/.test(masked.slice(scope.start, scope.end + 1).join("\n")) && !hasCacheControl) {
|
|
360
484
|
push({
|
|
361
485
|
severity: "low",
|
|
362
486
|
file: f.path,
|
|
@@ -414,28 +538,52 @@ function analyzeFiles(files, repo = null, manifests2 = []) {
|
|
|
414
538
|
function filesFromDiff(diff) {
|
|
415
539
|
const files = [];
|
|
416
540
|
let cur = null;
|
|
541
|
+
let lineNo = 0;
|
|
417
542
|
for (const line of diff.split("\n")) {
|
|
418
543
|
const m = /^\+\+\+ b\/(.+)$/.exec(line);
|
|
419
544
|
if (m) {
|
|
420
545
|
cur = { path: m[1], content: "" };
|
|
421
546
|
files.push(cur);
|
|
547
|
+
lineNo = 0;
|
|
422
548
|
continue;
|
|
423
549
|
}
|
|
424
550
|
if (!cur) continue;
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
551
|
+
const h = /^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@/.exec(line);
|
|
552
|
+
if (h) {
|
|
553
|
+
const start = Number(h[1]);
|
|
554
|
+
while (lineNo < start - 1) {
|
|
555
|
+
cur.content += "\n";
|
|
556
|
+
lineNo++;
|
|
557
|
+
}
|
|
558
|
+
continue;
|
|
559
|
+
}
|
|
560
|
+
if (line.startsWith("+") && !line.startsWith("+++")) {
|
|
561
|
+
cur.content += line.slice(1) + "\n";
|
|
562
|
+
lineNo++;
|
|
563
|
+
} else if (line.startsWith(" ")) {
|
|
564
|
+
cur.content += line.slice(1) + "\n";
|
|
565
|
+
lineNo++;
|
|
566
|
+
} else if (line === "\") continue;
|
|
428
567
|
}
|
|
429
568
|
return files;
|
|
430
569
|
}
|
|
431
|
-
|
|
570
|
+
var PR_COMMENT_MARKER = "<!-- tokenzero-cost-check -->";
|
|
571
|
+
var PUBLIC_REPO = "https://github.com/tokenzero-ai/tokenzero";
|
|
572
|
+
var inert = (t) => t.replace(/[\\`*_[\]<>|]/g, (c2) => "\\" + c2).replace(/\r?\n/g, " ");
|
|
573
|
+
var pipeSafe = (t) => t.replace(/\|/g, "\\|");
|
|
574
|
+
var code = (t) => "`" + t.replace(/`/g, "'").replace(/\|/g, "\\|").replace(/\r?\n/g, " ").slice(0, 200) + "`";
|
|
575
|
+
var usd2 = (n) => "$" + n.toLocaleString("en-US", { minimumFractionDigits: 2, maximumFractionDigits: 2 });
|
|
576
|
+
function prComment(report, opts = {}) {
|
|
432
577
|
const c2 = report.cost_per_10k_runs;
|
|
433
|
-
const head = c2 ? `**TokenZero
|
|
434
|
-
const rows = report.findings.slice(0, 8).map((f) => `| ${f.severity} |
|
|
578
|
+
const head = c2 ? `**TokenZero cost check:** estimated cost per 10k runs ${code(usd2(c2.before_usd))} \u2192 ${code(usd2(c2.after_usd))} (${report.estimated_waste_pct}% recoverable) on ${code(c2.model)}` : `**TokenZero cost check:** ${report.findings.length} finding(s), ${report.call_sites.length} LLM call site(s)`;
|
|
579
|
+
const rows = report.findings.slice(0, 8).map((f) => `| ${f.severity} | ${code(`${f.file}:${f.line}`)} | ${pipeSafe(f.title)} | ${f.severity === "info" ? "not counted" : f.est_savings_pct + "%"} |`).join("\n");
|
|
580
|
+
const g = opts.gatewayUrl?.replace(/\/+$/, "");
|
|
581
|
+
const fix = g ? `One-line fix for everything above: point your SDK at ${code(g + "/v1")} (BYOK; your keys pass straight through).${opts.hostedReport ? ` [Full report](${g}/audit/${report.id})` : ""}` : `How to fix: route calls through the TokenZero gateway, or apply the fixes above by hand. [TokenZero tools](${PUBLIC_REPO})`;
|
|
435
582
|
return [
|
|
583
|
+
PR_COMMENT_MARKER,
|
|
436
584
|
head,
|
|
437
585
|
"",
|
|
438
|
-
report.findings.length ? "| severity | where | finding |
|
|
586
|
+
report.findings.length ? "| severity | where | finding | savings on that call |\n|---|---|---|---|\n" + rows : "_No token-waste patterns found in the changed lines._",
|
|
439
587
|
"",
|
|
440
588
|
report.findings.slice(0, 3).map((f) => `<details><summary>${f.title}</summary>
|
|
441
589
|
|
|
@@ -444,10 +592,10 @@ ${f.detail}
|
|
|
444
592
|
**Fix:** ${f.fix}
|
|
445
593
|
</details>`).join("\n"),
|
|
446
594
|
"",
|
|
447
|
-
|
|
448
|
-
...report.notes.map((n) => `> ${n}`),
|
|
595
|
+
fix,
|
|
596
|
+
...report.notes.map((n) => `> ${inert(n)}`),
|
|
449
597
|
"",
|
|
450
|
-
"<sub>
|
|
598
|
+
"<sub>Static heuristics at list prices, not measurements. Each row is the saving on the call it points at; the headline weights rows by confidence and averages over every call site in the change. Verified numbers come from the signed ledger once traffic flows.</sub>"
|
|
451
599
|
].join("\n");
|
|
452
600
|
}
|
|
453
601
|
|
|
@@ -501,8 +649,8 @@ function canonical(obj) {
|
|
|
501
649
|
}
|
|
502
650
|
|
|
503
651
|
// cli/src/main.ts
|
|
504
|
-
var VERSION = "0.1.
|
|
505
|
-
var BAKED_URL =
|
|
652
|
+
var VERSION = "0.1.4";
|
|
653
|
+
var BAKED_URL = true ? "https://tokenzero.sh" : "__TOKENZERO_DEFAULT_URL__";
|
|
506
654
|
function parseArgs(argv) {
|
|
507
655
|
const a = { _: [], flags: {} };
|
|
508
656
|
for (let i = 0; i < argv.length; i++) {
|
|
@@ -523,7 +671,7 @@ function parseArgs(argv) {
|
|
|
523
671
|
}
|
|
524
672
|
var VALUE_FLAGS = /* @__PURE__ */ new Set(["url", "name", "key", "format", "diff", "fail-over", "env-file"]);
|
|
525
673
|
var tty = process.stdout.isTTY && !process.env.NO_COLOR;
|
|
526
|
-
var c = (
|
|
674
|
+
var c = (code2) => (s) => tty ? `\x1B[${code2}m${s}\x1B[0m` : s;
|
|
527
675
|
var bold = c("1");
|
|
528
676
|
var dim = c("2");
|
|
529
677
|
var green = c("32");
|
|
@@ -753,7 +901,7 @@ async function cmdAudit(a) {
|
|
|
753
901
|
} else report = analyzeFiles(walk(root), null, manifests(root));
|
|
754
902
|
const fmt = a.flags.format ?? (a.flags.json ? "json" : "text");
|
|
755
903
|
if (fmt === "json") out(JSON.stringify(report, null, 2));
|
|
756
|
-
else if (fmt === "markdown") out(prComment(report, baseUrl(a)
|
|
904
|
+
else if (fmt === "markdown") out(prComment(report, { gatewayUrl: baseUrl(a) }));
|
|
757
905
|
else {
|
|
758
906
|
summarize(report, 100);
|
|
759
907
|
for (const f of report.findings.slice(0, 3)) out(`
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tokenzero/cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.4",
|
|
4
4
|
"description": "TokenZero CLI: connect a project to the TokenZero BYOK gateway, audit LLM token waste locally, check usage and verify signed savings receipts.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|