@trazum/cli 1.81.0 → 1.82.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/gateway-server.d.ts.map +1 -1
- package/dist/gateway-server.js +14 -0
- package/dist/gateway-server.js.map +1 -1
- package/dist/i18n/en.d.ts.map +1 -1
- package/dist/i18n/en.js +61 -5
- package/dist/i18n/en.js.map +1 -1
- package/dist/i18n/es.d.ts.map +1 -1
- package/dist/i18n/es.js +63 -5
- package/dist/i18n/es.js.map +1 -1
- package/dist/i18n/types.d.ts +68 -6
- package/dist/i18n/types.d.ts.map +1 -1
- package/dist/index.d.ts +54 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +389 -52
- package/dist/index.js.map +1 -1
- package/dist/markdown.d.ts +10 -0
- package/dist/markdown.d.ts.map +1 -1
- package/dist/markdown.js +4 -4
- package/dist/markdown.js.map +1 -1
- package/package.json +2 -2
- package/src/gateway-server.ts +14 -0
- package/src/i18n/en.ts +76 -7
- package/src/i18n/es.ts +78 -7
- package/src/i18n/types.ts +70 -6
- package/src/index.ts +404 -51
- package/src/markdown.ts +14 -4
package/dist/index.js
CHANGED
|
@@ -7,7 +7,7 @@ import { createInterface } from 'node:readline/promises';
|
|
|
7
7
|
import { dirname, join, resolve as resolvePath } from 'node:path';
|
|
8
8
|
import { fileURLToPath } from 'node:url';
|
|
9
9
|
import { gunzipSync } from 'node:zlib';
|
|
10
|
-
import { CONTRACT_NAMES, contractSchema, applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, bucketedCacheEconomics, bucketedProfile, buildHistory, buildPlan, connectorFor, CONNECTORS, normalizeAnthropicUsage, normalizeOpenAIUsage, bucketsFromRecords, evaluateWatch, firedKey, pruneRecords, recordsFromBuckets, storeInventory, storedReportFrom, verifyPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, bandGoverns, countTokensAnthropic, DEFAULT_USAGE, budgetPositions, conform, BREAK_EVEN_BAND, allocate, annualRecord, replayCommitment, coversTheTerm, runExperiment, qualityGate, semanticPassCost, verifySemanticProposals, SEMANTIC_SYSTEM_PROMPT, ladderPosition, validateLadder, outcomeReport, rankPerOutcome, FAILURE_POLICIES, detectFromSource, matchLocale, parsePlanDocument, waiverDay, waiverHistory, proposeInit, MIN_RATE_DAYS, parseConfig, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, rollUp, heartbeats, ruleYield, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, matchGlob, MAX_INPUT_CHARS, moneyIsComparable, mostSpecificMatch, assemble, interview, nearestName, slot, SLOT_IDS, optimize, parseBaseline, PHRASE_LANGUAGES, detectTextLanguage, dictionaryStanding, languagesWithStanding, indexUsage, parseUsageLine, plannedCalls, claudeCodeRecords, otelRecords, ownRate, positionAt, positionReport, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, switchAnalysis, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
|
|
10
|
+
import { CONTRACT_NAMES, contractSchema, applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, bucketedCacheEconomics, bucketedProfile, buildHistory, buildPlan, connectorFor, CONNECTORS, normalizeAnthropicUsage, normalizeOpenAIUsage, bucketsFromRecords, evaluateWatch, firedKey, pruneRecords, recordsFromBuckets, storeInventory, storedReportFrom, verifyPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, bandGoverns, countTokensAnthropic, DEFAULT_USAGE, budgetPositions, conform, BREAK_EVEN_BAND, allocate, annualRecord, replayCommitment, coversTheTerm, runExperiment, qualityGate, semanticPassCost, verifySemanticProposals, SEMANTIC_SYSTEM_PROMPT, ladderPosition, validateLadder, outcomeReport, rankPerOutcome, FAILURE_POLICIES, detectFromSource, matchLocale, parsePlanDocument, waiverDay, waiverHistory, proposeInit, MIN_RATE_DAYS, parseConfig, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, rollUp, heartbeats, ruleYield, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, matchGlob, MAX_INPUT_CHARS, moneyIsComparable, mostSpecificMatch, assemble, interview, nearestName, slot, SLOT_IDS, optimize, parseBaseline, PHRASE_LANGUAGES, detectTextLanguage, dictionaryStanding, languagesWithStanding, indexUsage, parseUsageLine, plannedCalls, claudeCodeRecords, ESTIMATE_ERROR_BAND_PCT, bandFor, foreignTokenizer, measuredForeignError, heliconeRecords, litellmRecords, otelRecords, ownRate, positionAt, positionReport, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, switchAnalysis, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
|
|
11
11
|
import { cacheDir, cacheStats, cachingProvider, clearCache } from './suggest-cache.js';
|
|
12
12
|
import { dayOf, formatGap, median, spanDays } from './time.js';
|
|
13
13
|
// Everything that reads the filesystem, on its own entry point so the web
|
|
@@ -383,6 +383,8 @@ const COMMAND_FLAGS = {
|
|
|
383
383
|
position: ['json', 'html-out', 'pricing', 'pricing-live'],
|
|
384
384
|
'from-claude-code': ['label', 'label-from-project', 'out', 'o', 'state'],
|
|
385
385
|
'from-otel': ['label-from-service', 'out', 'o'],
|
|
386
|
+
'from-litellm': ['out', 'o'],
|
|
387
|
+
'from-helicone': ['out', 'o'],
|
|
386
388
|
switch: ['to', 'migration-usd', 'cases'],
|
|
387
389
|
ownrate: ['gpu-usd-hour', 'tokens-per-second', 'utilization'],
|
|
388
390
|
pulse: ['json', 'max-stale-hours'],
|
|
@@ -483,11 +485,11 @@ function renderDiff(before, after, t) {
|
|
|
483
485
|
/**
|
|
484
486
|
* The provider's name when the estimator was not calibrated for it.
|
|
485
487
|
*
|
|
486
|
-
* `estimateTokens` is a heuristic tuned against Claude's tokenizer, and
|
|
487
|
-
*
|
|
488
|
-
*
|
|
489
|
-
*
|
|
490
|
-
*
|
|
488
|
+
* `estimateTokens` is a heuristic tuned against Claude's tokenizer, and every
|
|
489
|
+
* band descends from that. Printing one beside a GPT or Kimi figure states a
|
|
490
|
+
* precision nobody has measured for that family — and since the catalogue grew
|
|
491
|
+
* past Anthropic, that is most of it. Returns null when the model is
|
|
492
|
+
* Anthropic's, where the band is at least the claim it was written for.
|
|
491
493
|
*/
|
|
492
494
|
function offFamilyName(modelId) {
|
|
493
495
|
const provider = getModel(modelId).provider;
|
|
@@ -495,6 +497,17 @@ function offFamilyName(modelId) {
|
|
|
495
497
|
return null;
|
|
496
498
|
return getModel(modelId).displayName;
|
|
497
499
|
}
|
|
500
|
+
/**
|
|
501
|
+
* How far off the estimator has been measured on this model's family.
|
|
502
|
+
*
|
|
503
|
+
* Read from the catalogue's provider rather than from the display name, because
|
|
504
|
+
* the name is what a reader sees and the id is what was measured. Null on the
|
|
505
|
+
* calibrated family and on every family nobody has run — an unmeasured one gets
|
|
506
|
+
* a sentence saying so, never a borrowed figure.
|
|
507
|
+
*/
|
|
508
|
+
function offFamilyError(modelId) {
|
|
509
|
+
return measuredForeignError(foreignTokenizer(getModel(modelId).provider ?? null));
|
|
510
|
+
}
|
|
498
511
|
/**
|
|
499
512
|
* Language codes as names, in the reader's language.
|
|
500
513
|
*
|
|
@@ -559,7 +572,7 @@ namedScenario = false,
|
|
|
559
572
|
measured = null) {
|
|
560
573
|
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
561
574
|
const sourceNote = result.tokenSource === 'heuristic'
|
|
562
|
-
? c.dim(t.report.estimated(offFamilyName(result.usage.model)))
|
|
575
|
+
? c.dim(t.report.estimated(offFamilyName(result.usage.model), bandFor(result.optimized), offFamilyError(result.usage.model)))
|
|
563
576
|
: c.dim(t.report.exactCount());
|
|
564
577
|
/**
|
|
565
578
|
* The largest lever, first.
|
|
@@ -2477,6 +2490,184 @@ async function commandFromOtel(args, t) {
|
|
|
2477
2490
|
if (unparseable > 0)
|
|
2478
2491
|
console.error(t.fromOtel.unparseable(unparseable));
|
|
2479
2492
|
}
|
|
2493
|
+
/**
|
|
2494
|
+
* `trazum from-litellm <file|dir>` — a LiteLLM spend log as a usage log.
|
|
2495
|
+
*
|
|
2496
|
+
* The same pure-converter pattern again, pointed at the gateway a great many
|
|
2497
|
+
* teams already put in front of every provider: `LiteLLM_SpendLogs` is the
|
|
2498
|
+
* export most likely to already exist on somebody's disk. `litellmRecords` in
|
|
2499
|
+
* core does the reading; this is the walk and the honesty.
|
|
2500
|
+
*
|
|
2501
|
+
* Three of the four counts it prints exist because the alternative is a
|
|
2502
|
+
* flattering silence. Rows naming no model are not priced by the route they
|
|
2503
|
+
* took, rows with no tokens are logged calls nobody can price, and a
|
|
2504
|
+
* `cache_hit` flag is not a token split — so the cache verdicts read "cannot
|
|
2505
|
+
* tell" rather than a fabricated one, exactly as `from-otel` does.
|
|
2506
|
+
*
|
|
2507
|
+
* The fourth is the one that matters most: LiteLLM prices the same calls with
|
|
2508
|
+
* its own table, and that figure is printed beside Trazum's and never merged
|
|
2509
|
+
* into it. Two price tables summed into one total is how a report becomes
|
|
2510
|
+
* quietly wrong.
|
|
2511
|
+
*
|
|
2512
|
+
* The row carries `messages`, `response`, `api_key`, `requester_ip_address`
|
|
2513
|
+
* and `end_user`. None of it is read; a fixture plants a marker in each and
|
|
2514
|
+
* greps the whole output.
|
|
2515
|
+
*/
|
|
2516
|
+
async function commandFromLiteLlm(args, t) {
|
|
2517
|
+
const target = args.positional[0];
|
|
2518
|
+
if (target === undefined)
|
|
2519
|
+
throw new Error(t.fromLiteLlm.noPath());
|
|
2520
|
+
let info;
|
|
2521
|
+
try {
|
|
2522
|
+
info = await stat(target);
|
|
2523
|
+
}
|
|
2524
|
+
catch {
|
|
2525
|
+
throw new Error(t.fromLiteLlm.notFound(target));
|
|
2526
|
+
}
|
|
2527
|
+
const files = [];
|
|
2528
|
+
if (info.isDirectory()) {
|
|
2529
|
+
const entries = await readdir(target, { recursive: true, withFileTypes: true });
|
|
2530
|
+
for (const entry of entries) {
|
|
2531
|
+
if (entry.isFile() && /\.(json|jsonl|ndjson)$/i.test(entry.name)) {
|
|
2532
|
+
files.push(join(entry.parentPath, entry.name));
|
|
2533
|
+
}
|
|
2534
|
+
}
|
|
2535
|
+
files.sort();
|
|
2536
|
+
if (files.length === 0)
|
|
2537
|
+
throw new Error(t.fromLiteLlm.noExports(target));
|
|
2538
|
+
}
|
|
2539
|
+
else {
|
|
2540
|
+
files.push(target);
|
|
2541
|
+
}
|
|
2542
|
+
const lines = [];
|
|
2543
|
+
let rows = 0;
|
|
2544
|
+
let unnamedModel = 0;
|
|
2545
|
+
let noTokens = 0;
|
|
2546
|
+
let cacheFlagged = 0;
|
|
2547
|
+
let unparseable = 0;
|
|
2548
|
+
let reportedSpend = 0;
|
|
2549
|
+
let sawSpend = false;
|
|
2550
|
+
for (const file of files) {
|
|
2551
|
+
const conversion = litellmRecords(await readFile(file, 'utf8'));
|
|
2552
|
+
for (const record of conversion.records)
|
|
2553
|
+
lines.push(JSON.stringify(record));
|
|
2554
|
+
rows += conversion.rows;
|
|
2555
|
+
unnamedModel += conversion.unnamedModel;
|
|
2556
|
+
noTokens += conversion.noTokens;
|
|
2557
|
+
cacheFlagged += conversion.cacheFlagged;
|
|
2558
|
+
unparseable += conversion.unparseable;
|
|
2559
|
+
if (conversion.reportedSpendUsd !== null) {
|
|
2560
|
+
reportedSpend += conversion.reportedSpendUsd;
|
|
2561
|
+
sawSpend = true;
|
|
2562
|
+
}
|
|
2563
|
+
}
|
|
2564
|
+
const out = stringFlag(args, 'out') ?? stringFlag(args, 'o');
|
|
2565
|
+
if (out !== undefined) {
|
|
2566
|
+
await writeFile(out, lines.join('\n') + (lines.length > 0 ? '\n' : ''), 'utf8');
|
|
2567
|
+
console.error(t.fromLiteLlm.written(out));
|
|
2568
|
+
}
|
|
2569
|
+
else {
|
|
2570
|
+
for (const line of lines)
|
|
2571
|
+
console.log(line);
|
|
2572
|
+
}
|
|
2573
|
+
console.error(t.fromLiteLlm.summary(files.length, rows));
|
|
2574
|
+
if (unnamedModel > 0)
|
|
2575
|
+
console.error(t.fromLiteLlm.unnamedModel(unnamedModel));
|
|
2576
|
+
if (noTokens > 0)
|
|
2577
|
+
console.error(t.fromLiteLlm.noTokens(noTokens));
|
|
2578
|
+
if (cacheFlagged > 0)
|
|
2579
|
+
console.error(t.fromLiteLlm.cacheFlagged(cacheFlagged));
|
|
2580
|
+
if (sawSpend)
|
|
2581
|
+
console.error(t.fromLiteLlm.reportedSpend(formatUsd(reportedSpend)));
|
|
2582
|
+
if (unparseable > 0)
|
|
2583
|
+
console.error(t.fromLiteLlm.unparseable(unparseable));
|
|
2584
|
+
}
|
|
2585
|
+
/**
|
|
2586
|
+
* `trazum from-helicone <file|dir>` — a Helicone request export as a usage log.
|
|
2587
|
+
*
|
|
2588
|
+
* The third converter, and the one that needs three columns where the others
|
|
2589
|
+
* need one: Helicone carries `request_model`, `model_override` and
|
|
2590
|
+
* `response_model`, and they can disagree. The response wins, because a bill
|
|
2591
|
+
* is about what was billed rather than what was intended, and the
|
|
2592
|
+
* disagreements are counted so a substitution is something the reader sees.
|
|
2593
|
+
*
|
|
2594
|
+
* Two absences are stated rather than filled in. There is no cache token
|
|
2595
|
+
* split on the row, only a flag; and a Helicone request id names one call and
|
|
2596
|
+
* never a conversation, so the records carry no session and the
|
|
2597
|
+
* conversation-shaped findings stay unavailable. Both are printed, because a
|
|
2598
|
+
* gap that is not said reads as a gap that is not there.
|
|
2599
|
+
*/
|
|
2600
|
+
async function commandFromHelicone(args, t) {
|
|
2601
|
+
const target = args.positional[0];
|
|
2602
|
+
if (target === undefined)
|
|
2603
|
+
throw new Error(t.fromHelicone.noPath());
|
|
2604
|
+
let info;
|
|
2605
|
+
try {
|
|
2606
|
+
info = await stat(target);
|
|
2607
|
+
}
|
|
2608
|
+
catch {
|
|
2609
|
+
throw new Error(t.fromHelicone.notFound(target));
|
|
2610
|
+
}
|
|
2611
|
+
const files = [];
|
|
2612
|
+
if (info.isDirectory()) {
|
|
2613
|
+
const entries = await readdir(target, { recursive: true, withFileTypes: true });
|
|
2614
|
+
for (const entry of entries) {
|
|
2615
|
+
if (entry.isFile() && /\.(json|jsonl|ndjson)$/i.test(entry.name)) {
|
|
2616
|
+
files.push(join(entry.parentPath, entry.name));
|
|
2617
|
+
}
|
|
2618
|
+
}
|
|
2619
|
+
files.sort();
|
|
2620
|
+
if (files.length === 0)
|
|
2621
|
+
throw new Error(t.fromHelicone.noExports(target));
|
|
2622
|
+
}
|
|
2623
|
+
else {
|
|
2624
|
+
files.push(target);
|
|
2625
|
+
}
|
|
2626
|
+
const lines = [];
|
|
2627
|
+
let rows = 0;
|
|
2628
|
+
let unnamedModel = 0;
|
|
2629
|
+
let disagreements = 0;
|
|
2630
|
+
let noTokens = 0;
|
|
2631
|
+
let cacheFlagged = 0;
|
|
2632
|
+
let unparseable = 0;
|
|
2633
|
+
for (const file of files) {
|
|
2634
|
+
const conversion = heliconeRecords(await readFile(file, 'utf8'));
|
|
2635
|
+
for (const record of conversion.records)
|
|
2636
|
+
lines.push(JSON.stringify(record));
|
|
2637
|
+
rows += conversion.rows;
|
|
2638
|
+
unnamedModel += conversion.unnamedModel;
|
|
2639
|
+
disagreements += conversion.modelDisagreements;
|
|
2640
|
+
noTokens += conversion.noTokens;
|
|
2641
|
+
cacheFlagged += conversion.cacheFlagged;
|
|
2642
|
+
unparseable += conversion.unparseable;
|
|
2643
|
+
}
|
|
2644
|
+
const out = stringFlag(args, 'out') ?? stringFlag(args, 'o');
|
|
2645
|
+
if (out !== undefined) {
|
|
2646
|
+
await writeFile(out, lines.join('\n') + (lines.length > 0 ? '\n' : ''), 'utf8');
|
|
2647
|
+
console.error(t.fromHelicone.written(out));
|
|
2648
|
+
}
|
|
2649
|
+
else {
|
|
2650
|
+
for (const line of lines)
|
|
2651
|
+
console.log(line);
|
|
2652
|
+
}
|
|
2653
|
+
console.error(t.fromHelicone.summary(files.length, rows));
|
|
2654
|
+
if (unnamedModel > 0)
|
|
2655
|
+
console.error(t.fromHelicone.unnamedModel(unnamedModel));
|
|
2656
|
+
if (disagreements > 0)
|
|
2657
|
+
console.error(t.fromHelicone.disagreements(disagreements));
|
|
2658
|
+
if (noTokens > 0)
|
|
2659
|
+
console.error(t.fromHelicone.noTokens(noTokens));
|
|
2660
|
+
if (cacheFlagged > 0)
|
|
2661
|
+
console.error(t.fromHelicone.cacheFlagged(cacheFlagged));
|
|
2662
|
+
// Said on every run that produced records, not only when something is
|
|
2663
|
+
// missing: the absence is a property of the format, and a reader who has
|
|
2664
|
+
// just converted a month needs to know which questions this log cannot
|
|
2665
|
+
// answer before they go looking for the answers.
|
|
2666
|
+
if (rows > 0)
|
|
2667
|
+
console.error(t.fromHelicone.noSessions());
|
|
2668
|
+
if (unparseable > 0)
|
|
2669
|
+
console.error(t.fromHelicone.unparseable(unparseable));
|
|
2670
|
+
}
|
|
2480
2671
|
/**
|
|
2481
2672
|
* `trazum position <usage.jsonl>` — where the month stands, measured.
|
|
2482
2673
|
*
|
|
@@ -4547,6 +4738,7 @@ async function commandCheck(args, config, pricing, t, locale) {
|
|
|
4547
4738
|
verdicts: [verdict],
|
|
4548
4739
|
level,
|
|
4549
4740
|
tokenSource: counter.source,
|
|
4741
|
+
band: bandFor(prompt),
|
|
4550
4742
|
truncated: false,
|
|
4551
4743
|
t,
|
|
4552
4744
|
}));
|
|
@@ -4654,11 +4846,16 @@ async function checkEmbedded(path, extraction, budget, args, counter, level, t,
|
|
|
4654
4846
|
}
|
|
4655
4847
|
const failures = verdicts.filter(isOverBudget);
|
|
4656
4848
|
const ok = failures.length === 0 && extraction.declined.length === 0;
|
|
4849
|
+
// Widest across the prompts this report covers, never an average: a run over
|
|
4850
|
+
// prose and a CSV covers text the estimator is 6% and 33% out on, and a
|
|
4851
|
+
// figure between them would describe neither.
|
|
4852
|
+
const band = widestBand(extraction.prompts.map((prompt) => prompt.text));
|
|
4657
4853
|
await writeMarkdown(args, () => renderCheckMarkdown({
|
|
4658
4854
|
target: path,
|
|
4659
4855
|
verdicts,
|
|
4660
4856
|
level,
|
|
4661
4857
|
tokenSource: counter.source,
|
|
4858
|
+
band,
|
|
4662
4859
|
truncated: false,
|
|
4663
4860
|
t,
|
|
4664
4861
|
}));
|
|
@@ -7896,12 +8093,102 @@ function reportProfileGaps(report, t, n, stalePricing = null) {
|
|
|
7896
8093
|
* exactly as `prune` does. A command that can spend somebody's money without
|
|
7897
8094
|
* telling them first is a command they stop trusting.
|
|
7898
8095
|
*/
|
|
8096
|
+
/**
|
|
8097
|
+
* The routing measurement, without the text that produced it.
|
|
8098
|
+
*
|
|
8099
|
+
* `route --json` printed `{ slice, evaluation }` — the whole `EvalReport`,
|
|
8100
|
+
* whose `cases[]` carries `input` (the caller's own case text), `baseline[0]`,
|
|
8101
|
+
* `baseline[1]` and `optimized` (three model answers). It carried no
|
|
8102
|
+
* `schemaVersion` either.
|
|
8103
|
+
*
|
|
8104
|
+
* `docs/json-output.md` opens by promising that **nothing here carries a
|
|
8105
|
+
* session key or prompt text**, and this is the document most likely to be
|
|
8106
|
+
* piped into a dashboard, pasted into a ticket or dropped into a browser. A
|
|
8107
|
+
* tool whose headline is that prompts never leave the machine cannot have its
|
|
8108
|
+
* most portable artefact be the one that carries them.
|
|
8109
|
+
*
|
|
8110
|
+
* The per-case *shape* survives: the two similarity scores are the evidence
|
|
8111
|
+
* behind the verdict, and a reader checking a borderline call needs them. Only
|
|
8112
|
+
* the strings go. Anyone who wants the answers has the human-readable output,
|
|
8113
|
+
* which never left the terminal.
|
|
8114
|
+
*
|
|
8115
|
+
* A named function rather than an inline object so the stripping is a thing a
|
|
8116
|
+
* test can call without spending a provider call to reach it.
|
|
8117
|
+
*/
|
|
8118
|
+
export function routeDocument(slice, result) {
|
|
8119
|
+
return {
|
|
8120
|
+
schemaVersion: 1,
|
|
8121
|
+
slice,
|
|
8122
|
+
evaluation: {
|
|
8123
|
+
provider: result.provider,
|
|
8124
|
+
model: result.model,
|
|
8125
|
+
candidateModel: result.candidateModel,
|
|
8126
|
+
verdict: result.verdict,
|
|
8127
|
+
selfAgreement: result.selfAgreement,
|
|
8128
|
+
crossAgreement: result.crossAgreement,
|
|
8129
|
+
callsMade: result.callsMade,
|
|
8130
|
+
cases: result.cases.map((one) => ({
|
|
8131
|
+
selfSimilarity: one.selfSimilarity,
|
|
8132
|
+
crossSimilarity: one.crossSimilarity,
|
|
8133
|
+
})),
|
|
8134
|
+
},
|
|
8135
|
+
};
|
|
8136
|
+
}
|
|
8137
|
+
/**
|
|
8138
|
+
* The example-pruning measurement, likewise.
|
|
8139
|
+
*
|
|
8140
|
+
* `ExampleContribution.text` is *"the example itself, so a report can quote its
|
|
8141
|
+
* first line"* — true of the terminal rendering and wrong of a document that
|
|
8142
|
+
* promises to carry no prompt text. A few-shot example **is** prompt text, and
|
|
8143
|
+
* this was the second command emitting it.
|
|
8144
|
+
*
|
|
8145
|
+
* `index` and `tokens` say which example without quoting it, which is what a
|
|
8146
|
+
* machine reading this needs: it already has the prompt.
|
|
8147
|
+
*/
|
|
8148
|
+
export function pruneDocument(report) {
|
|
8149
|
+
return {
|
|
8150
|
+
schemaVersion: 1,
|
|
8151
|
+
provider: report.provider,
|
|
8152
|
+
model: report.model,
|
|
8153
|
+
selfAgreement: report.selfAgreement,
|
|
8154
|
+
recoverableTokens: report.recoverableTokens,
|
|
8155
|
+
callsMade: report.callsMade,
|
|
8156
|
+
contributions: report.contributions.map((one) => ({
|
|
8157
|
+
index: one.index,
|
|
8158
|
+
tokens: one.tokens,
|
|
8159
|
+
agreementWithout: one.agreementWithout,
|
|
8160
|
+
verdict: one.verdict,
|
|
8161
|
+
})),
|
|
8162
|
+
};
|
|
8163
|
+
}
|
|
7899
8164
|
async function commandRoute(args, pricing, t) {
|
|
8165
|
+
/**
|
|
8166
|
+
* Under `--json`, stdout carries the document and nothing else.
|
|
8167
|
+
*
|
|
8168
|
+
* It carried the human preamble too — the slice it picked, the call count,
|
|
8169
|
+
* the "nothing has been spent yet" line — straight into the same stream as
|
|
8170
|
+
* the JSON, so `route --json > verdict.json` wrote a file no parser would
|
|
8171
|
+
* read. Found by running the command and feeding the file to the bridge
|
|
8172
|
+
* that reads it, which is the one thing the pure-function guards could not
|
|
8173
|
+
* do: `interchange.test.js` names `route` among the commands it cannot
|
|
8174
|
+
* drive, so nothing checked that its `--json` output was a document.
|
|
8175
|
+
*
|
|
8176
|
+
* The lines go to stderr rather than away. A reader running this in a
|
|
8177
|
+
* terminal still needs to see which slice was picked and what it will cost
|
|
8178
|
+
* before it costs it; a pipe needs the document. stderr gives both.
|
|
8179
|
+
*/
|
|
8180
|
+
const asJson = boolFlag(args, 'json');
|
|
8181
|
+
const say = (line = '') => {
|
|
8182
|
+
if (asJson)
|
|
8183
|
+
console.error(line);
|
|
8184
|
+
else
|
|
8185
|
+
console.log(line);
|
|
8186
|
+
};
|
|
7900
8187
|
const path = args.positional[0];
|
|
7901
8188
|
if (path === undefined) {
|
|
7902
|
-
|
|
7903
|
-
|
|
7904
|
-
|
|
8189
|
+
say();
|
|
8190
|
+
say(c.dim(wrap(t.route.noTarget(), 74, ' ')));
|
|
8191
|
+
say();
|
|
7905
8192
|
return;
|
|
7906
8193
|
}
|
|
7907
8194
|
const promptPath = stringFlag(args, 'prompt-file');
|
|
@@ -7923,16 +8210,16 @@ async function commandRoute(args, pricing, t) {
|
|
|
7923
8210
|
const available = report.byLabel
|
|
7924
8211
|
.map((r) => (r.label === UNLABELLED ? t.profile.unlabelled() : r.label))
|
|
7925
8212
|
.join(', ');
|
|
7926
|
-
|
|
7927
|
-
|
|
7928
|
-
|
|
8213
|
+
say();
|
|
8214
|
+
say(c.dim(wrap(t.route.labelNotFound(wanted, available), 74, ' ')));
|
|
8215
|
+
say();
|
|
7929
8216
|
return;
|
|
7930
8217
|
}
|
|
7931
8218
|
const slice = levers.slices.find((s) => s.route !== null && (wanted === undefined || s.label === wanted));
|
|
7932
8219
|
if (!slice?.route) {
|
|
7933
|
-
|
|
7934
|
-
|
|
7935
|
-
|
|
8220
|
+
say();
|
|
8221
|
+
say(c.dim(wrap(t.route.noRoute(), 74, ' ')));
|
|
8222
|
+
say();
|
|
7936
8223
|
return;
|
|
7937
8224
|
}
|
|
7938
8225
|
const prompt = capInput(await readFile(promptPath, 'utf8'), promptPath, maxInputFlag(args, t), t);
|
|
@@ -7956,8 +8243,8 @@ async function commandRoute(args, pricing, t) {
|
|
|
7956
8243
|
throw new Error(t.errors.llmNotConfigured());
|
|
7957
8244
|
const label = slice.label === UNLABELLED ? t.profile.unlabelled() : slice.label;
|
|
7958
8245
|
const worth = formatUsd(slice.route.savingUsd);
|
|
7959
|
-
|
|
7960
|
-
|
|
8246
|
+
say();
|
|
8247
|
+
say(` ${c.bold(t.route.picked(label, slice.modelName, slice.route.candidate.displayName, worth, `${(slice.shareOfBill * 100).toFixed(1)}%`))}`);
|
|
7961
8248
|
/**
|
|
7962
8249
|
* The money and the measurement have to describe the same calls.
|
|
7963
8250
|
*
|
|
@@ -7968,16 +8255,16 @@ async function commandRoute(args, pricing, t) {
|
|
|
7968
8255
|
* counts, so it is stated rather than guessed at.
|
|
7969
8256
|
*/
|
|
7970
8257
|
if (slice.label === UNLABELLED) {
|
|
7971
|
-
|
|
8258
|
+
say(` ${c.yellow('!')} ${c.dim(wrap(t.route.unlabelledSlice(), 74, ' '))}`);
|
|
7972
8259
|
}
|
|
7973
|
-
|
|
7974
|
-
|
|
8260
|
+
say();
|
|
8261
|
+
say(` ${c.dim(wrap(t.route.willSpend(inputs.length * 3, provider.model, candidate.model), 74, ' '))}`);
|
|
7975
8262
|
if (!boolFlag(args, 'yes')) {
|
|
7976
|
-
|
|
7977
|
-
|
|
8263
|
+
say(` ${c.dim(t.route.dryRun())}`);
|
|
8264
|
+
say();
|
|
7978
8265
|
return;
|
|
7979
8266
|
}
|
|
7980
|
-
|
|
8267
|
+
say(` ${c.dim(t.route.running(inputs.length))}`);
|
|
7981
8268
|
// Same prompt on both sides. The axis under test is the model, and passing the
|
|
7982
8269
|
// prompt twice is what says so at the call site.
|
|
7983
8270
|
const result = await evaluate(prompt, prompt, inputs, provider, {
|
|
@@ -7985,21 +8272,21 @@ async function commandRoute(args, pricing, t) {
|
|
|
7985
8272
|
concurrency: numberFlag(args, 'concurrency', 3, t),
|
|
7986
8273
|
});
|
|
7987
8274
|
if (boolFlag(args, 'json')) {
|
|
7988
|
-
console.log(JSON.stringify(
|
|
8275
|
+
console.log(JSON.stringify(routeDocument(slice, result), null, 2));
|
|
7989
8276
|
return;
|
|
7990
8277
|
}
|
|
7991
8278
|
const asPct = (v) => `${(v * 100).toFixed(0)}%`;
|
|
7992
|
-
|
|
7993
|
-
|
|
7994
|
-
|
|
8279
|
+
say();
|
|
8280
|
+
say(` ${c.dim(wrap(t.route.agreement(asPct(result.crossAgreement), asPct(result.selfAgreement)), 74, ' '))}`);
|
|
8281
|
+
say();
|
|
7995
8282
|
if (result.verdict === 'inconclusive') {
|
|
7996
|
-
|
|
8283
|
+
say(` ${c.bold(wrap(t.route.inconclusive(), 74, ' '))}`);
|
|
7997
8284
|
}
|
|
7998
8285
|
else if (result.verdict === 'diverges') {
|
|
7999
|
-
|
|
8286
|
+
say(` ${c.yellow('!')} ${c.bold(wrap(t.route.diverges(worth), 74, ' '))}`);
|
|
8000
8287
|
}
|
|
8001
8288
|
else {
|
|
8002
|
-
|
|
8289
|
+
say(` ${c.green('✓')} ${c.bold(wrap(t.route.holds(worth), 74, ' '))}`);
|
|
8003
8290
|
}
|
|
8004
8291
|
/**
|
|
8005
8292
|
* Printed on every verdict including the good one. Agreement is not
|
|
@@ -8007,8 +8294,8 @@ async function commandRoute(args, pricing, t) {
|
|
|
8007
8294
|
* ever right, and a green tick that let somebody forget that would be the tool
|
|
8008
8295
|
* overstating what it knows.
|
|
8009
8296
|
*/
|
|
8010
|
-
|
|
8011
|
-
|
|
8297
|
+
say(` ${c.dim(wrap(t.route.yours(), 74, ' '))}`);
|
|
8298
|
+
say();
|
|
8012
8299
|
}
|
|
8013
8300
|
/**
|
|
8014
8301
|
* `trazum baseline <dir>` — record what the estate costs now.
|
|
@@ -8163,8 +8450,12 @@ async function scanPrompts(root, args, flagBudget, config, counter, level, t, lo
|
|
|
8163
8450
|
}
|
|
8164
8451
|
const verdicts = [];
|
|
8165
8452
|
const declined = [];
|
|
8453
|
+
// Every prompt this run judged, so the report's band is the widest of them
|
|
8454
|
+
// rather than one figure asserted over text of several kinds.
|
|
8455
|
+
const judgedTexts = [];
|
|
8166
8456
|
for (const relativePath of files) {
|
|
8167
8457
|
const text = capInput(await readFile(join(root, relativePath), 'utf8'), relativePath, maxInputFlag(args, t), t);
|
|
8458
|
+
judgedTexts.push(text);
|
|
8168
8459
|
// Budgets are keyed on paths as written in the repository, so a pattern like
|
|
8169
8460
|
// `prompts/**` has to be matched against the path including the root the
|
|
8170
8461
|
// user passed — not against the name relative to it.
|
|
@@ -8194,7 +8485,7 @@ async function scanPrompts(root, args, flagBudget, config, counter, level, t, lo
|
|
|
8194
8485
|
if (verdicts.length === 0 && declined.length === 0) {
|
|
8195
8486
|
throw new Error(t.errors.noPromptsFound(root, extensions.join(' ')));
|
|
8196
8487
|
}
|
|
8197
|
-
return { verdicts, declined, truncated, extensions };
|
|
8488
|
+
return { verdicts, declined, truncated, extensions, band: widestBand(judgedTexts) };
|
|
8198
8489
|
}
|
|
8199
8490
|
async function checkDirectory(root, args, flagBudget, config, counter, level, t, locale, pricing, only) {
|
|
8200
8491
|
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
@@ -8213,7 +8504,7 @@ async function checkDirectory(root, args, flagBudget, config, counter, level, t,
|
|
|
8213
8504
|
return;
|
|
8214
8505
|
only = checkable;
|
|
8215
8506
|
}
|
|
8216
|
-
const { verdicts, declined, truncated, extensions } = await scanPrompts(root, args, flagBudget, config, counter, level, t, locale, pricing, only);
|
|
8507
|
+
const { verdicts, declined, truncated, extensions, band } = await scanPrompts(root, args, flagBudget, config, counter, level, t, locale, pricing, only);
|
|
8217
8508
|
/**
|
|
8218
8509
|
* The baseline gate, when the config declares one and `--no-baseline` did not
|
|
8219
8510
|
* switch it off for this run.
|
|
@@ -8270,6 +8561,7 @@ async function checkDirectory(root, args, flagBudget, config, counter, level, t,
|
|
|
8270
8561
|
await writeMarkdown(args, () => renderCheckMarkdown({
|
|
8271
8562
|
target: root,
|
|
8272
8563
|
verdicts,
|
|
8564
|
+
band,
|
|
8273
8565
|
level,
|
|
8274
8566
|
tokenSource: counter.source,
|
|
8275
8567
|
truncated,
|
|
@@ -8411,6 +8703,15 @@ function joinPosix(root, relativePath) {
|
|
|
8411
8703
|
* never "delete this", and nothing here edits the prompt.
|
|
8412
8704
|
*/
|
|
8413
8705
|
async function commandPrune(args, t) {
|
|
8706
|
+
// Same as `route`: under `--json` stdout is the document, and the estimate
|
|
8707
|
+
// a reader has to see before spending goes to stderr rather than away.
|
|
8708
|
+
const asJson = boolFlag(args, 'json');
|
|
8709
|
+
const say = (line = '') => {
|
|
8710
|
+
if (asJson)
|
|
8711
|
+
console.error(line);
|
|
8712
|
+
else
|
|
8713
|
+
console.log(line);
|
|
8714
|
+
};
|
|
8414
8715
|
const prompt = await readInput(args.positional[0], t, maxInputFlag(args, t));
|
|
8415
8716
|
const casesPath = stringFlag(args, 'cases');
|
|
8416
8717
|
if (!casesPath)
|
|
@@ -8424,10 +8725,10 @@ async function commandPrune(args, t) {
|
|
|
8424
8725
|
const calls = plannedCalls(examples.length, inputs.length);
|
|
8425
8726
|
// Printed before the key is even looked up, so somebody weighing it up does not
|
|
8426
8727
|
// need a configured provider to see the number.
|
|
8427
|
-
|
|
8428
|
-
|
|
8728
|
+
say();
|
|
8729
|
+
say(c.bold(t.prune.estimate(examples.length, inputs.length, calls)));
|
|
8429
8730
|
if (!boolFlag(args, 'yes')) {
|
|
8430
|
-
|
|
8731
|
+
say(c.yellow(` ${t.prune.needsConsent()}`));
|
|
8431
8732
|
return;
|
|
8432
8733
|
}
|
|
8433
8734
|
const provider = providerFromEnv();
|
|
@@ -8437,14 +8738,14 @@ async function commandPrune(args, t) {
|
|
|
8437
8738
|
concurrency: numberFlag(args, 'concurrency', 3, t),
|
|
8438
8739
|
});
|
|
8439
8740
|
if (boolFlag(args, 'json')) {
|
|
8440
|
-
console.log(JSON.stringify(report, null, 2));
|
|
8741
|
+
console.log(JSON.stringify(pruneDocument(report), null, 2));
|
|
8441
8742
|
return;
|
|
8442
8743
|
}
|
|
8443
8744
|
const pct = (value) => `${(value * 100).toFixed(0)}%`;
|
|
8444
|
-
|
|
8445
|
-
|
|
8446
|
-
|
|
8447
|
-
|
|
8745
|
+
say();
|
|
8746
|
+
say(sectionHeading(t.prune.heading(provider.model)));
|
|
8747
|
+
say(` ${c.dim(t.prune.selfAgreement(pct(report.selfAgreement)))}`);
|
|
8748
|
+
say();
|
|
8448
8749
|
for (const contribution of report.contributions) {
|
|
8449
8750
|
/**
|
|
8450
8751
|
* The mark points at what the reader can act on, which is the *recoverable*
|
|
@@ -8461,7 +8762,7 @@ async function commandPrune(args, t) {
|
|
|
8461
8762
|
: needed
|
|
8462
8763
|
? t.prune.verdictNeeded()
|
|
8463
8764
|
: t.prune.verdictRecoverable();
|
|
8464
|
-
|
|
8765
|
+
say(` ${mark} ${t.prune.line(contribution.index + 1, contribution.tokens, pct(contribution.agreementWithout))}`
|
|
8465
8766
|
+ ` ${unknown ? c.yellow(label) : needed ? c.dim(label) : c.green(label)}`);
|
|
8466
8767
|
/**
|
|
8467
8768
|
* The first line that is not the header, because the header is the same on
|
|
@@ -8471,15 +8772,15 @@ async function commandPrune(args, t) {
|
|
|
8471
8772
|
*/
|
|
8472
8773
|
const lines = contribution.text.split('\n').filter((line) => line.trim() !== '');
|
|
8473
8774
|
const body = lines.find((line) => !/^\s*(?:#+\s*)?(?:example|ejemplo)\b[\s:.-]*$/i.test(line));
|
|
8474
|
-
|
|
8775
|
+
say(` ${c.dim(truncate((body ?? lines[0] ?? '').trim(), 60))}`);
|
|
8475
8776
|
}
|
|
8476
|
-
|
|
8777
|
+
say();
|
|
8477
8778
|
if (report.recoverableTokens > 0) {
|
|
8478
|
-
|
|
8779
|
+
say(` ${t.prune.recoverable(report.recoverableTokens)}`);
|
|
8479
8780
|
}
|
|
8480
|
-
|
|
8481
|
-
|
|
8482
|
-
|
|
8781
|
+
say(` ${c.dim(wrap(t.prune.caveat(), 74, ' '))}`);
|
|
8782
|
+
say();
|
|
8783
|
+
say(c.dim(` ${t.eval.callsMade(report.callsMade)}`));
|
|
8483
8784
|
}
|
|
8484
8785
|
async function commandEval(args, config, t, locale) {
|
|
8485
8786
|
const prompt = await readInput(args.positional[0], t, maxInputFlag(args, t));
|
|
@@ -8729,6 +9030,18 @@ function printComparison(comparison, beforePath, afterPath, optimized, t) {
|
|
|
8729
9030
|
const DOCTOR_LIST_LIMIT = 8;
|
|
8730
9031
|
const BLAME_DEFAULT_LIMIT = 20;
|
|
8731
9032
|
const BLAME_MAX_LIMIT = 500;
|
|
9033
|
+
/**
|
|
9034
|
+
* The band for a report covering several prompts.
|
|
9035
|
+
*
|
|
9036
|
+
* The widest of them, and the empty case is the widest published band rather
|
|
9037
|
+
* than a narrow default: a report over nothing has measured nothing, and
|
|
9038
|
+
* guessing narrow there is the one direction that produces a false claim.
|
|
9039
|
+
*/
|
|
9040
|
+
function widestBand(texts) {
|
|
9041
|
+
if (texts.length === 0)
|
|
9042
|
+
return ESTIMATE_ERROR_BAND_PCT;
|
|
9043
|
+
return Math.max(...texts.map((text) => bandFor(text)));
|
|
9044
|
+
}
|
|
8732
9045
|
/**
|
|
8733
9046
|
* `trazum blame <file>` — what happened to this prompt's cost, and who did it.
|
|
8734
9047
|
*
|
|
@@ -8803,14 +9116,32 @@ async function commandBlame(args, config, pricing, t) {
|
|
|
8803
9116
|
// put the newest first: this is a history, and histories are read backwards.
|
|
8804
9117
|
const shown = rows.slice(revisions.length > limit ? 1 : 0).reverse();
|
|
8805
9118
|
const truncatedHistory = revisions.length > limit;
|
|
9119
|
+
/*
|
|
9120
|
+
The band of this file's own text, read once and handed to both renderers.
|
|
9121
|
+
|
|
9122
|
+
`blame` walks one file across revisions, so the kind of text is a property
|
|
9123
|
+
of the file, and today's content is the closest thing to it that exists
|
|
9124
|
+
here. A file that is gone from the working tree — renamed away, or deleted
|
|
9125
|
+
in the commit being blamed — falls back to the widest published band, which
|
|
9126
|
+
is exactly what that constant is for: a caller that cannot see the text can
|
|
9127
|
+
only guess in the safe direction.
|
|
9128
|
+
*/
|
|
9129
|
+
let band = ESTIMATE_ERROR_BAND_PCT;
|
|
9130
|
+
try {
|
|
9131
|
+
band = bandFor(await readFile(resolvePath(process.cwd(), target), 'utf8'));
|
|
9132
|
+
}
|
|
9133
|
+
catch {
|
|
9134
|
+
// Not on disk now. The wide band stands and says so by being wide.
|
|
9135
|
+
}
|
|
8806
9136
|
await writeMarkdown(args, () => renderBlameMarkdown({
|
|
8807
9137
|
repoPath,
|
|
8808
9138
|
rows: shown,
|
|
8809
9139
|
truncated: truncatedHistory,
|
|
8810
9140
|
netCost: netCostOf(shown, args, config, pricing, t),
|
|
9141
|
+
band,
|
|
8811
9142
|
t,
|
|
8812
9143
|
}));
|
|
8813
|
-
printBlame(shown, { repoPath, args, config, pricing, t, truncated: truncatedHistory });
|
|
9144
|
+
printBlame(shown, { repoPath, args, config, pricing, t, truncated: truncatedHistory, band });
|
|
8814
9145
|
}
|
|
8815
9146
|
/**
|
|
8816
9147
|
* What the movement across a history costs per month, or `null`.
|
|
@@ -8946,7 +9277,7 @@ function printBlame(rows, context) {
|
|
|
8946
9277
|
console.log(`\n${c.bold(t.blame.biggestRise())}`);
|
|
8947
9278
|
console.log(` ${t.blame.biggestRiseDetail(n(worst.delta), worst.revision.author, truncate(worst.revision.subject, 60), worst.revision.shortSha)}`);
|
|
8948
9279
|
}
|
|
8949
|
-
console.log(`\n${c.dim(t.blame.estimateNote())}\n`);
|
|
9280
|
+
console.log(`\n${c.dim(t.blame.estimateNote(context.band))}\n`);
|
|
8950
9281
|
}
|
|
8951
9282
|
/**
|
|
8952
9283
|
* `trazum rank <dir>` — which of these prompts to fix first.
|
|
@@ -9306,6 +9637,12 @@ async function main() {
|
|
|
9306
9637
|
case 'from-otel':
|
|
9307
9638
|
await commandFromOtel(args, t);
|
|
9308
9639
|
break;
|
|
9640
|
+
case 'from-litellm':
|
|
9641
|
+
await commandFromLiteLlm(args, t);
|
|
9642
|
+
break;
|
|
9643
|
+
case 'from-helicone':
|
|
9644
|
+
await commandFromHelicone(args, t);
|
|
9645
|
+
break;
|
|
9309
9646
|
case 'switch':
|
|
9310
9647
|
await commandSwitch(args, pricing, t);
|
|
9311
9648
|
break;
|