@trazum/cli 1.80.1 → 1.82.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/gateway-server.d.ts +38 -1
- package/dist/gateway-server.d.ts.map +1 -1
- package/dist/gateway-server.js +125 -9
- package/dist/gateway-server.js.map +1 -1
- package/dist/i18n/en.d.ts +1 -1
- package/dist/i18n/en.d.ts.map +1 -1
- package/dist/i18n/en.js +437 -376
- package/dist/i18n/en.js.map +1 -1
- package/dist/i18n/es.d.ts.map +1 -1
- package/dist/i18n/es.js +68 -5
- package/dist/i18n/es.js.map +1 -1
- package/dist/i18n/types.d.ts +74 -6
- package/dist/i18n/types.d.ts.map +1 -1
- package/dist/index.d.ts +54 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +545 -55
- package/dist/index.js.map +1 -1
- package/dist/markdown.d.ts +10 -0
- package/dist/markdown.d.ts.map +1 -1
- package/dist/markdown.js +4 -4
- package/dist/markdown.js.map +1 -1
- package/package.json +2 -2
- package/src/gateway-server.ts +160 -9
- package/src/i18n/en.ts +454 -378
- package/src/i18n/es.ts +85 -7
- package/src/i18n/types.ts +76 -6
- package/src/index.ts +602 -54
- package/src/markdown.ts +14 -4
package/dist/index.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
2
3
|
import { readdirSync, readFileSync } from 'node:fs';
|
|
3
4
|
import { mkdir, mkdtemp, open, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
|
|
4
5
|
import { cpus, tmpdir } from 'node:os';
|
|
@@ -6,7 +7,7 @@ import { createInterface } from 'node:readline/promises';
|
|
|
6
7
|
import { dirname, join, resolve as resolvePath } from 'node:path';
|
|
7
8
|
import { fileURLToPath } from 'node:url';
|
|
8
9
|
import { gunzipSync } from 'node:zlib';
|
|
9
|
-
import { CONTRACT_NAMES, contractSchema, applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, bucketedCacheEconomics, bucketedProfile, buildHistory, buildPlan, connectorFor, CONNECTORS, normalizeAnthropicUsage, normalizeOpenAIUsage, bucketsFromRecords, evaluateWatch, firedKey, pruneRecords, recordsFromBuckets, storeInventory, storedReportFrom, verifyPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, bandGoverns, countTokensAnthropic, DEFAULT_USAGE, budgetPositions, conform, BREAK_EVEN_BAND, allocate, annualRecord, replayCommitment, coversTheTerm, runExperiment, qualityGate, semanticPassCost, verifySemanticProposals, SEMANTIC_SYSTEM_PROMPT, ladderPosition, validateLadder, outcomeReport, rankPerOutcome, FAILURE_POLICIES, detectFromSource, matchLocale, parsePlanDocument, waiverDay, waiverHistory, proposeInit, MIN_RATE_DAYS, parseConfig, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, rollUp, heartbeats, ruleYield, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, matchGlob, MAX_INPUT_CHARS, moneyIsComparable, mostSpecificMatch, assemble, interview, nearestName, slot, SLOT_IDS, optimize, parseBaseline, PHRASE_LANGUAGES, detectTextLanguage, dictionaryStanding, languagesWithStanding, indexUsage, parseUsageLine, plannedCalls, claudeCodeRecords, otelRecords, ownRate, positionAt, positionReport, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, switchAnalysis, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
|
|
10
|
+
import { CONTRACT_NAMES, contractSchema, applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, bucketedCacheEconomics, bucketedProfile, buildHistory, buildPlan, connectorFor, CONNECTORS, normalizeAnthropicUsage, normalizeOpenAIUsage, bucketsFromRecords, evaluateWatch, firedKey, pruneRecords, recordsFromBuckets, storeInventory, storedReportFrom, verifyPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, bandGoverns, countTokensAnthropic, DEFAULT_USAGE, budgetPositions, conform, BREAK_EVEN_BAND, allocate, annualRecord, replayCommitment, coversTheTerm, runExperiment, qualityGate, semanticPassCost, verifySemanticProposals, SEMANTIC_SYSTEM_PROMPT, ladderPosition, validateLadder, outcomeReport, rankPerOutcome, FAILURE_POLICIES, detectFromSource, matchLocale, parsePlanDocument, waiverDay, waiverHistory, proposeInit, MIN_RATE_DAYS, parseConfig, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, rollUp, heartbeats, ruleYield, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, matchGlob, MAX_INPUT_CHARS, moneyIsComparable, mostSpecificMatch, assemble, interview, nearestName, slot, SLOT_IDS, optimize, parseBaseline, PHRASE_LANGUAGES, detectTextLanguage, dictionaryStanding, languagesWithStanding, indexUsage, parseUsageLine, plannedCalls, claudeCodeRecords, ESTIMATE_ERROR_BAND_PCT, bandFor, foreignTokenizer, measuredForeignError, heliconeRecords, litellmRecords, otelRecords, ownRate, positionAt, positionReport, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, switchAnalysis, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
|
|
10
11
|
import { cacheDir, cacheStats, cachingProvider, clearCache } from './suggest-cache.js';
|
|
11
12
|
import { dayOf, formatGap, median, spanDays } from './time.js';
|
|
12
13
|
// Everything that reads the filesystem, on its own entry point so the web
|
|
@@ -103,6 +104,7 @@ const VALUE_FLAGS = new Set([
|
|
|
103
104
|
'gpu-usd-hour',
|
|
104
105
|
'tokens-per-second',
|
|
105
106
|
'utilization',
|
|
107
|
+
'state',
|
|
106
108
|
]);
|
|
107
109
|
function parseArgs(argv, t) {
|
|
108
110
|
const flags = new Map();
|
|
@@ -379,8 +381,10 @@ const COMMAND_FLAGS = {
|
|
|
379
381
|
schema: [],
|
|
380
382
|
rollup: ['json', 'html-out'],
|
|
381
383
|
position: ['json', 'html-out', 'pricing', 'pricing-live'],
|
|
382
|
-
'from-claude-code': ['label', 'label-from-project', 'out', 'o'],
|
|
384
|
+
'from-claude-code': ['label', 'label-from-project', 'out', 'o', 'state'],
|
|
383
385
|
'from-otel': ['label-from-service', 'out', 'o'],
|
|
386
|
+
'from-litellm': ['out', 'o'],
|
|
387
|
+
'from-helicone': ['out', 'o'],
|
|
384
388
|
switch: ['to', 'migration-usd', 'cases'],
|
|
385
389
|
ownrate: ['gpu-usd-hour', 'tokens-per-second', 'utilization'],
|
|
386
390
|
pulse: ['json', 'max-stale-hours'],
|
|
@@ -481,11 +485,11 @@ function renderDiff(before, after, t) {
|
|
|
481
485
|
/**
|
|
482
486
|
* The provider's name when the estimator was not calibrated for it.
|
|
483
487
|
*
|
|
484
|
-
* `estimateTokens` is a heuristic tuned against Claude's tokenizer, and
|
|
485
|
-
*
|
|
486
|
-
*
|
|
487
|
-
*
|
|
488
|
-
*
|
|
488
|
+
* `estimateTokens` is a heuristic tuned against Claude's tokenizer, and every
|
|
489
|
+
* band descends from that. Printing one beside a GPT or Kimi figure states a
|
|
490
|
+
* precision nobody has measured for that family — and since the catalogue grew
|
|
491
|
+
* past Anthropic, that is most of it. Returns null when the model is
|
|
492
|
+
* Anthropic's, where the band is at least the claim it was written for.
|
|
489
493
|
*/
|
|
490
494
|
function offFamilyName(modelId) {
|
|
491
495
|
const provider = getModel(modelId).provider;
|
|
@@ -493,6 +497,17 @@ function offFamilyName(modelId) {
|
|
|
493
497
|
return null;
|
|
494
498
|
return getModel(modelId).displayName;
|
|
495
499
|
}
|
|
500
|
+
/**
|
|
501
|
+
* How far off the estimator has been measured on this model's family.
|
|
502
|
+
*
|
|
503
|
+
* Read from the catalogue's provider rather than from the display name, because
|
|
504
|
+
* the name is what a reader sees and the id is what was measured. Null on the
|
|
505
|
+
* calibrated family and on every family nobody has run — an unmeasured one gets
|
|
506
|
+
* a sentence saying so, never a borrowed figure.
|
|
507
|
+
*/
|
|
508
|
+
function offFamilyError(modelId) {
|
|
509
|
+
return measuredForeignError(foreignTokenizer(getModel(modelId).provider ?? null));
|
|
510
|
+
}
|
|
496
511
|
/**
|
|
497
512
|
* Language codes as names, in the reader's language.
|
|
498
513
|
*
|
|
@@ -557,7 +572,7 @@ namedScenario = false,
|
|
|
557
572
|
measured = null) {
|
|
558
573
|
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
559
574
|
const sourceNote = result.tokenSource === 'heuristic'
|
|
560
|
-
? c.dim(t.report.estimated(offFamilyName(result.usage.model)))
|
|
575
|
+
? c.dim(t.report.estimated(offFamilyName(result.usage.model), bandFor(result.optimized), offFamilyError(result.usage.model)))
|
|
561
576
|
: c.dim(t.report.exactCount());
|
|
562
577
|
/**
|
|
563
578
|
* The largest lever, first.
|
|
@@ -2025,6 +2040,76 @@ function projectLabelFor(file) {
|
|
|
2025
2040
|
const trimmed = folder.replace(/^-+/, '');
|
|
2026
2041
|
return trimmed !== '' ? trimmed : folder;
|
|
2027
2042
|
}
|
|
2043
|
+
/** How much of the run-up to `offset` is fingerprinted. */
|
|
2044
|
+
const STATE_DIGEST_BYTES = 4096;
|
|
2045
|
+
const digestOf = (bytes) => createHash('sha256').update(bytes).digest('hex');
|
|
2046
|
+
const readState = async (path) => {
|
|
2047
|
+
try {
|
|
2048
|
+
const parsed = JSON.parse(await readFile(path, 'utf8'));
|
|
2049
|
+
if (typeof parsed === 'object' &&
|
|
2050
|
+
parsed !== null &&
|
|
2051
|
+
parsed.schemaVersion === 1 &&
|
|
2052
|
+
typeof parsed.files === 'object') {
|
|
2053
|
+
return parsed;
|
|
2054
|
+
}
|
|
2055
|
+
}
|
|
2056
|
+
catch {
|
|
2057
|
+
// A missing, unreadable or unrecognised state file is a cold start, not an
|
|
2058
|
+
// error: the answer it produces is the same one, computed the slow way.
|
|
2059
|
+
}
|
|
2060
|
+
return { schemaVersion: 1, files: {} };
|
|
2061
|
+
};
|
|
2062
|
+
/**
|
|
2063
|
+
* The byte offset of a line index within a chunk, counted in bytes rather than
|
|
2064
|
+
* characters.
|
|
2065
|
+
*
|
|
2066
|
+
* `String.prototype.split` counts UTF-16 code units and a transcript is UTF-8
|
|
2067
|
+
* with prompts in it, so a character index used as a byte offset would land
|
|
2068
|
+
* mid-sequence on the first accented character and corrupt every resume after
|
|
2069
|
+
* it. The newlines are found in the Buffer.
|
|
2070
|
+
*/
|
|
2071
|
+
const byteOffsetOfLine = (chunk, line) => {
|
|
2072
|
+
if (line <= 0)
|
|
2073
|
+
return 0;
|
|
2074
|
+
let seen = 0;
|
|
2075
|
+
for (let i = 0; i < chunk.length; i += 1) {
|
|
2076
|
+
if (chunk[i] !== 0x0a)
|
|
2077
|
+
continue;
|
|
2078
|
+
seen += 1;
|
|
2079
|
+
if (seen === line)
|
|
2080
|
+
return i + 1;
|
|
2081
|
+
}
|
|
2082
|
+
return chunk.length;
|
|
2083
|
+
};
|
|
2084
|
+
const resumableRead = async (file, state) => {
|
|
2085
|
+
const key = resolvePath(file);
|
|
2086
|
+
const entry = state.files[key];
|
|
2087
|
+
const handle = await open(key, 'r');
|
|
2088
|
+
try {
|
|
2089
|
+
const { size } = await handle.stat();
|
|
2090
|
+
let start = 0;
|
|
2091
|
+
let truncateOutTo = 0;
|
|
2092
|
+
if (entry !== undefined && entry.offset > 0 && entry.offset <= size) {
|
|
2093
|
+
const runUp = Math.min(STATE_DIGEST_BYTES, entry.offset);
|
|
2094
|
+
const before = Buffer.alloc(runUp);
|
|
2095
|
+
await handle.read(before, 0, runUp, entry.offset - runUp);
|
|
2096
|
+
if (digestOf(before) === entry.digest) {
|
|
2097
|
+
start = entry.offset;
|
|
2098
|
+
truncateOutTo = entry.out;
|
|
2099
|
+
}
|
|
2100
|
+
}
|
|
2101
|
+
const chunk = Buffer.alloc(size - start);
|
|
2102
|
+
if (chunk.length > 0)
|
|
2103
|
+
await handle.read(chunk, 0, chunk.length, start);
|
|
2104
|
+
return {
|
|
2105
|
+
chunk,
|
|
2106
|
+
read: { text: chunk.toString('utf8'), skipped: start, truncateOutTo, chunkStart: start },
|
|
2107
|
+
};
|
|
2108
|
+
}
|
|
2109
|
+
finally {
|
|
2110
|
+
await handle.close();
|
|
2111
|
+
}
|
|
2112
|
+
};
|
|
2028
2113
|
async function commandFromClaudeCode(args, t) {
|
|
2029
2114
|
const target = args.positional[0];
|
|
2030
2115
|
if (target === undefined)
|
|
@@ -2066,6 +2151,28 @@ async function commandFromClaudeCode(args, t) {
|
|
|
2066
2151
|
* says so.
|
|
2067
2152
|
*/
|
|
2068
2153
|
const labelFromProject = boolFlag(args, 'label-from-project', info.isDirectory());
|
|
2154
|
+
/**
|
|
2155
|
+
* `--state` is for one transcript, deliberately.
|
|
2156
|
+
*
|
|
2157
|
+
* The state ties a transcript offset to a length of the output file, and
|
|
2158
|
+
* with several transcripts appending to one output there is no single length
|
|
2159
|
+
* that means "everything settled": the second file's records sit after the
|
|
2160
|
+
* first file's unsettled tail. A directory is re-read in full, which is what
|
|
2161
|
+
* it was doing before and is the right cost for a walk that is not a status
|
|
2162
|
+
* line refreshing every turn.
|
|
2163
|
+
*/
|
|
2164
|
+
const statePath = stringFlag(args, 'state');
|
|
2165
|
+
if (statePath !== undefined && info.isDirectory())
|
|
2166
|
+
throw new Error(t.fromClaudeCode.stateNeedsFile());
|
|
2167
|
+
if (statePath !== undefined && stringFlag(args, 'out') === undefined && stringFlag(args, 'o') === undefined) {
|
|
2168
|
+
throw new Error(t.fromClaudeCode.stateNeedsOut());
|
|
2169
|
+
}
|
|
2170
|
+
const state = statePath !== undefined ? await readState(statePath) : undefined;
|
|
2171
|
+
let resumed;
|
|
2172
|
+
let chunk;
|
|
2173
|
+
let resumeLine = 0;
|
|
2174
|
+
let settledRecords = 0;
|
|
2175
|
+
let settledOutBytes = 0;
|
|
2069
2176
|
const lines = [];
|
|
2070
2177
|
let records = 0;
|
|
2071
2178
|
let collapsed = 0;
|
|
@@ -2077,7 +2184,16 @@ async function commandFromClaudeCode(args, t) {
|
|
|
2077
2184
|
let disagreements = 0;
|
|
2078
2185
|
let synthetic = 0;
|
|
2079
2186
|
for (const file of files) {
|
|
2080
|
-
|
|
2187
|
+
let text;
|
|
2188
|
+
if (state !== undefined) {
|
|
2189
|
+
const read = await resumableRead(file, state);
|
|
2190
|
+
chunk = read.chunk;
|
|
2191
|
+
resumed = read.read;
|
|
2192
|
+
text = read.read.text;
|
|
2193
|
+
}
|
|
2194
|
+
else {
|
|
2195
|
+
text = await readFile(file, 'utf8');
|
|
2196
|
+
}
|
|
2081
2197
|
const projectLabel = labelFromProject ? projectLabelFor(file) : undefined;
|
|
2082
2198
|
const conversion = claudeCodeRecords(text, {
|
|
2083
2199
|
...(label !== undefined
|
|
@@ -2088,6 +2204,8 @@ async function commandFromClaudeCode(args, t) {
|
|
|
2088
2204
|
});
|
|
2089
2205
|
for (const record of conversion.records)
|
|
2090
2206
|
lines.push(JSON.stringify(record));
|
|
2207
|
+
resumeLine = conversion.resume.line;
|
|
2208
|
+
settledRecords = conversion.resume.records;
|
|
2091
2209
|
records += conversion.records.length;
|
|
2092
2210
|
collapsed += conversion.collapsed;
|
|
2093
2211
|
noRequestId += conversion.noRequestId;
|
|
@@ -2100,13 +2218,61 @@ async function commandFromClaudeCode(args, t) {
|
|
|
2100
2218
|
}
|
|
2101
2219
|
const out = stringFlag(args, 'out') ?? stringFlag(args, 'o');
|
|
2102
2220
|
if (out !== undefined) {
|
|
2103
|
-
|
|
2221
|
+
if (statePath !== undefined) {
|
|
2222
|
+
/**
|
|
2223
|
+
* Append rather than rewrite, and drop the tail first.
|
|
2224
|
+
*
|
|
2225
|
+
* The tail is the last call as the previous run recorded it, and this
|
|
2226
|
+
* run has just re-derived it from its own first line. Truncating is what
|
|
2227
|
+
* makes the two runs add up to exactly one reading of the transcript
|
|
2228
|
+
* rather than one and a bit.
|
|
2229
|
+
*/
|
|
2230
|
+
const handle = await open(out, resumed.truncateOutTo > 0 ? 'r+' : 'w');
|
|
2231
|
+
try {
|
|
2232
|
+
await handle.truncate(resumed.truncateOutTo);
|
|
2233
|
+
const body = lines.join('\n') + (lines.length > 0 ? '\n' : '');
|
|
2234
|
+
if (body !== '')
|
|
2235
|
+
await handle.write(body, resumed.truncateOutTo, 'utf8');
|
|
2236
|
+
settledOutBytes =
|
|
2237
|
+
resumed.truncateOutTo +
|
|
2238
|
+
lines
|
|
2239
|
+
.slice(0, settledRecords)
|
|
2240
|
+
.reduce((sum, line) => sum + Buffer.byteLength(line + '\n', 'utf8'), 0);
|
|
2241
|
+
}
|
|
2242
|
+
finally {
|
|
2243
|
+
await handle.close();
|
|
2244
|
+
}
|
|
2245
|
+
}
|
|
2246
|
+
else {
|
|
2247
|
+
await writeFile(out, lines.join('\n') + (lines.length > 0 ? '\n' : ''), 'utf8');
|
|
2248
|
+
}
|
|
2104
2249
|
console.error(t.fromClaudeCode.written(out));
|
|
2105
2250
|
}
|
|
2106
2251
|
else {
|
|
2107
2252
|
for (const line of lines)
|
|
2108
2253
|
console.log(line);
|
|
2109
2254
|
}
|
|
2255
|
+
if (statePath !== undefined) {
|
|
2256
|
+
const key = resolvePath(files[0]);
|
|
2257
|
+
const nextOffset = resumed.chunkStart + byteOffsetOfLine(chunk, resumeLine);
|
|
2258
|
+
const runUp = Math.min(STATE_DIGEST_BYTES, nextOffset);
|
|
2259
|
+
const before = Buffer.alloc(runUp);
|
|
2260
|
+
if (runUp > 0) {
|
|
2261
|
+
const handle = await open(key, 'r');
|
|
2262
|
+
try {
|
|
2263
|
+
await handle.read(before, 0, runUp, nextOffset - runUp);
|
|
2264
|
+
}
|
|
2265
|
+
finally {
|
|
2266
|
+
await handle.close();
|
|
2267
|
+
}
|
|
2268
|
+
}
|
|
2269
|
+
const next = {
|
|
2270
|
+
schemaVersion: 1,
|
|
2271
|
+
files: { [key]: { offset: nextOffset, out: settledOutBytes, digest: digestOf(before) } },
|
|
2272
|
+
};
|
|
2273
|
+
await writeFile(statePath, JSON.stringify(next, null, 2) + '\n', 'utf8');
|
|
2274
|
+
console.error(t.fromClaudeCode.resumed(resumed.skipped, nextOffset));
|
|
2275
|
+
}
|
|
2110
2276
|
console.error(t.fromClaudeCode.summary(files.length, records));
|
|
2111
2277
|
if (collapsed > 0)
|
|
2112
2278
|
console.error(t.fromClaudeCode.collapsed(collapsed));
|
|
@@ -2324,6 +2490,184 @@ async function commandFromOtel(args, t) {
|
|
|
2324
2490
|
if (unparseable > 0)
|
|
2325
2491
|
console.error(t.fromOtel.unparseable(unparseable));
|
|
2326
2492
|
}
|
|
2493
|
+
/**
|
|
2494
|
+
* `trazum from-litellm <file|dir>` — a LiteLLM spend log as a usage log.
|
|
2495
|
+
*
|
|
2496
|
+
* The same pure-converter pattern again, pointed at the gateway a great many
|
|
2497
|
+
* teams already put in front of every provider: `LiteLLM_SpendLogs` is the
|
|
2498
|
+
* export most likely to already exist on somebody's disk. `litellmRecords` in
|
|
2499
|
+
* core does the reading; this is the walk and the honesty.
|
|
2500
|
+
*
|
|
2501
|
+
* Three of the four counts it prints exist because the alternative is a
|
|
2502
|
+
* flattering silence. Rows naming no model are not priced by the route they
|
|
2503
|
+
* took, rows with no tokens are logged calls nobody can price, and a
|
|
2504
|
+
* `cache_hit` flag is not a token split — so the cache verdicts read "cannot
|
|
2505
|
+
* tell" rather than a fabricated one, exactly as `from-otel` does.
|
|
2506
|
+
*
|
|
2507
|
+
* The fourth is the one that matters most: LiteLLM prices the same calls with
|
|
2508
|
+
* its own table, and that figure is printed beside Trazum's and never merged
|
|
2509
|
+
* into it. Two price tables summed into one total is how a report becomes
|
|
2510
|
+
* quietly wrong.
|
|
2511
|
+
*
|
|
2512
|
+
* The row carries `messages`, `response`, `api_key`, `requester_ip_address`
|
|
2513
|
+
* and `end_user`. None of it is read; a fixture plants a marker in each and
|
|
2514
|
+
* greps the whole output.
|
|
2515
|
+
*/
|
|
2516
|
+
async function commandFromLiteLlm(args, t) {
|
|
2517
|
+
const target = args.positional[0];
|
|
2518
|
+
if (target === undefined)
|
|
2519
|
+
throw new Error(t.fromLiteLlm.noPath());
|
|
2520
|
+
let info;
|
|
2521
|
+
try {
|
|
2522
|
+
info = await stat(target);
|
|
2523
|
+
}
|
|
2524
|
+
catch {
|
|
2525
|
+
throw new Error(t.fromLiteLlm.notFound(target));
|
|
2526
|
+
}
|
|
2527
|
+
const files = [];
|
|
2528
|
+
if (info.isDirectory()) {
|
|
2529
|
+
const entries = await readdir(target, { recursive: true, withFileTypes: true });
|
|
2530
|
+
for (const entry of entries) {
|
|
2531
|
+
if (entry.isFile() && /\.(json|jsonl|ndjson)$/i.test(entry.name)) {
|
|
2532
|
+
files.push(join(entry.parentPath, entry.name));
|
|
2533
|
+
}
|
|
2534
|
+
}
|
|
2535
|
+
files.sort();
|
|
2536
|
+
if (files.length === 0)
|
|
2537
|
+
throw new Error(t.fromLiteLlm.noExports(target));
|
|
2538
|
+
}
|
|
2539
|
+
else {
|
|
2540
|
+
files.push(target);
|
|
2541
|
+
}
|
|
2542
|
+
const lines = [];
|
|
2543
|
+
let rows = 0;
|
|
2544
|
+
let unnamedModel = 0;
|
|
2545
|
+
let noTokens = 0;
|
|
2546
|
+
let cacheFlagged = 0;
|
|
2547
|
+
let unparseable = 0;
|
|
2548
|
+
let reportedSpend = 0;
|
|
2549
|
+
let sawSpend = false;
|
|
2550
|
+
for (const file of files) {
|
|
2551
|
+
const conversion = litellmRecords(await readFile(file, 'utf8'));
|
|
2552
|
+
for (const record of conversion.records)
|
|
2553
|
+
lines.push(JSON.stringify(record));
|
|
2554
|
+
rows += conversion.rows;
|
|
2555
|
+
unnamedModel += conversion.unnamedModel;
|
|
2556
|
+
noTokens += conversion.noTokens;
|
|
2557
|
+
cacheFlagged += conversion.cacheFlagged;
|
|
2558
|
+
unparseable += conversion.unparseable;
|
|
2559
|
+
if (conversion.reportedSpendUsd !== null) {
|
|
2560
|
+
reportedSpend += conversion.reportedSpendUsd;
|
|
2561
|
+
sawSpend = true;
|
|
2562
|
+
}
|
|
2563
|
+
}
|
|
2564
|
+
const out = stringFlag(args, 'out') ?? stringFlag(args, 'o');
|
|
2565
|
+
if (out !== undefined) {
|
|
2566
|
+
await writeFile(out, lines.join('\n') + (lines.length > 0 ? '\n' : ''), 'utf8');
|
|
2567
|
+
console.error(t.fromLiteLlm.written(out));
|
|
2568
|
+
}
|
|
2569
|
+
else {
|
|
2570
|
+
for (const line of lines)
|
|
2571
|
+
console.log(line);
|
|
2572
|
+
}
|
|
2573
|
+
console.error(t.fromLiteLlm.summary(files.length, rows));
|
|
2574
|
+
if (unnamedModel > 0)
|
|
2575
|
+
console.error(t.fromLiteLlm.unnamedModel(unnamedModel));
|
|
2576
|
+
if (noTokens > 0)
|
|
2577
|
+
console.error(t.fromLiteLlm.noTokens(noTokens));
|
|
2578
|
+
if (cacheFlagged > 0)
|
|
2579
|
+
console.error(t.fromLiteLlm.cacheFlagged(cacheFlagged));
|
|
2580
|
+
if (sawSpend)
|
|
2581
|
+
console.error(t.fromLiteLlm.reportedSpend(formatUsd(reportedSpend)));
|
|
2582
|
+
if (unparseable > 0)
|
|
2583
|
+
console.error(t.fromLiteLlm.unparseable(unparseable));
|
|
2584
|
+
}
|
|
2585
|
+
/**
|
|
2586
|
+
* `trazum from-helicone <file|dir>` — a Helicone request export as a usage log.
|
|
2587
|
+
*
|
|
2588
|
+
* The third converter, and the one that needs three columns where the others
|
|
2589
|
+
* need one: Helicone carries `request_model`, `model_override` and
|
|
2590
|
+
* `response_model`, and they can disagree. The response wins, because a bill
|
|
2591
|
+
* is about what was billed rather than what was intended, and the
|
|
2592
|
+
* disagreements are counted so a substitution is something the reader sees.
|
|
2593
|
+
*
|
|
2594
|
+
* Two absences are stated rather than filled in. There is no cache token
|
|
2595
|
+
* split on the row, only a flag; and a Helicone request id names one call and
|
|
2596
|
+
* never a conversation, so the records carry no session and the
|
|
2597
|
+
* conversation-shaped findings stay unavailable. Both are printed, because a
|
|
2598
|
+
* gap that is not said reads as a gap that is not there.
|
|
2599
|
+
*/
|
|
2600
|
+
async function commandFromHelicone(args, t) {
|
|
2601
|
+
const target = args.positional[0];
|
|
2602
|
+
if (target === undefined)
|
|
2603
|
+
throw new Error(t.fromHelicone.noPath());
|
|
2604
|
+
let info;
|
|
2605
|
+
try {
|
|
2606
|
+
info = await stat(target);
|
|
2607
|
+
}
|
|
2608
|
+
catch {
|
|
2609
|
+
throw new Error(t.fromHelicone.notFound(target));
|
|
2610
|
+
}
|
|
2611
|
+
const files = [];
|
|
2612
|
+
if (info.isDirectory()) {
|
|
2613
|
+
const entries = await readdir(target, { recursive: true, withFileTypes: true });
|
|
2614
|
+
for (const entry of entries) {
|
|
2615
|
+
if (entry.isFile() && /\.(json|jsonl|ndjson)$/i.test(entry.name)) {
|
|
2616
|
+
files.push(join(entry.parentPath, entry.name));
|
|
2617
|
+
}
|
|
2618
|
+
}
|
|
2619
|
+
files.sort();
|
|
2620
|
+
if (files.length === 0)
|
|
2621
|
+
throw new Error(t.fromHelicone.noExports(target));
|
|
2622
|
+
}
|
|
2623
|
+
else {
|
|
2624
|
+
files.push(target);
|
|
2625
|
+
}
|
|
2626
|
+
const lines = [];
|
|
2627
|
+
let rows = 0;
|
|
2628
|
+
let unnamedModel = 0;
|
|
2629
|
+
let disagreements = 0;
|
|
2630
|
+
let noTokens = 0;
|
|
2631
|
+
let cacheFlagged = 0;
|
|
2632
|
+
let unparseable = 0;
|
|
2633
|
+
for (const file of files) {
|
|
2634
|
+
const conversion = heliconeRecords(await readFile(file, 'utf8'));
|
|
2635
|
+
for (const record of conversion.records)
|
|
2636
|
+
lines.push(JSON.stringify(record));
|
|
2637
|
+
rows += conversion.rows;
|
|
2638
|
+
unnamedModel += conversion.unnamedModel;
|
|
2639
|
+
disagreements += conversion.modelDisagreements;
|
|
2640
|
+
noTokens += conversion.noTokens;
|
|
2641
|
+
cacheFlagged += conversion.cacheFlagged;
|
|
2642
|
+
unparseable += conversion.unparseable;
|
|
2643
|
+
}
|
|
2644
|
+
const out = stringFlag(args, 'out') ?? stringFlag(args, 'o');
|
|
2645
|
+
if (out !== undefined) {
|
|
2646
|
+
await writeFile(out, lines.join('\n') + (lines.length > 0 ? '\n' : ''), 'utf8');
|
|
2647
|
+
console.error(t.fromHelicone.written(out));
|
|
2648
|
+
}
|
|
2649
|
+
else {
|
|
2650
|
+
for (const line of lines)
|
|
2651
|
+
console.log(line);
|
|
2652
|
+
}
|
|
2653
|
+
console.error(t.fromHelicone.summary(files.length, rows));
|
|
2654
|
+
if (unnamedModel > 0)
|
|
2655
|
+
console.error(t.fromHelicone.unnamedModel(unnamedModel));
|
|
2656
|
+
if (disagreements > 0)
|
|
2657
|
+
console.error(t.fromHelicone.disagreements(disagreements));
|
|
2658
|
+
if (noTokens > 0)
|
|
2659
|
+
console.error(t.fromHelicone.noTokens(noTokens));
|
|
2660
|
+
if (cacheFlagged > 0)
|
|
2661
|
+
console.error(t.fromHelicone.cacheFlagged(cacheFlagged));
|
|
2662
|
+
// Said on every run that produced records, not only when something is
|
|
2663
|
+
// missing: the absence is a property of the format, and a reader who has
|
|
2664
|
+
// just converted a month needs to know which questions this log cannot
|
|
2665
|
+
// answer before they go looking for the answers.
|
|
2666
|
+
if (rows > 0)
|
|
2667
|
+
console.error(t.fromHelicone.noSessions());
|
|
2668
|
+
if (unparseable > 0)
|
|
2669
|
+
console.error(t.fromHelicone.unparseable(unparseable));
|
|
2670
|
+
}
|
|
2327
2671
|
/**
|
|
2328
2672
|
* `trazum position <usage.jsonl>` — where the month stands, measured.
|
|
2329
2673
|
*
|
|
@@ -4394,6 +4738,7 @@ async function commandCheck(args, config, pricing, t, locale) {
|
|
|
4394
4738
|
verdicts: [verdict],
|
|
4395
4739
|
level,
|
|
4396
4740
|
tokenSource: counter.source,
|
|
4741
|
+
band: bandFor(prompt),
|
|
4397
4742
|
truncated: false,
|
|
4398
4743
|
t,
|
|
4399
4744
|
}));
|
|
@@ -4501,11 +4846,16 @@ async function checkEmbedded(path, extraction, budget, args, counter, level, t,
|
|
|
4501
4846
|
}
|
|
4502
4847
|
const failures = verdicts.filter(isOverBudget);
|
|
4503
4848
|
const ok = failures.length === 0 && extraction.declined.length === 0;
|
|
4849
|
+
// Widest across the prompts this report covers, never an average: a run over
|
|
4850
|
+
// prose and a CSV covers text the estimator is 6% and 33% out on, and a
|
|
4851
|
+
// figure between them would describe neither.
|
|
4852
|
+
const band = widestBand(extraction.prompts.map((prompt) => prompt.text));
|
|
4504
4853
|
await writeMarkdown(args, () => renderCheckMarkdown({
|
|
4505
4854
|
target: path,
|
|
4506
4855
|
verdicts,
|
|
4507
4856
|
level,
|
|
4508
4857
|
tokenSource: counter.source,
|
|
4858
|
+
band,
|
|
4509
4859
|
truncated: false,
|
|
4510
4860
|
t,
|
|
4511
4861
|
}));
|
|
@@ -7743,12 +8093,102 @@ function reportProfileGaps(report, t, n, stalePricing = null) {
|
|
|
7743
8093
|
* exactly as `prune` does. A command that can spend somebody's money without
|
|
7744
8094
|
* telling them first is a command they stop trusting.
|
|
7745
8095
|
*/
|
|
8096
|
+
/**
|
|
8097
|
+
* The routing measurement, without the text that produced it.
|
|
8098
|
+
*
|
|
8099
|
+
* `route --json` printed `{ slice, evaluation }` — the whole `EvalReport`,
|
|
8100
|
+
* whose `cases[]` carries `input` (the caller's own case text), `baseline[0]`,
|
|
8101
|
+
* `baseline[1]` and `optimized` (three model answers). It carried no
|
|
8102
|
+
* `schemaVersion` either.
|
|
8103
|
+
*
|
|
8104
|
+
* `docs/json-output.md` opens by promising that **nothing here carries a
|
|
8105
|
+
* session key or prompt text**, and this is the document most likely to be
|
|
8106
|
+
* piped into a dashboard, pasted into a ticket or dropped into a browser. A
|
|
8107
|
+
* tool whose headline is that prompts never leave the machine cannot have its
|
|
8108
|
+
* most portable artefact be the one that carries them.
|
|
8109
|
+
*
|
|
8110
|
+
* The per-case *shape* survives: the two similarity scores are the evidence
|
|
8111
|
+
* behind the verdict, and a reader checking a borderline call needs them. Only
|
|
8112
|
+
* the strings go. Anyone who wants the answers has the human-readable output,
|
|
8113
|
+
* which never left the terminal.
|
|
8114
|
+
*
|
|
8115
|
+
* A named function rather than an inline object so the stripping is a thing a
|
|
8116
|
+
* test can call without spending a provider call to reach it.
|
|
8117
|
+
*/
|
|
8118
|
+
export function routeDocument(slice, result) {
|
|
8119
|
+
return {
|
|
8120
|
+
schemaVersion: 1,
|
|
8121
|
+
slice,
|
|
8122
|
+
evaluation: {
|
|
8123
|
+
provider: result.provider,
|
|
8124
|
+
model: result.model,
|
|
8125
|
+
candidateModel: result.candidateModel,
|
|
8126
|
+
verdict: result.verdict,
|
|
8127
|
+
selfAgreement: result.selfAgreement,
|
|
8128
|
+
crossAgreement: result.crossAgreement,
|
|
8129
|
+
callsMade: result.callsMade,
|
|
8130
|
+
cases: result.cases.map((one) => ({
|
|
8131
|
+
selfSimilarity: one.selfSimilarity,
|
|
8132
|
+
crossSimilarity: one.crossSimilarity,
|
|
8133
|
+
})),
|
|
8134
|
+
},
|
|
8135
|
+
};
|
|
8136
|
+
}
|
|
8137
|
+
/**
|
|
8138
|
+
* The example-pruning measurement, likewise.
|
|
8139
|
+
*
|
|
8140
|
+
* `ExampleContribution.text` is *"the example itself, so a report can quote its
|
|
8141
|
+
* first line"* — true of the terminal rendering and wrong of a document that
|
|
8142
|
+
* promises to carry no prompt text. A few-shot example **is** prompt text, and
|
|
8143
|
+
* this was the second command emitting it.
|
|
8144
|
+
*
|
|
8145
|
+
* `index` and `tokens` say which example without quoting it, which is what a
|
|
8146
|
+
* machine reading this needs: it already has the prompt.
|
|
8147
|
+
*/
|
|
8148
|
+
export function pruneDocument(report) {
|
|
8149
|
+
return {
|
|
8150
|
+
schemaVersion: 1,
|
|
8151
|
+
provider: report.provider,
|
|
8152
|
+
model: report.model,
|
|
8153
|
+
selfAgreement: report.selfAgreement,
|
|
8154
|
+
recoverableTokens: report.recoverableTokens,
|
|
8155
|
+
callsMade: report.callsMade,
|
|
8156
|
+
contributions: report.contributions.map((one) => ({
|
|
8157
|
+
index: one.index,
|
|
8158
|
+
tokens: one.tokens,
|
|
8159
|
+
agreementWithout: one.agreementWithout,
|
|
8160
|
+
verdict: one.verdict,
|
|
8161
|
+
})),
|
|
8162
|
+
};
|
|
8163
|
+
}
|
|
7746
8164
|
async function commandRoute(args, pricing, t) {
|
|
8165
|
+
/**
|
|
8166
|
+
* Under `--json`, stdout carries the document and nothing else.
|
|
8167
|
+
*
|
|
8168
|
+
* It carried the human preamble too — the slice it picked, the call count,
|
|
8169
|
+
* the "nothing has been spent yet" line — straight into the same stream as
|
|
8170
|
+
* the JSON, so `route --json > verdict.json` wrote a file no parser would
|
|
8171
|
+
* read. Found by running the command and feeding the file to the bridge
|
|
8172
|
+
* that reads it, which is the one thing the pure-function guards could not
|
|
8173
|
+
* do: `interchange.test.js` names `route` among the commands it cannot
|
|
8174
|
+
* drive, so nothing checked that its `--json` output was a document.
|
|
8175
|
+
*
|
|
8176
|
+
* The lines go to stderr rather than away. A reader running this in a
|
|
8177
|
+
* terminal still needs to see which slice was picked and what it will cost
|
|
8178
|
+
* before it costs it; a pipe needs the document. stderr gives both.
|
|
8179
|
+
*/
|
|
8180
|
+
const asJson = boolFlag(args, 'json');
|
|
8181
|
+
const say = (line = '') => {
|
|
8182
|
+
if (asJson)
|
|
8183
|
+
console.error(line);
|
|
8184
|
+
else
|
|
8185
|
+
console.log(line);
|
|
8186
|
+
};
|
|
7747
8187
|
const path = args.positional[0];
|
|
7748
8188
|
if (path === undefined) {
|
|
7749
|
-
|
|
7750
|
-
|
|
7751
|
-
|
|
8189
|
+
say();
|
|
8190
|
+
say(c.dim(wrap(t.route.noTarget(), 74, ' ')));
|
|
8191
|
+
say();
|
|
7752
8192
|
return;
|
|
7753
8193
|
}
|
|
7754
8194
|
const promptPath = stringFlag(args, 'prompt-file');
|
|
@@ -7770,16 +8210,16 @@ async function commandRoute(args, pricing, t) {
|
|
|
7770
8210
|
const available = report.byLabel
|
|
7771
8211
|
.map((r) => (r.label === UNLABELLED ? t.profile.unlabelled() : r.label))
|
|
7772
8212
|
.join(', ');
|
|
7773
|
-
|
|
7774
|
-
|
|
7775
|
-
|
|
8213
|
+
say();
|
|
8214
|
+
say(c.dim(wrap(t.route.labelNotFound(wanted, available), 74, ' ')));
|
|
8215
|
+
say();
|
|
7776
8216
|
return;
|
|
7777
8217
|
}
|
|
7778
8218
|
const slice = levers.slices.find((s) => s.route !== null && (wanted === undefined || s.label === wanted));
|
|
7779
8219
|
if (!slice?.route) {
|
|
7780
|
-
|
|
7781
|
-
|
|
7782
|
-
|
|
8220
|
+
say();
|
|
8221
|
+
say(c.dim(wrap(t.route.noRoute(), 74, ' ')));
|
|
8222
|
+
say();
|
|
7783
8223
|
return;
|
|
7784
8224
|
}
|
|
7785
8225
|
const prompt = capInput(await readFile(promptPath, 'utf8'), promptPath, maxInputFlag(args, t), t);
|
|
@@ -7803,8 +8243,8 @@ async function commandRoute(args, pricing, t) {
|
|
|
7803
8243
|
throw new Error(t.errors.llmNotConfigured());
|
|
7804
8244
|
const label = slice.label === UNLABELLED ? t.profile.unlabelled() : slice.label;
|
|
7805
8245
|
const worth = formatUsd(slice.route.savingUsd);
|
|
7806
|
-
|
|
7807
|
-
|
|
8246
|
+
say();
|
|
8247
|
+
say(` ${c.bold(t.route.picked(label, slice.modelName, slice.route.candidate.displayName, worth, `${(slice.shareOfBill * 100).toFixed(1)}%`))}`);
|
|
7808
8248
|
/**
|
|
7809
8249
|
* The money and the measurement have to describe the same calls.
|
|
7810
8250
|
*
|
|
@@ -7815,16 +8255,16 @@ async function commandRoute(args, pricing, t) {
|
|
|
7815
8255
|
* counts, so it is stated rather than guessed at.
|
|
7816
8256
|
*/
|
|
7817
8257
|
if (slice.label === UNLABELLED) {
|
|
7818
|
-
|
|
8258
|
+
say(` ${c.yellow('!')} ${c.dim(wrap(t.route.unlabelledSlice(), 74, ' '))}`);
|
|
7819
8259
|
}
|
|
7820
|
-
|
|
7821
|
-
|
|
8260
|
+
say();
|
|
8261
|
+
say(` ${c.dim(wrap(t.route.willSpend(inputs.length * 3, provider.model, candidate.model), 74, ' '))}`);
|
|
7822
8262
|
if (!boolFlag(args, 'yes')) {
|
|
7823
|
-
|
|
7824
|
-
|
|
8263
|
+
say(` ${c.dim(t.route.dryRun())}`);
|
|
8264
|
+
say();
|
|
7825
8265
|
return;
|
|
7826
8266
|
}
|
|
7827
|
-
|
|
8267
|
+
say(` ${c.dim(t.route.running(inputs.length))}`);
|
|
7828
8268
|
// Same prompt on both sides. The axis under test is the model, and passing the
|
|
7829
8269
|
// prompt twice is what says so at the call site.
|
|
7830
8270
|
const result = await evaluate(prompt, prompt, inputs, provider, {
|
|
@@ -7832,21 +8272,21 @@ async function commandRoute(args, pricing, t) {
|
|
|
7832
8272
|
concurrency: numberFlag(args, 'concurrency', 3, t),
|
|
7833
8273
|
});
|
|
7834
8274
|
if (boolFlag(args, 'json')) {
|
|
7835
|
-
console.log(JSON.stringify(
|
|
8275
|
+
console.log(JSON.stringify(routeDocument(slice, result), null, 2));
|
|
7836
8276
|
return;
|
|
7837
8277
|
}
|
|
7838
8278
|
const asPct = (v) => `${(v * 100).toFixed(0)}%`;
|
|
7839
|
-
|
|
7840
|
-
|
|
7841
|
-
|
|
8279
|
+
say();
|
|
8280
|
+
say(` ${c.dim(wrap(t.route.agreement(asPct(result.crossAgreement), asPct(result.selfAgreement)), 74, ' '))}`);
|
|
8281
|
+
say();
|
|
7842
8282
|
if (result.verdict === 'inconclusive') {
|
|
7843
|
-
|
|
8283
|
+
say(` ${c.bold(wrap(t.route.inconclusive(), 74, ' '))}`);
|
|
7844
8284
|
}
|
|
7845
8285
|
else if (result.verdict === 'diverges') {
|
|
7846
|
-
|
|
8286
|
+
say(` ${c.yellow('!')} ${c.bold(wrap(t.route.diverges(worth), 74, ' '))}`);
|
|
7847
8287
|
}
|
|
7848
8288
|
else {
|
|
7849
|
-
|
|
8289
|
+
say(` ${c.green('✓')} ${c.bold(wrap(t.route.holds(worth), 74, ' '))}`);
|
|
7850
8290
|
}
|
|
7851
8291
|
/**
|
|
7852
8292
|
* Printed on every verdict including the good one. Agreement is not
|
|
@@ -7854,8 +8294,8 @@ async function commandRoute(args, pricing, t) {
|
|
|
7854
8294
|
* ever right, and a green tick that let somebody forget that would be the tool
|
|
7855
8295
|
* overstating what it knows.
|
|
7856
8296
|
*/
|
|
7857
|
-
|
|
7858
|
-
|
|
8297
|
+
say(` ${c.dim(wrap(t.route.yours(), 74, ' '))}`);
|
|
8298
|
+
say();
|
|
7859
8299
|
}
|
|
7860
8300
|
/**
|
|
7861
8301
|
* `trazum baseline <dir>` — record what the estate costs now.
|
|
@@ -8010,8 +8450,12 @@ async function scanPrompts(root, args, flagBudget, config, counter, level, t, lo
|
|
|
8010
8450
|
}
|
|
8011
8451
|
const verdicts = [];
|
|
8012
8452
|
const declined = [];
|
|
8453
|
+
// Every prompt this run judged, so the report's band is the widest of them
|
|
8454
|
+
// rather than one figure asserted over text of several kinds.
|
|
8455
|
+
const judgedTexts = [];
|
|
8013
8456
|
for (const relativePath of files) {
|
|
8014
8457
|
const text = capInput(await readFile(join(root, relativePath), 'utf8'), relativePath, maxInputFlag(args, t), t);
|
|
8458
|
+
judgedTexts.push(text);
|
|
8015
8459
|
// Budgets are keyed on paths as written in the repository, so a pattern like
|
|
8016
8460
|
// `prompts/**` has to be matched against the path including the root the
|
|
8017
8461
|
// user passed — not against the name relative to it.
|
|
@@ -8041,7 +8485,7 @@ async function scanPrompts(root, args, flagBudget, config, counter, level, t, lo
|
|
|
8041
8485
|
if (verdicts.length === 0 && declined.length === 0) {
|
|
8042
8486
|
throw new Error(t.errors.noPromptsFound(root, extensions.join(' ')));
|
|
8043
8487
|
}
|
|
8044
|
-
return { verdicts, declined, truncated, extensions };
|
|
8488
|
+
return { verdicts, declined, truncated, extensions, band: widestBand(judgedTexts) };
|
|
8045
8489
|
}
|
|
8046
8490
|
async function checkDirectory(root, args, flagBudget, config, counter, level, t, locale, pricing, only) {
|
|
8047
8491
|
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
@@ -8060,7 +8504,7 @@ async function checkDirectory(root, args, flagBudget, config, counter, level, t,
|
|
|
8060
8504
|
return;
|
|
8061
8505
|
only = checkable;
|
|
8062
8506
|
}
|
|
8063
|
-
const { verdicts, declined, truncated, extensions } = await scanPrompts(root, args, flagBudget, config, counter, level, t, locale, pricing, only);
|
|
8507
|
+
const { verdicts, declined, truncated, extensions, band } = await scanPrompts(root, args, flagBudget, config, counter, level, t, locale, pricing, only);
|
|
8064
8508
|
/**
|
|
8065
8509
|
* The baseline gate, when the config declares one and `--no-baseline` did not
|
|
8066
8510
|
* switch it off for this run.
|
|
@@ -8117,6 +8561,7 @@ async function checkDirectory(root, args, flagBudget, config, counter, level, t,
|
|
|
8117
8561
|
await writeMarkdown(args, () => renderCheckMarkdown({
|
|
8118
8562
|
target: root,
|
|
8119
8563
|
verdicts,
|
|
8564
|
+
band,
|
|
8120
8565
|
level,
|
|
8121
8566
|
tokenSource: counter.source,
|
|
8122
8567
|
truncated,
|
|
@@ -8258,6 +8703,15 @@ function joinPosix(root, relativePath) {
|
|
|
8258
8703
|
* never "delete this", and nothing here edits the prompt.
|
|
8259
8704
|
*/
|
|
8260
8705
|
async function commandPrune(args, t) {
|
|
8706
|
+
// Same as `route`: under `--json` stdout is the document, and the estimate
|
|
8707
|
+
// a reader has to see before spending goes to stderr rather than away.
|
|
8708
|
+
const asJson = boolFlag(args, 'json');
|
|
8709
|
+
const say = (line = '') => {
|
|
8710
|
+
if (asJson)
|
|
8711
|
+
console.error(line);
|
|
8712
|
+
else
|
|
8713
|
+
console.log(line);
|
|
8714
|
+
};
|
|
8261
8715
|
const prompt = await readInput(args.positional[0], t, maxInputFlag(args, t));
|
|
8262
8716
|
const casesPath = stringFlag(args, 'cases');
|
|
8263
8717
|
if (!casesPath)
|
|
@@ -8271,10 +8725,10 @@ async function commandPrune(args, t) {
|
|
|
8271
8725
|
const calls = plannedCalls(examples.length, inputs.length);
|
|
8272
8726
|
// Printed before the key is even looked up, so somebody weighing it up does not
|
|
8273
8727
|
// need a configured provider to see the number.
|
|
8274
|
-
|
|
8275
|
-
|
|
8728
|
+
say();
|
|
8729
|
+
say(c.bold(t.prune.estimate(examples.length, inputs.length, calls)));
|
|
8276
8730
|
if (!boolFlag(args, 'yes')) {
|
|
8277
|
-
|
|
8731
|
+
say(c.yellow(` ${t.prune.needsConsent()}`));
|
|
8278
8732
|
return;
|
|
8279
8733
|
}
|
|
8280
8734
|
const provider = providerFromEnv();
|
|
@@ -8284,14 +8738,14 @@ async function commandPrune(args, t) {
|
|
|
8284
8738
|
concurrency: numberFlag(args, 'concurrency', 3, t),
|
|
8285
8739
|
});
|
|
8286
8740
|
if (boolFlag(args, 'json')) {
|
|
8287
|
-
console.log(JSON.stringify(report, null, 2));
|
|
8741
|
+
console.log(JSON.stringify(pruneDocument(report), null, 2));
|
|
8288
8742
|
return;
|
|
8289
8743
|
}
|
|
8290
8744
|
const pct = (value) => `${(value * 100).toFixed(0)}%`;
|
|
8291
|
-
|
|
8292
|
-
|
|
8293
|
-
|
|
8294
|
-
|
|
8745
|
+
say();
|
|
8746
|
+
say(sectionHeading(t.prune.heading(provider.model)));
|
|
8747
|
+
say(` ${c.dim(t.prune.selfAgreement(pct(report.selfAgreement)))}`);
|
|
8748
|
+
say();
|
|
8295
8749
|
for (const contribution of report.contributions) {
|
|
8296
8750
|
/**
|
|
8297
8751
|
* The mark points at what the reader can act on, which is the *recoverable*
|
|
@@ -8308,7 +8762,7 @@ async function commandPrune(args, t) {
|
|
|
8308
8762
|
: needed
|
|
8309
8763
|
? t.prune.verdictNeeded()
|
|
8310
8764
|
: t.prune.verdictRecoverable();
|
|
8311
|
-
|
|
8765
|
+
say(` ${mark} ${t.prune.line(contribution.index + 1, contribution.tokens, pct(contribution.agreementWithout))}`
|
|
8312
8766
|
+ ` ${unknown ? c.yellow(label) : needed ? c.dim(label) : c.green(label)}`);
|
|
8313
8767
|
/**
|
|
8314
8768
|
* The first line that is not the header, because the header is the same on
|
|
@@ -8318,15 +8772,15 @@ async function commandPrune(args, t) {
|
|
|
8318
8772
|
*/
|
|
8319
8773
|
const lines = contribution.text.split('\n').filter((line) => line.trim() !== '');
|
|
8320
8774
|
const body = lines.find((line) => !/^\s*(?:#+\s*)?(?:example|ejemplo)\b[\s:.-]*$/i.test(line));
|
|
8321
|
-
|
|
8775
|
+
say(` ${c.dim(truncate((body ?? lines[0] ?? '').trim(), 60))}`);
|
|
8322
8776
|
}
|
|
8323
|
-
|
|
8777
|
+
say();
|
|
8324
8778
|
if (report.recoverableTokens > 0) {
|
|
8325
|
-
|
|
8779
|
+
say(` ${t.prune.recoverable(report.recoverableTokens)}`);
|
|
8326
8780
|
}
|
|
8327
|
-
|
|
8328
|
-
|
|
8329
|
-
|
|
8781
|
+
say(` ${c.dim(wrap(t.prune.caveat(), 74, ' '))}`);
|
|
8782
|
+
say();
|
|
8783
|
+
say(c.dim(` ${t.eval.callsMade(report.callsMade)}`));
|
|
8330
8784
|
}
|
|
8331
8785
|
async function commandEval(args, config, t, locale) {
|
|
8332
8786
|
const prompt = await readInput(args.positional[0], t, maxInputFlag(args, t));
|
|
@@ -8576,6 +9030,18 @@ function printComparison(comparison, beforePath, afterPath, optimized, t) {
|
|
|
8576
9030
|
const DOCTOR_LIST_LIMIT = 8;
|
|
8577
9031
|
const BLAME_DEFAULT_LIMIT = 20;
|
|
8578
9032
|
const BLAME_MAX_LIMIT = 500;
|
|
9033
|
+
/**
|
|
9034
|
+
* The band for a report covering several prompts.
|
|
9035
|
+
*
|
|
9036
|
+
* The widest of them, and the empty case is the widest published band rather
|
|
9037
|
+
* than a narrow default: a report over nothing has measured nothing, and
|
|
9038
|
+
* guessing narrow there is the one direction that produces a false claim.
|
|
9039
|
+
*/
|
|
9040
|
+
function widestBand(texts) {
|
|
9041
|
+
if (texts.length === 0)
|
|
9042
|
+
return ESTIMATE_ERROR_BAND_PCT;
|
|
9043
|
+
return Math.max(...texts.map((text) => bandFor(text)));
|
|
9044
|
+
}
|
|
8579
9045
|
/**
|
|
8580
9046
|
* `trazum blame <file>` — what happened to this prompt's cost, and who did it.
|
|
8581
9047
|
*
|
|
@@ -8650,14 +9116,32 @@ async function commandBlame(args, config, pricing, t) {
|
|
|
8650
9116
|
// put the newest first: this is a history, and histories are read backwards.
|
|
8651
9117
|
const shown = rows.slice(revisions.length > limit ? 1 : 0).reverse();
|
|
8652
9118
|
const truncatedHistory = revisions.length > limit;
|
|
9119
|
+
/*
|
|
9120
|
+
The band of this file's own text, read once and handed to both renderers.
|
|
9121
|
+
|
|
9122
|
+
`blame` walks one file across revisions, so the kind of text is a property
|
|
9123
|
+
of the file, and today's content is the closest thing to it that exists
|
|
9124
|
+
here. A file that is gone from the working tree — renamed away, or deleted
|
|
9125
|
+
in the commit being blamed — falls back to the widest published band, which
|
|
9126
|
+
is exactly what that constant is for: a caller that cannot see the text can
|
|
9127
|
+
only guess in the safe direction.
|
|
9128
|
+
*/
|
|
9129
|
+
let band = ESTIMATE_ERROR_BAND_PCT;
|
|
9130
|
+
try {
|
|
9131
|
+
band = bandFor(await readFile(resolvePath(process.cwd(), target), 'utf8'));
|
|
9132
|
+
}
|
|
9133
|
+
catch {
|
|
9134
|
+
// Not on disk now. The wide band stands and says so by being wide.
|
|
9135
|
+
}
|
|
8653
9136
|
await writeMarkdown(args, () => renderBlameMarkdown({
|
|
8654
9137
|
repoPath,
|
|
8655
9138
|
rows: shown,
|
|
8656
9139
|
truncated: truncatedHistory,
|
|
8657
9140
|
netCost: netCostOf(shown, args, config, pricing, t),
|
|
9141
|
+
band,
|
|
8658
9142
|
t,
|
|
8659
9143
|
}));
|
|
8660
|
-
printBlame(shown, { repoPath, args, config, pricing, t, truncated: truncatedHistory });
|
|
9144
|
+
printBlame(shown, { repoPath, args, config, pricing, t, truncated: truncatedHistory, band });
|
|
8661
9145
|
}
|
|
8662
9146
|
/**
|
|
8663
9147
|
* What the movement across a history costs per month, or `null`.
|
|
@@ -8793,7 +9277,7 @@ function printBlame(rows, context) {
|
|
|
8793
9277
|
console.log(`\n${c.bold(t.blame.biggestRise())}`);
|
|
8794
9278
|
console.log(` ${t.blame.biggestRiseDetail(n(worst.delta), worst.revision.author, truncate(worst.revision.subject, 60), worst.revision.shortSha)}`);
|
|
8795
9279
|
}
|
|
8796
|
-
console.log(`\n${c.dim(t.blame.estimateNote())}\n`);
|
|
9280
|
+
console.log(`\n${c.dim(t.blame.estimateNote(context.band))}\n`);
|
|
8797
9281
|
}
|
|
8798
9282
|
/**
|
|
8799
9283
|
* `trazum rank <dir>` — which of these prompts to fix first.
|
|
@@ -9153,6 +9637,12 @@ async function main() {
|
|
|
9153
9637
|
case 'from-otel':
|
|
9154
9638
|
await commandFromOtel(args, t);
|
|
9155
9639
|
break;
|
|
9640
|
+
case 'from-litellm':
|
|
9641
|
+
await commandFromLiteLlm(args, t);
|
|
9642
|
+
break;
|
|
9643
|
+
case 'from-helicone':
|
|
9644
|
+
await commandFromHelicone(args, t);
|
|
9645
|
+
break;
|
|
9156
9646
|
case 'switch':
|
|
9157
9647
|
await commandSwitch(args, pricing, t);
|
|
9158
9648
|
break;
|