@trazum/cli 1.62.0 → 1.63.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/git.d.ts +11 -0
- package/dist/git.d.ts.map +1 -1
- package/dist/git.js +34 -3
- package/dist/git.js.map +1 -1
- package/dist/i18n/en.d.ts.map +1 -1
- package/dist/i18n/en.js +50 -0
- package/dist/i18n/en.js.map +1 -1
- package/dist/i18n/es.d.ts.map +1 -1
- package/dist/i18n/es.js +50 -0
- package/dist/i18n/es.js.map +1 -1
- package/dist/i18n/types.d.ts +23 -0
- package/dist/i18n/types.d.ts.map +1 -1
- package/dist/index.js +417 -22
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
- package/src/git.ts +35 -3
- package/src/i18n/en.ts +60 -0
- package/src/i18n/es.ts +60 -0
- package/src/i18n/types.ts +24 -0
- package/src/index.ts +475 -20
package/dist/index.js
CHANGED
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { readFileSync } from 'node:fs';
|
|
3
|
-
import { open, readdir, readFile, stat, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { readdirSync, readFileSync } from 'node:fs';
|
|
3
|
+
import { mkdir, mkdtemp, open, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
|
|
4
|
+
import { cpus, tmpdir } from 'node:os';
|
|
4
5
|
import { createInterface } from 'node:readline/promises';
|
|
5
6
|
import { dirname, join, resolve as resolvePath } from 'node:path';
|
|
6
7
|
import { fileURLToPath } from 'node:url';
|
|
7
8
|
import { gunzipSync } from 'node:zlib';
|
|
8
|
-
import { CONTRACT_NAMES, applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, bucketedCacheEconomics, bucketedProfile, buildHistory, buildPlan, connectorFor, CONNECTORS, normalizeAnthropicUsage, normalizeOpenAIUsage, bucketsFromRecords, evaluateWatch, firedKey, pruneRecords, recordsFromBuckets, storeInventory, storedReportFrom, verifyPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, bandGoverns, countTokensAnthropic, DEFAULT_USAGE, budgetPositions, conform, BREAK_EVEN_BAND, allocate, annualRecord, replayCommitment, coversTheTerm, runExperiment, qualityGate, semanticPassCost, verifySemanticProposals, SEMANTIC_SYSTEM_PROMPT, ladderPosition, validateLadder, outcomeReport, rankPerOutcome, FAILURE_POLICIES, detectFromSource, matchLocale, parsePlanDocument, waiverDay, waiverHistory, proposeInit, MIN_RATE_DAYS, parseConfig, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, rollUp, heartbeats, ruleYield, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, moneyIsComparable, mostSpecificMatch, assemble, interview, nearestName, slot, SLOT_IDS, optimize, parseBaseline, PHRASE_LANGUAGES, detectTextLanguage, dictionaryStanding, languagesWithStanding, plannedCalls, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
|
|
9
|
+
import { CONTRACT_NAMES, applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, bucketedCacheEconomics, bucketedProfile, buildHistory, buildPlan, connectorFor, CONNECTORS, normalizeAnthropicUsage, normalizeOpenAIUsage, bucketsFromRecords, evaluateWatch, firedKey, pruneRecords, recordsFromBuckets, storeInventory, storedReportFrom, verifyPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, bandGoverns, countTokensAnthropic, DEFAULT_USAGE, budgetPositions, conform, BREAK_EVEN_BAND, allocate, annualRecord, replayCommitment, coversTheTerm, runExperiment, qualityGate, semanticPassCost, verifySemanticProposals, SEMANTIC_SYSTEM_PROMPT, ladderPosition, validateLadder, outcomeReport, rankPerOutcome, FAILURE_POLICIES, detectFromSource, matchLocale, parsePlanDocument, waiverDay, waiverHistory, proposeInit, MIN_RATE_DAYS, parseConfig, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, rollUp, heartbeats, ruleYield, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, MAX_INPUT_CHARS, moneyIsComparable, mostSpecificMatch, assemble, interview, nearestName, slot, SLOT_IDS, optimize, parseBaseline, PHRASE_LANGUAGES, detectTextLanguage, dictionaryStanding, languagesWithStanding, plannedCalls, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
|
|
9
10
|
import { cacheDir, cacheStats, cachingProvider, clearCache } from './suggest-cache.js';
|
|
10
11
|
import { dayOf, formatGap, median, spanDays } from './time.js';
|
|
11
12
|
// Everything that reads the filesystem, on its own entry point so the web
|
|
12
13
|
// bundle cannot reach it. See packages/core/src/node.ts.
|
|
13
14
|
import { CONFIG_FILENAME, DEFAULT_EXTENSIONS, budgetFor, BUNDLED_CATALOGUE, SAFE_FETCH_INIT, applyPricingOverlay, catalogueFromOverlay, checkedEndpoint, openrouterOverlay, detectHost, loadConfig, walkPrompts, } from '@trazum/core/node';
|
|
14
|
-
import { contentAt, gitAvailable, namesByRevision, pathInRepository, repositoryRoot, revisionsFor, } from './git.js';
|
|
15
|
+
import { contentAt, gitAvailable, namesByRevision, pathInRepository, repositoryRoot, revisionsFor, runSelf, } from './git.js';
|
|
15
16
|
import { fetchProviderUsage, findCredential } from './connect.js';
|
|
16
17
|
import { STORE_DIR, appendRecords, readStore, rewriteStore } from './store-fs.js';
|
|
17
18
|
import { WAIVER_LOG, appendWaiverUse, readWaiverLog } from './waiver-log.js';
|
|
@@ -75,6 +76,10 @@ const VALUE_FLAGS = new Set([
|
|
|
75
76
|
'max-cache-loss-usd',
|
|
76
77
|
'max-stale-hours',
|
|
77
78
|
'measure',
|
|
79
|
+
'workload',
|
|
80
|
+
'record',
|
|
81
|
+
'max-ratio',
|
|
82
|
+
'max-input',
|
|
78
83
|
'max-day-usd',
|
|
79
84
|
'max-session-usd',
|
|
80
85
|
'csv-out',
|
|
@@ -321,7 +326,7 @@ function disabledRules(args, config) {
|
|
|
321
326
|
* a threshold is set — `--max-growh 5` would have been ignored and the build
|
|
322
327
|
* gone green. Silence is the wrong answer for a typo.
|
|
323
328
|
*/
|
|
324
|
-
const GLOBAL_FLAGS = ['help', 'h', 'version', 'v', 'locale', 'json', 'config', 'pricing', 'pricing-live'];
|
|
329
|
+
const GLOBAL_FLAGS = ['help', 'h', 'version', 'v', 'locale', 'json', 'config', 'pricing', 'pricing-live', 'max-input'];
|
|
325
330
|
const COMMAND_FLAGS = {
|
|
326
331
|
optimize: [
|
|
327
332
|
'level', 'model', 'calls', 'output-tokens', 'cache-hit-rate', 'batch',
|
|
@@ -349,6 +354,7 @@ const COMMAND_FLAGS = {
|
|
|
349
354
|
conform: ['contract', 'json'],
|
|
350
355
|
rollup: ['json'],
|
|
351
356
|
pulse: ['json', 'max-stale-hours'],
|
|
357
|
+
bench: ['workload', 'json', 'record', 'against', 'max-ratio'],
|
|
352
358
|
write: ['answers', 'json', 'out', 'o', 'calls', 'avg-output'],
|
|
353
359
|
feedback: [],
|
|
354
360
|
gateway: ['on-cannot-tell', 'port', 'socket', 'pricing', 'pricing-live'],
|
|
@@ -1538,7 +1544,8 @@ async function commandConform(args, t) {
|
|
|
1538
1544
|
if (named !== undefined && !isContract(named)) {
|
|
1539
1545
|
throw new Error(t.conform.badContract(named, CONTRACT_NAMES.join(', ')));
|
|
1540
1546
|
}
|
|
1541
|
-
|
|
1547
|
+
// A document or a log, not a prompt: deliberately uncapped (limit null).
|
|
1548
|
+
const text = target === '-' ? await readInput('-', t, null) : await readUsageLog(target, t);
|
|
1542
1549
|
const report = conform(text, named === undefined ? {} : { contract: named });
|
|
1543
1550
|
if (boolFlag(args, 'json')) {
|
|
1544
1551
|
console.log(JSON.stringify(report, null, 2));
|
|
@@ -2006,6 +2013,354 @@ async function commandPulse(args, t) {
|
|
|
2006
2013
|
console.log(` ${c.dim(wrap(t.pulse.notAService(), 74, ' '))}`);
|
|
2007
2014
|
console.log();
|
|
2008
2015
|
}
|
|
2016
|
+
/**
|
|
2017
|
+
* `trazum bench` — measures this machine, honestly.
|
|
2018
|
+
*
|
|
2019
|
+
* The standard workloads, one shot each, wall time and peak RSS. **No
|
|
2020
|
+
* comparison and no judgement**: a number a person runs before and after a
|
|
2021
|
+
* change and reads side by side. The pathological cases were timed once, by
|
|
2022
|
+
* hand, during a stress session — 1MB of prose in about a second, a
|
|
2023
|
+
* 200,000-line log in about 1.3 — and nothing held them there. This command is
|
|
2024
|
+
* that measurement made repeatable; the gate that holds it belongs to a ratio
|
|
2025
|
+
* against in-process calibration, not to these wall clocks.
|
|
2026
|
+
*
|
|
2027
|
+
* Each workload runs in its own child process, because peak RSS is a fact
|
|
2028
|
+
* about a process: five workloads sharing one heap would each report the
|
|
2029
|
+
* high-water mark of whichever ran biggest before them. The child is this same
|
|
2030
|
+
* CLI with `--workload`, so what the bench measures is exactly what a user
|
|
2031
|
+
* runs.
|
|
2032
|
+
*
|
|
2033
|
+
* The workloads are **generated, deterministic and never committed** — the
|
|
2034
|
+
* fuzzer's own LCG, a fixed pricing date — so two runs on one machine differ
|
|
2035
|
+
* by the machine's weather, never by the input. Generation happens outside the
|
|
2036
|
+
* timed window: the bench times the product, not the bench.
|
|
2037
|
+
*
|
|
2038
|
+
* "Peak heap" in the plan is reported here as **peak RSS** —
|
|
2039
|
+
* `process.resourceUsage().maxRSS`, what the operating system actually billed
|
|
2040
|
+
* the process — because a true heap high-water mark is not observable from
|
|
2041
|
+
* inside a synchronous run without instrumentation that would itself move the
|
|
2042
|
+
* number. The field says what it is.
|
|
2043
|
+
*/
|
|
2044
|
+
const BENCH_WORKLOADS = [
|
|
2045
|
+
'optimize-1mb-safe',
|
|
2046
|
+
'optimize-1mb-aggressive',
|
|
2047
|
+
'profile-200k',
|
|
2048
|
+
'walk-10k',
|
|
2049
|
+
'rollup-20k',
|
|
2050
|
+
];
|
|
2051
|
+
/** The hostile-input suite's LCG: same seed, same workload, any machine. */
|
|
2052
|
+
function benchGenerator(seed) {
|
|
2053
|
+
let state = seed;
|
|
2054
|
+
return () => (state = (state * 1103515245 + 12345) % 2147483648) / 2147483648;
|
|
2055
|
+
}
|
|
2056
|
+
/** Prices resolve against a fixed date, so the workload cannot drift with the calendar. */
|
|
2057
|
+
const BENCH_DATE = new Date('2026-01-01T00:00:00Z');
|
|
2058
|
+
/**
|
|
2059
|
+
* Enough integer work for the loop to take a stable fraction of a second on
|
|
2060
|
+
* anything that can run Node, and little enough that calibrating five
|
|
2061
|
+
* workloads is not itself the bench.
|
|
2062
|
+
*/
|
|
2063
|
+
const CALIBRATION_ITERATIONS = 1 << 24;
|
|
2064
|
+
/**
|
|
2065
|
+
* The yardstick a ratio divides by: a fixed integer loop, timed in the same
|
|
2066
|
+
* process as the workload it calibrates.
|
|
2067
|
+
*
|
|
2068
|
+
* **Deliberately not the product's own code.** A calibration built on
|
|
2069
|
+
* `estimateTokens` would speed up when the tokenizer does, and every ratio in
|
|
2070
|
+
* every committed baseline would silently mean something new. This loop is
|
|
2071
|
+
* arithmetic that no release has a reason to touch — the same LCG the corpus
|
|
2072
|
+
* generators use, run hot — so a ratio moves only when the workload does.
|
|
2073
|
+
*
|
|
2074
|
+
* CI machines lie about wall time; they lie to the workload and the yardstick
|
|
2075
|
+
* by roughly the same amount, and the ratio is what is left when the lie
|
|
2076
|
+
* cancels out. That is the whole of chapter two's argument, and the reason a
|
|
2077
|
+
* gate on `ratio` can hold where a gate on `wallMs` would fail on weather.
|
|
2078
|
+
*/
|
|
2079
|
+
function benchCalibration() {
|
|
2080
|
+
const startedAt = performance.now();
|
|
2081
|
+
let state = 1;
|
|
2082
|
+
for (let i = 0; i < CALIBRATION_ITERATIONS; i += 1) {
|
|
2083
|
+
state = (state * 1103515245 + 12345) % 2147483648;
|
|
2084
|
+
}
|
|
2085
|
+
const elapsed = performance.now() - startedAt;
|
|
2086
|
+
// Reading the accumulator keeps the loop observable — a JIT that could prove
|
|
2087
|
+
// the result unused could also skip the work being timed.
|
|
2088
|
+
if (state < 0)
|
|
2089
|
+
throw new Error('unreachable: the LCG stays in [0, 2^31)');
|
|
2090
|
+
return elapsed;
|
|
2091
|
+
}
|
|
2092
|
+
/**
|
|
2093
|
+
* Prose the rules have work in — verbose phrases, duplicate lines, emphasis,
|
|
2094
|
+
* spacing — with code fences and URLs mixed in so the segmenter and the masks
|
|
2095
|
+
* are part of what is timed, not skipped by an input too clean to exercise them.
|
|
2096
|
+
*/
|
|
2097
|
+
function benchPrompt(targetBytes) {
|
|
2098
|
+
const rnd = benchGenerator(97);
|
|
2099
|
+
const sentences = [
|
|
2100
|
+
'Please kindly note that in order to get the best results you should always read the entire document.\n',
|
|
2101
|
+
'It is very very important that the answer is concise and complete.\n',
|
|
2102
|
+
'IMPORTANT: the sections below repeat their own headers.\n',
|
|
2103
|
+
'The quick summary follows the long summary, which follows the summary.\n',
|
|
2104
|
+
'See https://example.com/guide/section?step=3&mode=full for the walkthrough.\n',
|
|
2105
|
+
'```js\nconst total = items.reduce((sum, item) => sum + item.cost, 0);\n```\n',
|
|
2106
|
+
'- keep the tone neutral\n- keep the tone neutral\n- cite every claim\n',
|
|
2107
|
+
'In the event that the input is empty, respond with an empty list.\n',
|
|
2108
|
+
];
|
|
2109
|
+
let text = '';
|
|
2110
|
+
while (text.length < targetBytes) {
|
|
2111
|
+
text += sentences[Math.floor(rnd() * sentences.length)];
|
|
2112
|
+
if (rnd() < 0.15)
|
|
2113
|
+
text += '\n\n';
|
|
2114
|
+
}
|
|
2115
|
+
return text;
|
|
2116
|
+
}
|
|
2117
|
+
/** A usage log in the documented shape: timestamps, labels, sessions, cache fields. */
|
|
2118
|
+
function benchLog(lines) {
|
|
2119
|
+
const rnd = benchGenerator(53);
|
|
2120
|
+
const models = ['claude-opus-5', 'claude-sonnet-5', 'claude-haiku-4-5'];
|
|
2121
|
+
const labels = ['support-rag', 'classify', 'agent'];
|
|
2122
|
+
const startMs = Date.parse('2026-01-01T00:00:00Z');
|
|
2123
|
+
const out = [];
|
|
2124
|
+
for (let i = 0; i < lines; i += 1) {
|
|
2125
|
+
const record = {
|
|
2126
|
+
timestamp: new Date(startMs + i * 7000).toISOString(),
|
|
2127
|
+
model: models[Math.floor(rnd() * models.length)],
|
|
2128
|
+
label: labels[Math.floor(rnd() * labels.length)],
|
|
2129
|
+
session: `conv-${Math.floor(rnd() * 400)}`,
|
|
2130
|
+
stop_reason: rnd() < 0.05 ? 'max_tokens' : 'end_turn',
|
|
2131
|
+
usage: {
|
|
2132
|
+
input_tokens: 200 + Math.floor(rnd() * 4000),
|
|
2133
|
+
output_tokens: 50 + Math.floor(rnd() * 800),
|
|
2134
|
+
cache_read_input_tokens: rnd() < 0.6 ? Math.floor(rnd() * 16000) : 0,
|
|
2135
|
+
cache_creation_input_tokens: rnd() < 0.2 ? Math.floor(rnd() * 4000) : 0,
|
|
2136
|
+
},
|
|
2137
|
+
};
|
|
2138
|
+
out.push(JSON.stringify(record));
|
|
2139
|
+
}
|
|
2140
|
+
return out.join('\n');
|
|
2141
|
+
}
|
|
2142
|
+
/** Runs one workload: generation outside the window, the product inside it. */
|
|
2143
|
+
async function benchRun(id) {
|
|
2144
|
+
const measure = (work) => {
|
|
2145
|
+
const startedAt = performance.now();
|
|
2146
|
+
work();
|
|
2147
|
+
return performance.now() - startedAt;
|
|
2148
|
+
};
|
|
2149
|
+
const done = (wallMs, sizes) => {
|
|
2150
|
+
const calibrationMs = benchCalibration();
|
|
2151
|
+
return {
|
|
2152
|
+
id,
|
|
2153
|
+
wallMs,
|
|
2154
|
+
calibrationMs,
|
|
2155
|
+
ratio: wallMs / calibrationMs,
|
|
2156
|
+
// getrusage reports kilobytes; the field name promises bytes, so convert here.
|
|
2157
|
+
maxRssBytes: process.resourceUsage().maxRSS * 1024,
|
|
2158
|
+
bytes: sizes.bytes ?? null,
|
|
2159
|
+
lines: sizes.lines ?? null,
|
|
2160
|
+
files: sizes.files ?? null,
|
|
2161
|
+
};
|
|
2162
|
+
};
|
|
2163
|
+
switch (id) {
|
|
2164
|
+
case 'optimize-1mb-safe':
|
|
2165
|
+
case 'optimize-1mb-aggressive': {
|
|
2166
|
+
const prompt = benchPrompt(1024 * 1024);
|
|
2167
|
+
const level = id === 'optimize-1mb-safe' ? 'safe' : 'aggressive';
|
|
2168
|
+
return done(measure(() => optimize(prompt, { level })), { bytes: Buffer.byteLength(prompt, 'utf8') });
|
|
2169
|
+
}
|
|
2170
|
+
case 'profile-200k': {
|
|
2171
|
+
const lines = 200_000;
|
|
2172
|
+
const text = benchLog(lines);
|
|
2173
|
+
return done(measure(() => profileUsage(text, { catalogue: BUNDLED_CATALOGUE, on: BENCH_DATE })), { lines });
|
|
2174
|
+
}
|
|
2175
|
+
case 'walk-10k': {
|
|
2176
|
+
/**
|
|
2177
|
+
* The discovery half of `rank` and directory `check`: find every file,
|
|
2178
|
+
* read it, estimate it. Generated under the system temp directory and
|
|
2179
|
+
* removed afterwards — ten thousand files are a workload, not a residue.
|
|
2180
|
+
*/
|
|
2181
|
+
const files = 10_000;
|
|
2182
|
+
const root = await mkdtemp(join(tmpdir(), 'trazum-bench-'));
|
|
2183
|
+
try {
|
|
2184
|
+
const rnd = benchGenerator(11);
|
|
2185
|
+
for (let dir = 0; dir < 100; dir += 1) {
|
|
2186
|
+
const dirPath = join(root, `d${dir}`);
|
|
2187
|
+
await mkdir(dirPath);
|
|
2188
|
+
for (let i = 0; i < files / 100; i += 1) {
|
|
2189
|
+
await writeFile(join(dirPath, `p${i}.txt`), `Summarise the report in ${3 + Math.floor(rnd() * 9)} bullet points.\nKeep every figure.\n`);
|
|
2190
|
+
}
|
|
2191
|
+
}
|
|
2192
|
+
const startedAt = performance.now();
|
|
2193
|
+
let estimated = 0;
|
|
2194
|
+
const walk = (path) => {
|
|
2195
|
+
for (const entry of readdirSync(path, { withFileTypes: true })) {
|
|
2196
|
+
const child = join(path, entry.name);
|
|
2197
|
+
if (entry.isDirectory())
|
|
2198
|
+
walk(child);
|
|
2199
|
+
else
|
|
2200
|
+
estimated += estimateTokens(readFileSync(child, 'utf8'));
|
|
2201
|
+
}
|
|
2202
|
+
};
|
|
2203
|
+
walk(root);
|
|
2204
|
+
const wallMs = performance.now() - startedAt;
|
|
2205
|
+
if (estimated <= 0)
|
|
2206
|
+
throw new Error('bench walk read nothing — the workload is broken');
|
|
2207
|
+
return done(wallMs, { files });
|
|
2208
|
+
}
|
|
2209
|
+
finally {
|
|
2210
|
+
await rm(root, { recursive: true, force: true });
|
|
2211
|
+
}
|
|
2212
|
+
}
|
|
2213
|
+
case 'rollup-20k': {
|
|
2214
|
+
/**
|
|
2215
|
+
* Twenty contributors of a thousand lines each. Their profiles are built
|
|
2216
|
+
* outside the window — profiling is `profile-200k`'s measurement — so
|
|
2217
|
+
* what this times is the roll-up itself: twenty documents parsed,
|
|
2218
|
+
* checked and merged.
|
|
2219
|
+
*/
|
|
2220
|
+
const contributors = 20;
|
|
2221
|
+
const linesEach = 1000;
|
|
2222
|
+
const inputs = [];
|
|
2223
|
+
for (let i = 0; i < contributors; i += 1) {
|
|
2224
|
+
const report = profileUsage(benchLog(linesEach), { catalogue: BUNDLED_CATALOGUE, on: BENCH_DATE });
|
|
2225
|
+
inputs.push({ name: `team-${i}`, text: JSON.stringify(report) });
|
|
2226
|
+
}
|
|
2227
|
+
return done(measure(() => rollUp(inputs)), { lines: contributors * linesEach });
|
|
2228
|
+
}
|
|
2229
|
+
}
|
|
2230
|
+
}
|
|
2231
|
+
function printBenchTable(measurements, t, machine) {
|
|
2232
|
+
const n = (value) => Math.round(value).toLocaleString(t.numberLocale);
|
|
2233
|
+
const mb = (bytes) => `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
|
2234
|
+
const idWidth = Math.max(...measurements.map((m) => m.id.length), t.bench.colWorkload().length);
|
|
2235
|
+
console.log();
|
|
2236
|
+
console.log(c.bold(t.bench.heading()));
|
|
2237
|
+
if (machine !== undefined) {
|
|
2238
|
+
console.log(` ${c.dim(t.bench.machine(machine.node, machine.platform, machine.cpus, machine.cpuModel))}`);
|
|
2239
|
+
}
|
|
2240
|
+
console.log();
|
|
2241
|
+
console.log(` ${c.dim(t.bench.colWorkload().padEnd(idWidth))} ${c.dim(t.bench.colWall().padStart(10))} ${c.dim(t.bench.colRatio().padStart(7))} ${c.dim(t.bench.colPeakRss().padStart(10))}`);
|
|
2242
|
+
for (const m of measurements) {
|
|
2243
|
+
console.log(` ${m.id.padEnd(idWidth)} ${n(m.wallMs).padStart(10)} ${m.ratio.toFixed(2).padStart(7)} ${mb(m.maxRssBytes).padStart(10)}`);
|
|
2244
|
+
}
|
|
2245
|
+
console.log();
|
|
2246
|
+
console.log(` ${c.dim(wrap(t.bench.note(), 74, ' '))}`);
|
|
2247
|
+
console.log();
|
|
2248
|
+
}
|
|
2249
|
+
/** Reads and validates a committed ratio baseline, loudly on anything else. */
|
|
2250
|
+
async function readBenchBaseline(path, t) {
|
|
2251
|
+
let parsed;
|
|
2252
|
+
try {
|
|
2253
|
+
parsed = JSON.parse(await readFile(path, 'utf8'));
|
|
2254
|
+
}
|
|
2255
|
+
catch {
|
|
2256
|
+
throw new Error(t.bench.unreadableBaseline(path));
|
|
2257
|
+
}
|
|
2258
|
+
const candidate = parsed;
|
|
2259
|
+
// A version this Trazum does not know is a loud error naming the fix, never
|
|
2260
|
+
// a best-effort read: the file is committed, so it crosses upgrades, and a
|
|
2261
|
+
// gate on misread numbers is a gate on numbers somebody invented.
|
|
2262
|
+
if (candidate?.schemaVersion !== 1 || !Array.isArray(candidate.workloads)) {
|
|
2263
|
+
throw new Error(t.bench.badBaseline(path, JSON.stringify(parsed?.schemaVersion)));
|
|
2264
|
+
}
|
|
2265
|
+
for (const entry of candidate.workloads) {
|
|
2266
|
+
if (typeof entry?.id !== 'string' || !Number.isFinite(entry?.ratio) || entry.ratio <= 0) {
|
|
2267
|
+
throw new Error(t.bench.badBaseline(path, 'workloads'));
|
|
2268
|
+
}
|
|
2269
|
+
}
|
|
2270
|
+
return candidate;
|
|
2271
|
+
}
|
|
2272
|
+
async function commandBench(args, t) {
|
|
2273
|
+
const asJson = boolFlag(args, 'json');
|
|
2274
|
+
const chosen = stringFlag(args, 'workload');
|
|
2275
|
+
const recordPath = stringFlag(args, 'record');
|
|
2276
|
+
const againstPath = stringFlag(args, 'against');
|
|
2277
|
+
const maxRatioRaw = args.flags.get('max-ratio');
|
|
2278
|
+
if (recordPath !== undefined && againstPath !== undefined) {
|
|
2279
|
+
throw new Error(t.bench.recordAndAgainst());
|
|
2280
|
+
}
|
|
2281
|
+
let maxRatio = null;
|
|
2282
|
+
if (againstPath !== undefined) {
|
|
2283
|
+
// The factor is a policy, so it is stated by the caller rather than
|
|
2284
|
+
// defaulted here — the same rule as pulse's threshold.
|
|
2285
|
+
if (maxRatioRaw === undefined)
|
|
2286
|
+
throw new Error(t.bench.needsMaxRatio());
|
|
2287
|
+
const factor = Number(maxRatioRaw);
|
|
2288
|
+
if (!Number.isFinite(factor) || factor < 1)
|
|
2289
|
+
throw new Error(t.bench.badMaxRatio(String(maxRatioRaw)));
|
|
2290
|
+
maxRatio = factor;
|
|
2291
|
+
}
|
|
2292
|
+
else if (maxRatioRaw !== undefined) {
|
|
2293
|
+
throw new Error(t.bench.maxRatioNeedsAgainst());
|
|
2294
|
+
}
|
|
2295
|
+
// Read before measuring, so a baseline this run cannot gate on refuses in
|
|
2296
|
+
// milliseconds instead of after the workloads have been paid for.
|
|
2297
|
+
const baseline = againstPath !== undefined ? await readBenchBaseline(againstPath, t) : null;
|
|
2298
|
+
let measurements;
|
|
2299
|
+
let machine;
|
|
2300
|
+
if (chosen !== undefined) {
|
|
2301
|
+
if (!BENCH_WORKLOADS.includes(chosen)) {
|
|
2302
|
+
throw new Error(t.bench.unknownWorkload(chosen, BENCH_WORKLOADS.join(', ')));
|
|
2303
|
+
}
|
|
2304
|
+
measurements = [await benchRun(chosen)];
|
|
2305
|
+
}
|
|
2306
|
+
else {
|
|
2307
|
+
const script = fileURLToPath(import.meta.url);
|
|
2308
|
+
measurements = [];
|
|
2309
|
+
for (const id of BENCH_WORKLOADS) {
|
|
2310
|
+
const stdout = runSelf(script, ['bench', '--workload', id, '--json', '--locale', t.locale]);
|
|
2311
|
+
measurements.push(JSON.parse(stdout));
|
|
2312
|
+
}
|
|
2313
|
+
machine = {
|
|
2314
|
+
schemaVersion: 1,
|
|
2315
|
+
node: process.version,
|
|
2316
|
+
platform: process.platform,
|
|
2317
|
+
arch: process.arch,
|
|
2318
|
+
cpus: cpus().length,
|
|
2319
|
+
cpuModel: cpus()[0]?.model ?? null,
|
|
2320
|
+
workloads: measurements,
|
|
2321
|
+
};
|
|
2322
|
+
}
|
|
2323
|
+
if (recordPath !== undefined) {
|
|
2324
|
+
const baseline = {
|
|
2325
|
+
schemaVersion: 1,
|
|
2326
|
+
workloads: measurements.map((m) => ({ id: m.id, ratio: m.ratio })),
|
|
2327
|
+
};
|
|
2328
|
+
await writeFile(recordPath, `${JSON.stringify(baseline, null, 2)}\n`);
|
|
2329
|
+
}
|
|
2330
|
+
if (asJson) {
|
|
2331
|
+
// The JSON shape never changes with the gate flags: a gate verdict is the
|
|
2332
|
+
// exit code and the sentences on stderr, the way `check` has always gated.
|
|
2333
|
+
console.log(JSON.stringify(chosen !== undefined ? measurements[0] : machine, null, 2));
|
|
2334
|
+
}
|
|
2335
|
+
else {
|
|
2336
|
+
printBenchTable(measurements, t, machine);
|
|
2337
|
+
}
|
|
2338
|
+
if (recordPath !== undefined) {
|
|
2339
|
+
console.error(t.bench.recorded(recordPath));
|
|
2340
|
+
}
|
|
2341
|
+
if (againstPath !== undefined && baseline !== null && maxRatio !== null) {
|
|
2342
|
+
const byId = new Map(baseline.workloads.map((entry) => [entry.id, entry.ratio]));
|
|
2343
|
+
let over = false;
|
|
2344
|
+
for (const m of measurements) {
|
|
2345
|
+
const recorded = byId.get(m.id);
|
|
2346
|
+
// Measured but never recorded is not a pass: a gate that silently skips
|
|
2347
|
+
// a workload reads as green coverage it does not have.
|
|
2348
|
+
if (recorded === undefined)
|
|
2349
|
+
throw new Error(t.bench.notInBaseline(m.id, againstPath));
|
|
2350
|
+
const allowed = recorded * maxRatio;
|
|
2351
|
+
if (m.ratio > allowed) {
|
|
2352
|
+
over = true;
|
|
2353
|
+
console.error(c.red(t.bench.gateOver(m.id, m.ratio.toFixed(2), allowed.toFixed(2))));
|
|
2354
|
+
}
|
|
2355
|
+
}
|
|
2356
|
+
if (over) {
|
|
2357
|
+
process.exitCode = 1;
|
|
2358
|
+
}
|
|
2359
|
+
else {
|
|
2360
|
+
console.error(c.dim(t.bench.gateWithin(String(maxRatio))));
|
|
2361
|
+
}
|
|
2362
|
+
}
|
|
2363
|
+
}
|
|
2009
2364
|
/**
|
|
2010
2365
|
* `trazum feedback` — where to say it, and what to say.
|
|
2011
2366
|
*
|
|
@@ -2506,7 +2861,7 @@ async function commandQuality(args, config, pricing, t) {
|
|
|
2506
2861
|
* the first thing audited by its own arithmetic, and it has to ask.
|
|
2507
2862
|
*/
|
|
2508
2863
|
async function commandSemantic(args, config, pricing, t) {
|
|
2509
|
-
const prompt = await readInput(args.positional[0], t);
|
|
2864
|
+
const prompt = await readInput(args.positional[0], t, maxInputFlag(args, t));
|
|
2510
2865
|
const modelId = stringFlag(args, 'model') ?? config.usage?.model ?? DEFAULT_USAGE.model;
|
|
2511
2866
|
const model = pricing.byId.get(modelId) ?? getModel(DEFAULT_USAGE.model);
|
|
2512
2867
|
const rates = { inputPerMTok: model.inputPerMTok, outputPerMTok: model.outputPerMTok };
|
|
@@ -3020,7 +3375,7 @@ async function commandRulesMeasure(args, config, pricing, t) {
|
|
|
3020
3375
|
// `walkPrompts` returns names relative to the root it was given, so the
|
|
3021
3376
|
// root has to go back on. Reading them bare only worked from a run whose
|
|
3022
3377
|
// cwd happened to be the root, which is the one case a first probe uses.
|
|
3023
|
-
const raw = await readFile(join(root, file), 'utf8');
|
|
3378
|
+
const raw = capInput(await readFile(join(root, file), 'utf8'), file, maxInputFlag(args, t), t);
|
|
3024
3379
|
// A marked source file contributes its marked prompts; anything else
|
|
3025
3380
|
// contributes itself. The same rule `check` and `doctor` already follow,
|
|
3026
3381
|
// so the three commands measure the same text.
|
|
@@ -3067,7 +3422,41 @@ async function commandRulesMeasure(args, config, pricing, t) {
|
|
|
3067
3422
|
console.log(` ${c.dim(wrap(t.rules.measureBand(report.tokenSource), 74, ' '))}`);
|
|
3068
3423
|
console.log();
|
|
3069
3424
|
}
|
|
3070
|
-
|
|
3425
|
+
/**
|
|
3426
|
+
* The refusal ceiling for a prompt door: the flag, or the number written once
|
|
3427
|
+
* in `@trazum/core`.
|
|
3428
|
+
*
|
|
3429
|
+
* Raising it is deliberate — a flag typed by the person paying the wall time —
|
|
3430
|
+
* and nothing raises it by accident. Lowering it is equally legitimate: a CI
|
|
3431
|
+
* job that knows its prompts are small can make anything bigger a red build.
|
|
3432
|
+
*/
|
|
3433
|
+
function maxInputFlag(args, t) {
|
|
3434
|
+
const raw = args.flags.get('max-input');
|
|
3435
|
+
if (raw === undefined || typeof raw === 'boolean')
|
|
3436
|
+
return MAX_INPUT_CHARS;
|
|
3437
|
+
const value = Number(raw);
|
|
3438
|
+
if (!Number.isFinite(value) || value <= 0) {
|
|
3439
|
+
throw new Error(t.errors.badMaxInput(String(raw)));
|
|
3440
|
+
}
|
|
3441
|
+
return Math.floor(value);
|
|
3442
|
+
}
|
|
3443
|
+
/** Refuses a prompt past the ceiling — the size and the limit named, never a grind. */
|
|
3444
|
+
function capInput(text, source, limit, t) {
|
|
3445
|
+
if (text.length > limit) {
|
|
3446
|
+
throw new Error(t.errors.inputTooLarge(source, text.length, limit));
|
|
3447
|
+
}
|
|
3448
|
+
return text;
|
|
3449
|
+
}
|
|
3450
|
+
/**
|
|
3451
|
+
* Reads a prompt-sized input, held to the ceiling.
|
|
3452
|
+
*
|
|
3453
|
+
* `limit: null` is for the callers whose input is a *document or a log* — a
|
|
3454
|
+
* 200,000-line export is ordinary and its size is the product's subject, so
|
|
3455
|
+
* holding it to a prompt's ceiling would refuse the exact input `profile`
|
|
3456
|
+
* exists to read. The null is written at the call site, so a reader of the
|
|
3457
|
+
* caller sees the decision.
|
|
3458
|
+
*/
|
|
3459
|
+
async function readInput(source, t, limit = null) {
|
|
3071
3460
|
if (!source) {
|
|
3072
3461
|
throw new Error(t.errors.missingInputFile());
|
|
3073
3462
|
}
|
|
@@ -3075,9 +3464,11 @@ async function readInput(source, t) {
|
|
|
3075
3464
|
const chunks = [];
|
|
3076
3465
|
for await (const chunk of process.stdin)
|
|
3077
3466
|
chunks.push(Buffer.from(chunk));
|
|
3078
|
-
|
|
3467
|
+
const text = Buffer.concat(chunks).toString('utf8');
|
|
3468
|
+
return limit === null ? text : capInput(text, 'stdin', limit, t);
|
|
3079
3469
|
}
|
|
3080
|
-
|
|
3470
|
+
const text = await readFile(source, 'utf8');
|
|
3471
|
+
return limit === null ? text : capInput(text, source, limit, t);
|
|
3081
3472
|
}
|
|
3082
3473
|
async function commandOptimize(args, config, pricing, t, locale) {
|
|
3083
3474
|
/**
|
|
@@ -3116,6 +3507,7 @@ async function commandOptimize(args, config, pricing, t, locale) {
|
|
|
3116
3507
|
unreadable.push({ label, path: promptPath });
|
|
3117
3508
|
continue;
|
|
3118
3509
|
}
|
|
3510
|
+
text = capInput(text, promptPath, maxInputFlag(args, t), t);
|
|
3119
3511
|
const r = optimize(text, { level, usage: m.profile, locale, pricing });
|
|
3120
3512
|
rows.push({
|
|
3121
3513
|
label,
|
|
@@ -3155,7 +3547,7 @@ async function commandOptimize(args, config, pricing, t, locale) {
|
|
|
3155
3547
|
return;
|
|
3156
3548
|
}
|
|
3157
3549
|
const target = args.positional[0];
|
|
3158
|
-
const raw = await readInput(target, t);
|
|
3550
|
+
const raw = await readInput(target, t, maxInputFlag(args, t));
|
|
3159
3551
|
const level = levelFlag(args, config, t);
|
|
3160
3552
|
// A source file is not a prompt.
|
|
3161
3553
|
//
|
|
@@ -3478,7 +3870,7 @@ async function commandCheck(args, config, pricing, t, locale) {
|
|
|
3478
3870
|
await checkDirectory(target, args, flagBudget, config, counter, level, t, locale, pricing);
|
|
3479
3871
|
return;
|
|
3480
3872
|
}
|
|
3481
|
-
const prompt = await readInput(target, t);
|
|
3873
|
+
const prompt = await readInput(target, t, maxInputFlag(args, t));
|
|
3482
3874
|
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
3483
3875
|
// A single file falls back to the config budget for its own path, so
|
|
3484
3876
|
// `trazum check prompts/system.txt` works with no flag once budgets exist.
|
|
@@ -6849,7 +7241,7 @@ async function commandRoute(args, pricing, t) {
|
|
|
6849
7241
|
console.log();
|
|
6850
7242
|
return;
|
|
6851
7243
|
}
|
|
6852
|
-
const prompt = await readFile(promptPath, 'utf8');
|
|
7244
|
+
const prompt = capInput(await readFile(promptPath, 'utf8'), promptPath, maxInputFlag(args, t), t);
|
|
6853
7245
|
const inputs = parseCases(await readFile(casesPath, 'utf8'));
|
|
6854
7246
|
if (inputs.length === 0)
|
|
6855
7247
|
throw new Error(t.errors.evalNoCases(casesPath));
|
|
@@ -7048,7 +7440,7 @@ async function scanPrompts(root, args, flagBudget, config, counter, level, t, lo
|
|
|
7048
7440
|
const verdicts = [];
|
|
7049
7441
|
const declined = [];
|
|
7050
7442
|
for (const relativePath of files) {
|
|
7051
|
-
const text = await readFile(join(root, relativePath), 'utf8');
|
|
7443
|
+
const text = capInput(await readFile(join(root, relativePath), 'utf8'), relativePath, maxInputFlag(args, t), t);
|
|
7052
7444
|
// Budgets are keyed on paths as written in the repository, so a pattern like
|
|
7053
7445
|
// `prompts/**` has to be matched against the path including the root the
|
|
7054
7446
|
// user passed — not against the name relative to it.
|
|
@@ -7271,7 +7663,7 @@ function joinPosix(root, relativePath) {
|
|
|
7271
7663
|
* never "delete this", and nothing here edits the prompt.
|
|
7272
7664
|
*/
|
|
7273
7665
|
async function commandPrune(args, t) {
|
|
7274
|
-
const prompt = await readInput(args.positional[0], t);
|
|
7666
|
+
const prompt = await readInput(args.positional[0], t, maxInputFlag(args, t));
|
|
7275
7667
|
const casesPath = stringFlag(args, 'cases');
|
|
7276
7668
|
if (!casesPath)
|
|
7277
7669
|
throw new Error(t.errors.evalNeedsCases());
|
|
@@ -7342,7 +7734,7 @@ async function commandPrune(args, t) {
|
|
|
7342
7734
|
console.log(c.dim(` ${t.eval.callsMade(report.callsMade)}`));
|
|
7343
7735
|
}
|
|
7344
7736
|
async function commandEval(args, config, t, locale) {
|
|
7345
|
-
const prompt = await readInput(args.positional[0], t);
|
|
7737
|
+
const prompt = await readInput(args.positional[0], t, maxInputFlag(args, t));
|
|
7346
7738
|
const level = levelFlag(args, config, t);
|
|
7347
7739
|
const casesPath = stringFlag(args, 'cases');
|
|
7348
7740
|
if (!casesPath)
|
|
@@ -7492,8 +7884,8 @@ async function commandDiff(args, config, pricing, t, locale) {
|
|
|
7492
7884
|
return;
|
|
7493
7885
|
}
|
|
7494
7886
|
const [before, after] = await Promise.all([
|
|
7495
|
-
readInput(beforePath, t),
|
|
7496
|
-
readInput(afterPath, t),
|
|
7887
|
+
readInput(beforePath, t, maxInputFlag(args, t)),
|
|
7888
|
+
readInput(afterPath, t, maxInputFlag(args, t)),
|
|
7497
7889
|
]);
|
|
7498
7890
|
const comparison = comparePrompts(before, after, {
|
|
7499
7891
|
level: levelFlag(args, config, t),
|
|
@@ -7835,7 +8227,7 @@ async function commandRank(args, config, pricing, t, locale) {
|
|
|
7835
8227
|
const ranked = [];
|
|
7836
8228
|
let skipped = 0;
|
|
7837
8229
|
for (const file of files) {
|
|
7838
|
-
const raw = await readFile(join(root, file), 'utf8');
|
|
8230
|
+
const raw = capInput(await readFile(join(root, file), 'utf8'), file, maxInputFlag(args, t), t);
|
|
7839
8231
|
// A source file contributes its marked prompt, or nothing. Ranking
|
|
7840
8232
|
// `src/prompts.ts` by the size of its imports would put the wrong file at
|
|
7841
8233
|
// the top of a list whose whole job is to point somewhere.
|
|
@@ -8157,6 +8549,9 @@ async function main() {
|
|
|
8157
8549
|
case 'pulse':
|
|
8158
8550
|
await commandPulse(args, t);
|
|
8159
8551
|
break;
|
|
8552
|
+
case 'bench':
|
|
8553
|
+
await commandBench(args, t);
|
|
8554
|
+
break;
|
|
8160
8555
|
case 'write':
|
|
8161
8556
|
await commandWrite(args, t);
|
|
8162
8557
|
break;
|
|
@@ -8228,7 +8623,7 @@ async function commandDoctor(args, config, pricing, t, locale) {
|
|
|
8228
8623
|
const seen = [];
|
|
8229
8624
|
let skipped = 0;
|
|
8230
8625
|
for (const file of files) {
|
|
8231
|
-
const raw = await readFile(join(root, file), 'utf8');
|
|
8626
|
+
const raw = capInput(await readFile(join(root, file), 'utf8'), file, maxInputFlag(args, t), t);
|
|
8232
8627
|
// Same contract as `rank`: a source file contributes its marked prompt or
|
|
8233
8628
|
// nothing, and an unmarked one is counted rather than allowed to abort a
|
|
8234
8629
|
// survey of the other thirty-nine.
|
|
@@ -8455,7 +8850,7 @@ async function diffDirectories(beforeRoot, afterRoot, args, config, pricing, t,
|
|
|
8455
8850
|
const afterFiles = new Set(afterWalk.files);
|
|
8456
8851
|
/** The prompt at a path, or null when the file holds no marked prompt. */
|
|
8457
8852
|
const textAt = async (root, file) => {
|
|
8458
|
-
const raw = await readFile(join(root, file), 'utf8');
|
|
8853
|
+
const raw = capInput(await readFile(join(root, file), 'utf8'), file, maxInputFlag(args, t), t);
|
|
8459
8854
|
let source;
|
|
8460
8855
|
try {
|
|
8461
8856
|
source = sourceFileOf(file, raw, pricing, stringFlag(args, 'prompt'));
|