@trazum/core 1.80.1 → 1.82.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -18
- package/dist/advisories.d.ts +33 -1
- package/dist/advisories.d.ts.map +1 -1
- package/dist/advisories.js +103 -16
- package/dist/advisories.js.map +1 -1
- package/dist/band.d.ts +131 -0
- package/dist/band.d.ts.map +1 -0
- package/dist/band.js +210 -0
- package/dist/band.js.map +1 -0
- package/dist/claude-code.d.ts +28 -0
- package/dist/claude-code.d.ts.map +1 -1
- package/dist/claude-code.js +25 -1
- package/dist/claude-code.js.map +1 -1
- package/dist/conform.d.ts +1 -1
- package/dist/conform.d.ts.map +1 -1
- package/dist/conform.js +84 -1
- package/dist/conform.js.map +1 -1
- package/dist/helicone.d.ts +92 -0
- package/dist/helicone.d.ts.map +1 -0
- package/dist/helicone.js +194 -0
- package/dist/helicone.js.map +1 -0
- package/dist/i18n/en.d.ts.map +1 -1
- package/dist/i18n/en.js +26 -20
- package/dist/i18n/en.js.map +1 -1
- package/dist/i18n/es.d.ts.map +1 -1
- package/dist/i18n/es.js +9 -3
- package/dist/i18n/es.js.map +1 -1
- package/dist/i18n/types.d.ts +43 -1
- package/dist/i18n/types.d.ts.map +1 -1
- package/dist/index.d.ts +10 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -1
- package/dist/index.js.map +1 -1
- package/dist/litellm.d.ts +98 -0
- package/dist/litellm.d.ts.map +1 -0
- package/dist/litellm.js +207 -0
- package/dist/litellm.js.map +1 -0
- package/dist/measured-profile.d.ts +4 -0
- package/dist/measured-profile.d.ts.map +1 -1
- package/dist/measured-profile.js +22 -1
- package/dist/measured-profile.js.map +1 -1
- package/dist/schemas.d.ts.map +1 -1
- package/dist/schemas.js +36 -0
- package/dist/schemas.js.map +1 -1
- package/dist/tokenizer.d.ts +45 -49
- package/dist/tokenizer.d.ts.map +1 -1
- package/dist/tokenizer.js +100 -51
- package/dist/tokenizer.js.map +1 -1
- package/dist/types.d.ts +1 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/verdict-bridge.d.ts +113 -0
- package/dist/verdict-bridge.d.ts.map +1 -0
- package/dist/verdict-bridge.js +127 -0
- package/dist/verdict-bridge.js.map +1 -0
- package/package.json +1 -1
- package/src/advisories.ts +132 -15
- package/src/band.ts +213 -0
- package/src/claude-code.ts +53 -1
- package/src/conform.ts +100 -1
- package/src/helicone.ts +227 -0
- package/src/i18n/en.ts +27 -20
- package/src/i18n/es.ts +10 -3
- package/src/i18n/types.ts +45 -1
- package/src/index.ts +17 -1
- package/src/litellm.ts +243 -0
- package/src/measured-profile.ts +26 -2
- package/src/schemas.ts +36 -0
- package/src/tokenizer.ts +101 -51
- package/src/types.ts +1 -0
- package/src/verdict-bridge.ts +200 -0
package/src/conform.ts
CHANGED
|
@@ -54,6 +54,11 @@ export const CONTRACT_NAMES = [
|
|
|
54
54
|
'bench',
|
|
55
55
|
'prompt-draft',
|
|
56
56
|
'position',
|
|
57
|
+
'routing-measurement',
|
|
58
|
+
'example-pruning',
|
|
59
|
+
'store-inventory',
|
|
60
|
+
'conformance',
|
|
61
|
+
'watch-cycle',
|
|
57
62
|
] as const;
|
|
58
63
|
|
|
59
64
|
export type ContractName = (typeof CONTRACT_NAMES)[number];
|
|
@@ -394,6 +399,62 @@ const DOCUMENT_RULES: Record<Exclude<ContractName, 'usage-log'>, FieldRule[]> =
|
|
|
394
399
|
rule('cannotSay', 'an array of sentences — what is deliberately not answered', isArray),
|
|
395
400
|
rule('unpricedRecords', 'a count of records the catalogue cannot price', isNumber),
|
|
396
401
|
],
|
|
402
|
+
'routing-measurement': [
|
|
403
|
+
rule('slice', 'the workload object — the calls this measured', isObject),
|
|
404
|
+
rule('evaluation', 'the measurement object — verdict and agreement rates', isObject),
|
|
405
|
+
],
|
|
406
|
+
'store-inventory': [
|
|
407
|
+
rule('providers', 'an array — one entry per provider held', isArray),
|
|
408
|
+
rule('totalRecords', 'records the store resolves to', isNumber),
|
|
409
|
+
rule(
|
|
410
|
+
'span',
|
|
411
|
+
'an object, or null when the store holds nothing',
|
|
412
|
+
(v) => v === null || isObject(v),
|
|
413
|
+
),
|
|
414
|
+
rule('possiblyDouble', 'records the store could not tell apart', isNumber),
|
|
415
|
+
rule('unknownVersion', 'records written by a newer schema', isNumber),
|
|
416
|
+
rule('totalUsd', 'what the held records priced to', isNumber),
|
|
417
|
+
rule('unreadable', 'an array — files the store could not parse', isArray),
|
|
418
|
+
],
|
|
419
|
+
conformance: [
|
|
420
|
+
rule(
|
|
421
|
+
'contract',
|
|
422
|
+
'the contract the input looked like, or null when nothing did',
|
|
423
|
+
(v) => v === null || typeof v === 'string',
|
|
424
|
+
),
|
|
425
|
+
rule(
|
|
426
|
+
'because',
|
|
427
|
+
'why nothing matched, or null when something did — never a bare refusal',
|
|
428
|
+
(v) => v === null || typeof v === 'string',
|
|
429
|
+
),
|
|
430
|
+
rule(
|
|
431
|
+
'records',
|
|
432
|
+
'records examined for a log, or null for a single document',
|
|
433
|
+
(v) => v === null || isNumber(v),
|
|
434
|
+
),
|
|
435
|
+
rule('problems', 'an array — one entry per fault found', isArray),
|
|
436
|
+
rule('unavailable', 'an array — findings this shape cannot support', isArray),
|
|
437
|
+
rule('conforms', 'a boolean — the verdict', (v) => typeof v === 'boolean'),
|
|
438
|
+
],
|
|
439
|
+
'watch-cycle': [
|
|
440
|
+
rule('firedAtMs', 'the instant the cycle ran', isNumber),
|
|
441
|
+
rule('crossings', 'an array — gates over their limit, news this cycle', isArray),
|
|
442
|
+
rule('suppressed', 'an array — still over, already reported', isArray),
|
|
443
|
+
rule('abstentions', 'an array — gates that could not be judged', isArray),
|
|
444
|
+
rule(
|
|
445
|
+
'gap',
|
|
446
|
+
'an object, or null when this cycle watched the whole stretch',
|
|
447
|
+
(v) => v === null || isObject(v),
|
|
448
|
+
),
|
|
449
|
+
],
|
|
450
|
+
'example-pruning': [
|
|
451
|
+
rule('provider', 'the provider the calls went to', (v) => typeof v === 'string'),
|
|
452
|
+
rule('model', 'the model that answered', (v) => typeof v === 'string'),
|
|
453
|
+
rule('selfAgreement', 'the model’s agreement with itself, the yardstick every removal is judged against', isNumber),
|
|
454
|
+
rule('recoverableTokens', 'tokens held by examples whose removal changed nothing measurable', isNumber),
|
|
455
|
+
rule('callsMade', 'calls actually spent', isNumber),
|
|
456
|
+
rule('contributions', 'an array — one entry per example', isArray),
|
|
457
|
+
],
|
|
397
458
|
pulse: [
|
|
398
459
|
rule('beats', 'an array — one per kind', isArray),
|
|
399
460
|
rule('nowMs', 'the instant the ages were taken against', isNumber),
|
|
@@ -554,6 +615,44 @@ function contractOf(doc: Record<string, unknown>): Exclude<ContractName, 'usage-
|
|
|
554
615
|
if (Array.isArray(doc.actions) && typeof doc.arrived === 'number') return 'verification';
|
|
555
616
|
if (Array.isArray(doc.actions)) return 'plan';
|
|
556
617
|
if (typeof doc.verdict === 'string' && 'restsOn' in doc) return 'cost-answer';
|
|
618
|
+
/*
|
|
619
|
+
Two the inference could never reach, found by the guard that demands every
|
|
620
|
+
contract be recognisable from a document of its own shape.
|
|
621
|
+
|
|
622
|
+
The cost answer's branch above tested `restsOn`, which its contract does
|
|
623
|
+
not require: the required set is verdict, call and budget, so a conforming
|
|
624
|
+
answer that omitted the optional field was recognised as nothing at all.
|
|
625
|
+
The prompt draft had no branch of any kind — its schema has been published
|
|
626
|
+
since 1.68 and `trazum conform draft.json` has never known what it was.
|
|
627
|
+
*/
|
|
628
|
+
if (typeof doc.verdict === 'string' && 'call' in doc && 'budget' in doc) return 'cost-answer';
|
|
629
|
+
if (typeof doc.prompt === 'string' && Array.isArray(doc.sections) && Array.isArray(doc.missing)) {
|
|
630
|
+
return 'prompt-draft';
|
|
631
|
+
}
|
|
632
|
+
/*
|
|
633
|
+
The five the JSON sweep contracted, and the reason they are here.
|
|
634
|
+
|
|
635
|
+
All five arrived with rules, a schema, a row in the interchange table and
|
|
636
|
+
a parity test, and none of them arrived with a branch in this function —
|
|
637
|
+
so `trazum conform verdict.json` read a real routing measurement as a
|
|
638
|
+
usage log and answered with advice about `stop_reason`. A contract that
|
|
639
|
+
cannot be recognised is a contract only reachable by somebody who already
|
|
640
|
+
knows its name, which is the opposite of what this page is for. Found by
|
|
641
|
+
running the command against its own output; `contract-coverage.test.js`
|
|
642
|
+
now requires every name to be inferable from a document.
|
|
643
|
+
|
|
644
|
+
Conformance sits before the connected report deliberately: both carry an
|
|
645
|
+
`unavailable` array, and only the conformance verdict carries `conforms`.
|
|
646
|
+
*/
|
|
647
|
+
if (Array.isArray(doc.problems) && Array.isArray(doc.unavailable) && typeof doc.conforms === 'boolean') {
|
|
648
|
+
return 'conformance';
|
|
649
|
+
}
|
|
650
|
+
if (isObject(doc.evaluation) && 'slice' in doc) return 'routing-measurement';
|
|
651
|
+
if (Array.isArray(doc.contributions) && typeof doc.recoverableTokens === 'number') {
|
|
652
|
+
return 'example-pruning';
|
|
653
|
+
}
|
|
654
|
+
if (Array.isArray(doc.providers) && typeof doc.totalRecords === 'number') return 'store-inventory';
|
|
655
|
+
if (Array.isArray(doc.crossings) && Array.isArray(doc.abstentions)) return 'watch-cycle';
|
|
557
656
|
if (typeof doc.provider === 'string' && Array.isArray(doc.unavailable)) return 'connected';
|
|
558
657
|
return null;
|
|
559
658
|
}
|
|
@@ -650,7 +749,7 @@ export function conform(text: string, options: ConformOptions = {}): Conformance
|
|
|
650
749
|
schemaVersion: 1,
|
|
651
750
|
contract: null,
|
|
652
751
|
because:
|
|
653
|
-
'no contract recognised it — a roll-up has contributors and notMerged, a profile has byLabelAndModel, a plan has actions, a verification adds arrived, a history has periods and runs, a connected report has provider and unavailable, a cost answer has verdict and
|
|
752
|
+
'no contract recognised it — a roll-up has contributors and notMerged, a profile has byLabelAndModel, a plan has actions, a verification adds arrived, a history has periods and runs, a connected report has provider and unavailable, a cost answer has verdict with call and budget, a prompt draft has prompt, sections and missing, a routing measurement has slice and evaluation, an example pruning has contributions and recoverableTokens, a store inventory has providers and totalRecords, a conformance verdict has problems and conforms, a watch cycle has crossings and abstentions',
|
|
654
753
|
problems: [],
|
|
655
754
|
unavailable: [],
|
|
656
755
|
records: null,
|
package/src/helicone.ts
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Helicone's request export, read as a usage log.
|
|
3
|
+
*
|
|
4
|
+
* The third converter in the pattern `from-claude-code` started: a pure
|
|
5
|
+
* function turns one tool's export into Trazum's usage-log records, and every
|
|
6
|
+
* door prices it from there. Helicone sits as a proxy in front of the provider
|
|
7
|
+
* and keeps every request it saw, so a team using it already has the export.
|
|
8
|
+
*
|
|
9
|
+
* **The format is derived, not guessed.** The columns below are the SELECT
|
|
10
|
+
* that builds Helicone's own request table — `web/lib/api/request/request.ts`
|
|
11
|
+
* in Helicone/helicone — and the response shape its `POST /v1/request/query`
|
|
12
|
+
* endpoint documents. A converter written from memory of an API is a converter
|
|
13
|
+
* that silently mis-reads somebody's bill.
|
|
14
|
+
*
|
|
15
|
+
* ## The model, and why it takes three columns to answer
|
|
16
|
+
*
|
|
17
|
+
* Helicone carries `request_model`, `model_override` and `response_model`, and
|
|
18
|
+
* they can disagree: the override exists precisely because a proxy can send a
|
|
19
|
+
* different model than the caller asked for, and the response says what
|
|
20
|
+
* actually answered. **The response wins**, then the override, then the
|
|
21
|
+
* request — the bill is about what was billed, not what was intended — and
|
|
22
|
+
* `modelDisagreements` counts the rows where they differed so a reader can see
|
|
23
|
+
* the substitution happened rather than discover it in a total.
|
|
24
|
+
*
|
|
25
|
+
* ## What deliberately does not cross
|
|
26
|
+
*
|
|
27
|
+
* A Helicone row carries the request body and the response body — the prompt
|
|
28
|
+
* and the completion — and `request_user_id`, which is an email address in
|
|
29
|
+
* Helicone's own documented example. **None of it is read.** The converter
|
|
30
|
+
* names the fields it takes and takes nothing else, and a fixture plants a
|
|
31
|
+
* marker in each and greps the whole output.
|
|
32
|
+
*
|
|
33
|
+
* ## What it refuses to invent
|
|
34
|
+
*
|
|
35
|
+
* There is no cache-token split anywhere on the row: `cache_enabled` is a flag
|
|
36
|
+
* on the analytics table. A converted record therefore carries no cache fields
|
|
37
|
+
* and the caching questions come back `cannot-tell` rather than answered from
|
|
38
|
+
* a guess — the same refusal `from-otel` and `from-litellm` make.
|
|
39
|
+
*
|
|
40
|
+
* There is no conversation identity either. `request_id` is one call, so the
|
|
41
|
+
* records carry no `session` and the conversation-shaped findings stay
|
|
42
|
+
* unavailable. A custom property can carry one, and then it is there because
|
|
43
|
+
* the operator put it there — never because this file inferred it.
|
|
44
|
+
*/
|
|
45
|
+
|
|
46
|
+
/** One converted record, shaped exactly as `parseUsageLine` reads it. */
|
|
47
|
+
export interface HeliconeRecord {
|
|
48
|
+
model: string;
|
|
49
|
+
ts?: string;
|
|
50
|
+
label?: string;
|
|
51
|
+
usage: {
|
|
52
|
+
input_tokens: number;
|
|
53
|
+
output_tokens: number;
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export interface HeliconeConversion {
|
|
58
|
+
records: HeliconeRecord[];
|
|
59
|
+
/** Rows that were requests and converted. */
|
|
60
|
+
rows: number;
|
|
61
|
+
/** Rows naming no model in any of the three columns: counted, never guessed. */
|
|
62
|
+
unnamedModel: number;
|
|
63
|
+
/**
|
|
64
|
+
* Rows where the model that answered was not the model that was asked for.
|
|
65
|
+
*
|
|
66
|
+
* The response's model is what the bill rests on, and a proxy substituting
|
|
67
|
+
* one model for another is a fact worth seeing rather than a difference
|
|
68
|
+
* that only shows up as an unexplained total.
|
|
69
|
+
*/
|
|
70
|
+
modelDisagreements: number;
|
|
71
|
+
/** Rows carrying no token counts at all — a logged request nobody can price. */
|
|
72
|
+
noTokens: number;
|
|
73
|
+
/** Rows Helicone served from its cache, with no token split behind the flag. */
|
|
74
|
+
cacheFlagged: number;
|
|
75
|
+
/** Lines or documents that did not parse as JSON at all. */
|
|
76
|
+
unparseable: number;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const asObject = (value: unknown): Record<string, unknown> | null =>
|
|
80
|
+
typeof value === 'object' && value !== null && !Array.isArray(value)
|
|
81
|
+
? (value as Record<string, unknown>)
|
|
82
|
+
: null;
|
|
83
|
+
|
|
84
|
+
const tryParse = (text: string): unknown => {
|
|
85
|
+
try {
|
|
86
|
+
return JSON.parse(text);
|
|
87
|
+
} catch {
|
|
88
|
+
return undefined;
|
|
89
|
+
}
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
const asCount = (value: unknown): number | undefined => {
|
|
93
|
+
const n = typeof value === 'string' ? Number(value) : value;
|
|
94
|
+
return typeof n === 'number' && Number.isFinite(n) && n >= 0 ? n : undefined;
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
const asName = (value: unknown): string | null =>
|
|
98
|
+
typeof value === 'string' && value.trim() !== '' ? value.trim() : null;
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* The label, from Helicone's custom properties.
|
|
102
|
+
*
|
|
103
|
+
* `request_properties` is the documented place for whatever a team tags its
|
|
104
|
+
* calls with, and the keys are theirs. Trazum looks for the ones a workload
|
|
105
|
+
* would plausibly be named by, in order, and takes the first — never joins
|
|
106
|
+
* several, because a workload has one name in a bill.
|
|
107
|
+
*/
|
|
108
|
+
function labelOf(row: Record<string, unknown>): string | undefined {
|
|
109
|
+
const properties = asObject(row.request_properties) ?? asObject(row.properties);
|
|
110
|
+
if (properties === null) return undefined;
|
|
111
|
+
for (const key of ['Helicone-Property-Label', 'label', 'workload', 'app', 'environment']) {
|
|
112
|
+
const found = asName(properties[key]);
|
|
113
|
+
if (found !== null) return found;
|
|
114
|
+
}
|
|
115
|
+
return undefined;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** The request's creation time as an ISO instant, or undefined. */
|
|
119
|
+
function timeOf(row: Record<string, unknown>): string | undefined {
|
|
120
|
+
const raw = row.request_created_at ?? row.created_at ?? row.response_created_at;
|
|
121
|
+
if (typeof raw !== 'string' && typeof raw !== 'number') return undefined;
|
|
122
|
+
const ms = typeof raw === 'number' ? raw : Date.parse(raw);
|
|
123
|
+
if (!Number.isFinite(ms)) return undefined;
|
|
124
|
+
return new Date(ms).toISOString();
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Convert a Helicone request export.
|
|
129
|
+
*
|
|
130
|
+
* Accepts a JSON array of rows, a single row, `{ data: [...] }` as the query
|
|
131
|
+
* endpoint returns it, or newline-delimited rows. Pure over its input.
|
|
132
|
+
*/
|
|
133
|
+
export function heliconeRecords(text: string): HeliconeConversion {
|
|
134
|
+
const out: HeliconeConversion = {
|
|
135
|
+
records: [],
|
|
136
|
+
rows: 0,
|
|
137
|
+
unnamedModel: 0,
|
|
138
|
+
modelDisagreements: 0,
|
|
139
|
+
noTokens: 0,
|
|
140
|
+
cacheFlagged: 0,
|
|
141
|
+
unparseable: 0,
|
|
142
|
+
};
|
|
143
|
+
|
|
144
|
+
const rows: unknown[] = [];
|
|
145
|
+
const whole = tryParse(text);
|
|
146
|
+
if (whole !== undefined) {
|
|
147
|
+
if (Array.isArray(whole)) rows.push(...whole);
|
|
148
|
+
else {
|
|
149
|
+
const doc = asObject(whole);
|
|
150
|
+
if (doc !== null && Array.isArray(doc.data)) rows.push(...doc.data);
|
|
151
|
+
else if (doc !== null) rows.push(doc);
|
|
152
|
+
}
|
|
153
|
+
} else {
|
|
154
|
+
for (const line of text.split('\n')) {
|
|
155
|
+
if (line.trim() === '') continue;
|
|
156
|
+
const parsed = tryParse(line);
|
|
157
|
+
if (parsed === undefined) out.unparseable += 1;
|
|
158
|
+
else if (Array.isArray(parsed)) rows.push(...parsed);
|
|
159
|
+
else rows.push(parsed);
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
for (const raw of rows) {
|
|
164
|
+
const row = asObject(raw);
|
|
165
|
+
if (row === null) continue;
|
|
166
|
+
const input = asCount(row.prompt_tokens);
|
|
167
|
+
const output = asCount(row.completion_tokens);
|
|
168
|
+
// A request row is one carrying at least one token column. Anything else
|
|
169
|
+
// in the same file is not a call and is skipped rather than recorded as a
|
|
170
|
+
// zero, which would be a call nobody made.
|
|
171
|
+
if (input === undefined && output === undefined) continue;
|
|
172
|
+
|
|
173
|
+
out.rows += 1;
|
|
174
|
+
if (input === 0 && output === 0) out.noTokens += 1;
|
|
175
|
+
if (row.cache_enabled === true || row.cache_enabled === 1 || row.cache_enabled === '1') {
|
|
176
|
+
out.cacheFlagged += 1;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/*
|
|
180
|
+
The response first, and this order is the whole point of counting the
|
|
181
|
+
disagreement. A proxy that substituted a model billed for the model that
|
|
182
|
+
answered, so that is what a bill has to price; taking the requested one
|
|
183
|
+
would report the intention and call it the cost.
|
|
184
|
+
*/
|
|
185
|
+
const answered = asName(row.response_model);
|
|
186
|
+
const overridden = asName(row.model_override);
|
|
187
|
+
const requested = asName(row.request_model);
|
|
188
|
+
const model = answered ?? overridden ?? requested;
|
|
189
|
+
if (model === null) {
|
|
190
|
+
out.unnamedModel += 1;
|
|
191
|
+
continue;
|
|
192
|
+
}
|
|
193
|
+
if (requested !== null && answered !== null && requested !== answered) {
|
|
194
|
+
out.modelDisagreements += 1;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
const ts = timeOf(row);
|
|
198
|
+
const label = labelOf(row);
|
|
199
|
+
out.records.push({
|
|
200
|
+
model,
|
|
201
|
+
...(ts !== undefined ? { ts } : {}),
|
|
202
|
+
...(label !== undefined ? { label } : {}),
|
|
203
|
+
usage: { input_tokens: input ?? 0, output_tokens: output ?? 0 },
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
return out;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/**
|
|
211
|
+
* Whether a file looks like a Helicone request export, by shape.
|
|
212
|
+
*
|
|
213
|
+
* Deliberately narrow: a token column beside one of the names only Helicone's
|
|
214
|
+
* own SELECT produces. A looser test would claim an OpenAI usage response or a
|
|
215
|
+
* LiteLLM spend log — both of which also carry `prompt_tokens` — and refuse it
|
|
216
|
+
* in this file's words instead of letting the code that can read it try.
|
|
217
|
+
*/
|
|
218
|
+
export function looksLikeHelicone(text: string, prefixBytes = 8192): boolean {
|
|
219
|
+
const head = text.slice(0, prefixBytes);
|
|
220
|
+
if (!head.includes('prompt_tokens') && !head.includes('completion_tokens')) return false;
|
|
221
|
+
return (
|
|
222
|
+
head.includes('"request_created_at"') ||
|
|
223
|
+
head.includes('"request_model"') ||
|
|
224
|
+
head.includes('"response_model"') ||
|
|
225
|
+
head.includes('"request_properties"')
|
|
226
|
+
);
|
|
227
|
+
}
|
package/src/i18n/en.ts
CHANGED
|
@@ -82,7 +82,7 @@ export const en: CoreMessages = {
|
|
|
82
82
|
|
|
83
83
|
suggest: {
|
|
84
84
|
'not-found': () =>
|
|
85
|
-
'the quoted phrase is not in the prompt
|
|
85
|
+
'the quoted phrase is not in the prompt: the model paraphrased what it was copying',
|
|
86
86
|
'touches-protected': () =>
|
|
87
87
|
'it would edit code, a URL, a placeholder or a tag, which are copied verbatim',
|
|
88
88
|
'introduces-protected': () => 'the replacement adds a placeholder or URL that was not there',
|
|
@@ -95,10 +95,10 @@ export const en: CoreMessages = {
|
|
|
95
95
|
* Three shapes, not two, because *"how wrong might this estimate be"* has
|
|
96
96
|
* three answers and only two were written down.
|
|
97
97
|
*
|
|
98
|
-
* The count is exact
|
|
99
|
-
* estimate on the family the estimator was measured against
|
|
98
|
+
* The count is exact, and then the window verdict is a fact. The count is an
|
|
99
|
+
* estimate on the family the estimator was measured against, and then the
|
|
100
100
|
* margin is known and the verdict is a probability. Or the count is an
|
|
101
|
-
* estimate on a family nobody has measured
|
|
101
|
+
* estimate on a family nobody has measured, and then the margin itself is
|
|
102
102
|
* the unknown, which is neither of the first two and used to be told as the
|
|
103
103
|
* second.
|
|
104
104
|
*
|
|
@@ -116,15 +116,15 @@ export const en: CoreMessages = {
|
|
|
116
116
|
detail: !uncertain
|
|
117
117
|
? `The optimised prompt is ~${n(tokens)} tokens and ${modelName} accepts ${n(contextWindow)}. The call will fail: split the content or move to a model with a larger window.`
|
|
118
118
|
: bandApplies
|
|
119
|
-
? `The optimised prompt is ~${n(tokens)} tokens against ${modelName}'s ${n(contextWindow)}. That count is an estimate and it is close to the line, so the call will probably fail but might not
|
|
120
|
-
: `The optimised prompt is ~${n(tokens)} tokens against ${modelName}'s ${n(contextWindow)}, which is over. That count is an estimate, and the estimator's error was measured against Claude's tokenizer
|
|
119
|
+
? `The optimised prompt is ~${n(tokens)} tokens against ${modelName}'s ${n(contextWindow)}. That count is an estimate and it is close to the line, so the call will probably fail but might not. Settle it with --exact-tokens before rewriting anything. The counting endpoint is free. If it does exceed the window, split the content or move to a model with a larger one.`
|
|
120
|
+
: `The optimised prompt is ~${n(tokens)} tokens against ${modelName}'s ${n(contextWindow)}, which is over. That count is an estimate, and the estimator's error was measured against Claude's tokenizer rather than ${modelName}'s, so how far over it really is cannot be said from here. Trazum will not tell you the call fails on a number it has not measured for this family. Count with your provider's own tooling before rewriting anything.`,
|
|
121
121
|
}),
|
|
122
122
|
|
|
123
123
|
contextNearLimit: ({ tokens, modelName, contextWindow, bandApplies }) => ({
|
|
124
124
|
title: 'The prompt may not fit in the context window',
|
|
125
125
|
detail: bandApplies
|
|
126
|
-
? `The optimised prompt is ~${n(tokens)} tokens against ${modelName}'s ${n(contextWindow)}, which fits
|
|
127
|
-
: `The optimised prompt is ~${n(tokens)} tokens against ${modelName}'s ${n(contextWindow)}, which fits on the estimate. This warning is raised because the estimator's measured error
|
|
126
|
+
? `The optimised prompt is ~${n(tokens)} tokens against ${modelName}'s ${n(contextWindow)}, which fits, but that count is an estimate and its error range reaches past the window, so the real prompt may not. A call that exceeds the window fails outright rather than degrading, and nothing else here warns about it. Confirm with --exact-tokens; the counting endpoint is free.`
|
|
127
|
+
: `The optimised prompt is ~${n(tokens)} tokens against ${modelName}'s ${n(contextWindow)}, which fits on the estimate. This warning is raised because the estimator's measured error, against Claude's tokenizer rather than ${modelName}'s, would reach past the window, and nobody has measured what that error is on this family. A call that exceeds the window fails outright rather than degrading. Count with your provider's own tooling before relying on the margin.`,
|
|
128
128
|
}),
|
|
129
129
|
|
|
130
130
|
promptCaching: ({
|
|
@@ -140,13 +140,13 @@ export const en: CoreMessages = {
|
|
|
140
140
|
nearMinimum,
|
|
141
141
|
}) => {
|
|
142
142
|
const scope = placeholder
|
|
143
|
-
? `The stable prefix
|
|
143
|
+
? `The stable prefix (everything before the first placeholder ${placeholder}) is ~${n(prefixTokens)} of the prompt's ${n(totalTokens)} tokens, and clears ${modelName}'s ${n(minTokens)}-token cacheable minimum.`
|
|
144
144
|
: `The prompt has no variable placeholders, so the whole thing is a cacheable prefix and it clears ${modelName}'s ${n(minTokens)}-token minimum.`;
|
|
145
145
|
const how = explicit
|
|
146
146
|
? 'Put the cache marker at the end of the stable prefix: any byte that changes before the cut invalidates everything after it.'
|
|
147
|
-
: `${modelName} caches automatically above its minimum, so there is nothing to set
|
|
147
|
+
: `${modelName} caches automatically above its minimum, so there is nothing to set, but the same rule applies: any byte that changes before the cut invalidates everything after it.`;
|
|
148
148
|
const hedge = nearMinimum
|
|
149
|
-
? ` One caveat on the figure: that prefix count is an estimate and it is close to the line, so the real one may be below the ${n(minTokens)}-token minimum
|
|
149
|
+
? ` One caveat on the figure: that prefix count is an estimate and it is close to the line, so the real one may be below the ${n(minTokens)}-token minimum, in which case nothing caches and this saving is not there. Settle it with --exact-tokens before budgeting from it. The counting endpoint is free.`
|
|
150
150
|
: '';
|
|
151
151
|
return {
|
|
152
152
|
title: 'Turn on prompt caching for the stable prefix',
|
|
@@ -154,10 +154,12 @@ export const en: CoreMessages = {
|
|
|
154
154
|
};
|
|
155
155
|
},
|
|
156
156
|
|
|
157
|
-
promptCachingNotWorthIt: () => ({
|
|
157
|
+
promptCachingNotWorthIt: ({ readPct, writePct, breakEvenPct }) => ({
|
|
158
158
|
title: 'At that hit rate, caching does not pay off',
|
|
159
159
|
detail:
|
|
160
|
-
|
|
160
|
+
breakEvenPct === null
|
|
161
|
+
? `A cache write costs ${writePct}% of the input price on this model, the same as not caching, and a read costs ${readPct}%. So caching cannot lose you money here: at this hit rate it simply buys nothing. Raise prefix reuse and it starts paying immediately.`
|
|
162
|
+
: `A cache write costs ${writePct}% of the input price on this model and a read costs ${readPct}%. Below a ${breakEvenPct}% hit rate you pay more than you save. Either raise prefix reuse or leave caching off.`,
|
|
161
163
|
}),
|
|
162
164
|
|
|
163
165
|
belowCacheMinimum: ({
|
|
@@ -180,14 +182,14 @@ export const en: CoreMessages = {
|
|
|
180
182
|
? ' Claude Opus 5 lowers that minimum to 512 tokens, so short prompts that miss here would cache there.'
|
|
181
183
|
: '') +
|
|
182
184
|
(couldReachMinimum
|
|
183
|
-
? ' That prefix count is an estimate and it is close to the line, so the real one may already be above it
|
|
185
|
+
? ' That prefix count is an estimate and it is close to the line, so the real one may already be above it. Check with --exact-tokens before deciding this is not available to you. The counting endpoint is free.'
|
|
184
186
|
: ''),
|
|
185
187
|
};
|
|
186
188
|
},
|
|
187
189
|
|
|
188
190
|
cachePrefixReorder: ({ staticTokensAfter, sharePct, placeholder, command }) => ({
|
|
189
191
|
title: 'Move the stable instructions ahead of the first placeholder',
|
|
190
|
-
detail: `About ~${n(staticTokensAfter)} tokens of stable content (${sharePct}% of the prompt) sit after the first variable placeholder ${placeholder}, so today they never get cached. Fixed instructions and context first, placeholders last, and that content starts being read from cache at 10% of the price. Run \`${command}\` to attempt it: whole blocks only, and it refuses to move anything that refers back to earlier text. Read the diff
|
|
192
|
+
detail: `About ~${n(staticTokensAfter)} tokens of stable content (${sharePct}% of the prompt) sit after the first variable placeholder ${placeholder}, so today they never get cached. Fixed instructions and context first, placeholders last, and that content starts being read from cache at 10% of the price. Run \`${command}\` to attempt it: whole blocks only, and it refuses to move anything that refers back to earlier text. Read the diff: order carries meaning, and "summarise the text above" is nonsense in front of the text it points at.`,
|
|
191
193
|
}),
|
|
192
194
|
|
|
193
195
|
batchApi: () => ({
|
|
@@ -198,7 +200,12 @@ export const en: CoreMessages = {
|
|
|
198
200
|
|
|
199
201
|
modelDowngrade: ({ modelName, tier, candidateName, currentUsd, candidateUsd }) => ({
|
|
200
202
|
title: `This task may not need ${modelName}`,
|
|
201
|
-
detail: `By length and vocabulary the prompt looks like "${tier}" complexity. On ${candidateName} you would go from ${currentUsd} to ${candidateUsd} per month. This is a keyword heuristic, not a judgement about answer quality:
|
|
203
|
+
detail: `By length and vocabulary the prompt looks like "${tier}" complexity. On ${candidateName} you would go from ${currentUsd} to ${candidateUsd} per month. This is a keyword heuristic, not a judgement about answer quality. Measure it before switching in production: \`trazum route <log.jsonl> --prompt-file <prompt> --cases <cases> --yes\` sends the same cases to both models and reports whether the cheaper one still does the job.`,
|
|
204
|
+
}),
|
|
205
|
+
|
|
206
|
+
tierSignalsConflict: ({ complexSignals, simpleSignals }) => ({
|
|
207
|
+
title: 'The prompt asks for depth and for brevity at once',
|
|
208
|
+
detail: `It carries ${complexSignals} signal${complexSignals === 1 ? '' : 's'} of a hard task and ${simpleSignals} of an easy one, and this heuristic scores those against each other, so they cancel. No tier is recommended here: a number produced by two opposite readings cancelling out looks exactly like a prompt with no signals at all, and they are not the same thing. Decide it by measurement rather than by vocabulary: \`trazum route <log.jsonl> --prompt-file <prompt> --cases <cases> --yes\`.`,
|
|
202
209
|
}),
|
|
203
210
|
|
|
204
211
|
outputDominated: ({ outputUsd, inputUsd }) => ({
|
|
@@ -222,7 +229,7 @@ export const en: CoreMessages = {
|
|
|
222
229
|
title: `Two instructions disagree about ${axis}`,
|
|
223
230
|
detail:
|
|
224
231
|
`One says ${firstValue} ("${firstSnippet}") and another says ${secondValue} ("${secondSnippet}"). ` +
|
|
225
|
-
'The model has to pick one, and which one it picks can change between calls
|
|
232
|
+
'The model has to pick one, and which one it picks can change between calls, so this is a correctness problem before it is a cost one. ' +
|
|
226
233
|
'Deleting the instruction you did not mean also shortens the prompt.' +
|
|
227
234
|
(otherCount > 0
|
|
228
235
|
? ` ${otherCount} other ${otherCount === 1 ? 'pair' : 'pairs'} of instructions disagree too.`
|
|
@@ -241,8 +248,8 @@ export const en: CoreMessages = {
|
|
|
241
248
|
title: 'The output schema could travel in the request instead of the prompt',
|
|
242
249
|
detail:
|
|
243
250
|
`${blocks === 1 ? 'A schema block' : `${blocks} schema blocks`} introduced by "${cue}" defines ${keyList}, costing about ${n(tokens)} tokens on every call. ` +
|
|
244
|
-
'Every major API now takes a response schema as a request parameter
|
|
245
|
-
'Trazum cannot make this edit, because it changes the call rather than the prompt, and it does not check whether your provider offers the parameter
|
|
251
|
+
'Every major API now takes a response schema as a request parameter (output_config.format, response_format, responseSchema), and moving it there is the rare change that is both cheaper and stricter: prose asks the model to comply, a parameter makes the decoder comply. ' +
|
|
252
|
+
'Trazum cannot make this edit, because it changes the call rather than the prompt, and it does not check whether your provider offers the parameter; if it does not, this is not available to you.',
|
|
246
253
|
}),
|
|
247
254
|
|
|
248
255
|
restatedOutputFormat: ({ restatedCount, totalCount, restatedTokens, keyList }) => ({
|
|
@@ -250,7 +257,7 @@ export const en: CoreMessages = {
|
|
|
250
257
|
detail:
|
|
251
258
|
`The prose describes ${restatedCount} of the ${totalCount} fields your schema already defines (${keyList}), costing about ${n(restatedTokens)} tokens on every call. ` +
|
|
252
259
|
'The code block is the version worth keeping: it is unambiguous, and Trazum never edits it. ' +
|
|
253
|
-
'Check the two agree before deleting either
|
|
260
|
+
'Check the two agree before deleting either: when a prompt says one thing in prose and another in its schema, the prose is usually the one that went stale.',
|
|
254
261
|
}),
|
|
255
262
|
},
|
|
256
263
|
|
package/src/i18n/es.ts
CHANGED
|
@@ -138,10 +138,12 @@ export const es: CoreMessages = {
|
|
|
138
138
|
};
|
|
139
139
|
},
|
|
140
140
|
|
|
141
|
-
promptCachingNotWorthIt: () => ({
|
|
141
|
+
promptCachingNotWorthIt: ({ readPct, writePct, breakEvenPct }) => ({
|
|
142
142
|
title: 'Con esa tasa de acierto, la caché no compensa',
|
|
143
143
|
detail:
|
|
144
|
-
|
|
144
|
+
breakEvenPct === null
|
|
145
|
+
? `En este modelo escribir en caché cuesta un ${writePct}% del precio de entrada, lo mismo que no usarla, y leer un ${readPct}%. Así que aquí la caché no puede costarte dinero: con esta tasa de acierto simplemente no te aporta nada. En cuanto subas la reutilización del prefijo, empieza a compensar.`
|
|
146
|
+
: `En este modelo escribir en caché cuesta un ${writePct}% del precio de entrada y leer un ${readPct}%. Por debajo de un ${breakEvenPct}% de aciertos pagas más de lo que ahorras. Sube la reutilización del prefijo o deja la caché desactivada.`,
|
|
145
147
|
}),
|
|
146
148
|
|
|
147
149
|
belowCacheMinimum: ({
|
|
@@ -182,7 +184,12 @@ export const es: CoreMessages = {
|
|
|
182
184
|
|
|
183
185
|
modelDowngrade: ({ modelName, tier, candidateName, currentUsd, candidateUsd }) => ({
|
|
184
186
|
title: `Esta tarea quizá no necesite ${modelName}`,
|
|
185
|
-
detail: `Por longitud y vocabulario, el prompt parece de complejidad "${tier}". Con ${candidateName} pasarías de ${currentUsd} a ${candidateUsd} al mes. Es una heurística por palabras clave, no un juicio de calidad:
|
|
187
|
+
detail: `Por longitud y vocabulario, el prompt parece de complejidad "${tier}". Con ${candidateName} pasarías de ${currentUsd} a ${candidateUsd} al mes. Es una heurística por palabras clave, no un juicio de calidad. Mídelo antes de cambiar en producción: \`trazum route <log.jsonl> --prompt-file <prompt> --cases <casos> --yes\` envía los mismos casos a ambos modelos y dice si el barato sigue haciendo el trabajo.`,
|
|
188
|
+
}),
|
|
189
|
+
|
|
190
|
+
tierSignalsConflict: ({ complexSignals, simpleSignals }) => ({
|
|
191
|
+
title: 'El prompt pide profundidad y brevedad a la vez',
|
|
192
|
+
detail: `Lleva ${complexSignals} señal${complexSignals === 1 ? '' : 'es'} de tarea difícil y ${simpleSignals} de tarea fácil, y esta heurística las resta entre sí, así que se anulan. Aquí no se recomienda ningún nivel: un número que sale de dos lecturas opuestas cancelándose se parece exactamente a un prompt sin ninguna señal, y no son lo mismo. Decídelo midiendo y no por vocabulario: \`trazum route <log.jsonl> --prompt-file <prompt> --cases <casos> --yes\`.`,
|
|
186
193
|
}),
|
|
187
194
|
|
|
188
195
|
outputDominated: ({ outputUsd, inputUsd }) => ({
|
package/src/i18n/types.ts
CHANGED
|
@@ -215,6 +215,49 @@ export interface LlmMessages {
|
|
|
215
215
|
suspiciousShrink(retainedPct: number): string;
|
|
216
216
|
}
|
|
217
217
|
|
|
218
|
+
/**
|
|
219
|
+
* What the caching-does-not-pay advisory needs to state a threshold that is
|
|
220
|
+
* true about **this** model.
|
|
221
|
+
*
|
|
222
|
+
* It used to take nothing and print one sentence for every model: *"a cache
|
|
223
|
+
* write costs 125% of the input price and a read costs 10%. Below roughly a 28%
|
|
224
|
+
* hit rate you pay more than you save."* Two of those three numbers were the
|
|
225
|
+
* Anthropic multipliers stated as universal, and the third was not derivable
|
|
226
|
+
* from any model in the catalogue: at 125% and 10% the break-even is 21.74%,
|
|
227
|
+
* not 28%.
|
|
228
|
+
*
|
|
229
|
+
* The multipliers were already in hand at the call site and already passed to
|
|
230
|
+
* the advisory next door. Only this message invented its own.
|
|
231
|
+
*/
|
|
232
|
+
export interface PromptCachingNotWorthItParams {
|
|
233
|
+
/** The model's cache read multiplier as a percentage of the input price. */
|
|
234
|
+
readPct: number;
|
|
235
|
+
/** Its 5-minute cache write multiplier, likewise. */
|
|
236
|
+
writePct: number;
|
|
237
|
+
/**
|
|
238
|
+
* The hit rate at which caching stops costing money, derived rather than
|
|
239
|
+
* quoted: `(1 - write) / (read - write)`.
|
|
240
|
+
*
|
|
241
|
+
* `null` when the write multiplier is 1, which is not a smaller threshold but
|
|
242
|
+
* a different situation: writing costs exactly what not caching costs, so
|
|
243
|
+
* caching can never lose money and there is no rate below which it does. Eight
|
|
244
|
+
* of the eighteen models in the catalogue are that shape, and the old sentence
|
|
245
|
+
* told all eight to consider turning caching off.
|
|
246
|
+
*/
|
|
247
|
+
breakEvenPct: number | null;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/**
|
|
251
|
+
* What the tier heuristic saw when it declined to answer.
|
|
252
|
+
*
|
|
253
|
+
* Both counts, because "three of each" and "one of each" are different degrees
|
|
254
|
+
* of the same refusal and the reader can act on the difference.
|
|
255
|
+
*/
|
|
256
|
+
export interface TierSignalsConflictParams {
|
|
257
|
+
complexSignals: number;
|
|
258
|
+
simpleSignals: number;
|
|
259
|
+
}
|
|
260
|
+
|
|
218
261
|
/** Why a proposed rewrite did not survive checking against the prompt. */
|
|
219
262
|
export interface SuggestMessages {
|
|
220
263
|
'not-found'(): string;
|
|
@@ -236,11 +279,12 @@ export interface CoreMessages {
|
|
|
236
279
|
contextOverflow(p: ContextOverflowParams): LocalizedMessage;
|
|
237
280
|
contextNearLimit(p: ContextNearLimitParams): LocalizedMessage;
|
|
238
281
|
promptCaching(p: PromptCachingParams): LocalizedMessage;
|
|
239
|
-
promptCachingNotWorthIt(): LocalizedMessage;
|
|
282
|
+
promptCachingNotWorthIt(p: PromptCachingNotWorthItParams): LocalizedMessage;
|
|
240
283
|
belowCacheMinimum(p: BelowCacheMinimumParams): LocalizedMessage;
|
|
241
284
|
cachePrefixReorder(p: CachePrefixReorderParams): LocalizedMessage;
|
|
242
285
|
batchApi(): LocalizedMessage;
|
|
243
286
|
modelDowngrade(p: ModelDowngradeParams): LocalizedMessage;
|
|
287
|
+
tierSignalsConflict(p: TierSignalsConflictParams): LocalizedMessage;
|
|
244
288
|
outputDominated(p: OutputDominatedParams): LocalizedMessage;
|
|
245
289
|
promoPricing(p: PromoPricingParams): LocalizedMessage;
|
|
246
290
|
contradictoryInstructions(p: ContradictoryInstructionsParams): LocalizedMessage;
|
package/src/index.ts
CHANGED
|
@@ -166,7 +166,18 @@ export type {
|
|
|
166
166
|
GatewayStanding,
|
|
167
167
|
RefuseReason,
|
|
168
168
|
} from './gateway.js';
|
|
169
|
+
export {
|
|
170
|
+
BANDS,
|
|
171
|
+
MEASURED_FOREIGN_ERROR_PCT,
|
|
172
|
+
bandFor,
|
|
173
|
+
bucketFor,
|
|
174
|
+
foreignTokenizer,
|
|
175
|
+
measuredForeignError,
|
|
176
|
+
} from './band.js';
|
|
177
|
+
export type { BandBucket } from './band.js';
|
|
169
178
|
export { conform, CONTRACT_NAMES, requiredFieldsOf } from './conform.js';
|
|
179
|
+
export { readDroppedVerdict, verdictMatchesSlice } from './verdict-bridge.js';
|
|
180
|
+
export type { DroppedVerdict, BridgeReading } from './verdict-bridge.js';
|
|
170
181
|
export { CONTRACT_SCHEMAS, contractSchema } from './schemas.js';
|
|
171
182
|
export type {
|
|
172
183
|
ConformOptions,
|
|
@@ -203,6 +214,10 @@ export type { MeasuredIndex } from './position.js';
|
|
|
203
214
|
export { claudeCodeRecords, looksLikeClaudeCodeTranscript } from './claude-code.js';
|
|
204
215
|
export { otelRecords, looksLikeOtel } from './otel.js';
|
|
205
216
|
export type { OtelConversion, OtelRecord } from './otel.js';
|
|
217
|
+
export { litellmRecords, looksLikeLiteLlm } from './litellm.js';
|
|
218
|
+
export type { LiteLlmConversion, LiteLlmRecord } from './litellm.js';
|
|
219
|
+
export { heliconeRecords, looksLikeHelicone } from './helicone.js';
|
|
220
|
+
export type { HeliconeConversion, HeliconeRecord } from './helicone.js';
|
|
206
221
|
export type { ClaudeCodeConversion, ClaudeCodeRecord } from './claude-code.js';
|
|
207
222
|
export { positionReport } from './position-report.js';
|
|
208
223
|
export type {
|
|
@@ -424,7 +439,8 @@ export type {
|
|
|
424
439
|
export { RULES, getRule } from './rules.js';
|
|
425
440
|
export { segment, join, protectedTexts } from './segment.js';
|
|
426
441
|
export { computeSavings, costOfCall, formatUsd, formatSignedUsd } from './savings.js';
|
|
427
|
-
export { buildAdvisories, recommendTier } from './advisories.js';
|
|
442
|
+
export { buildAdvisories, recommendTier, recommendTierDetailed } from './advisories.js';
|
|
443
|
+
export type { TierRecommendation } from './advisories.js';
|
|
428
444
|
export type { AdvisoryOptions } from './advisories.js';
|
|
429
445
|
export { analyzeCachePrefix } from './cache.js';
|
|
430
446
|
|