@trazum/core 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +163 -0
- package/dist/advisories.d.ts +23 -0
- package/dist/advisories.d.ts.map +1 -0
- package/dist/advisories.js +376 -0
- package/dist/advisories.js.map +1 -0
- package/dist/aws-sigv4.d.ts +88 -0
- package/dist/aws-sigv4.d.ts.map +1 -0
- package/dist/aws-sigv4.js +117 -0
- package/dist/aws-sigv4.js.map +1 -0
- package/dist/baseline.d.ts +171 -0
- package/dist/baseline.d.ts.map +1 -0
- package/dist/baseline.js +273 -0
- package/dist/baseline.js.map +1 -0
- package/dist/cache.d.ts +26 -0
- package/dist/cache.d.ts.map +1 -0
- package/dist/cache.js +28 -0
- package/dist/cache.js.map +1 -0
- package/dist/changes.d.ts +29 -0
- package/dist/changes.d.ts.map +1 -0
- package/dist/changes.js +142 -0
- package/dist/changes.js.map +1 -0
- package/dist/compare.d.ts +65 -0
- package/dist/compare.d.ts.map +1 -0
- package/dist/compare.js +58 -0
- package/dist/compare.js.map +1 -0
- package/dist/config-schema.d.ts +118 -0
- package/dist/config-schema.d.ts.map +1 -0
- package/dist/config-schema.js +315 -0
- package/dist/config-schema.js.map +1 -0
- package/dist/config.d.ts +47 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +112 -0
- package/dist/config.js.map +1 -0
- package/dist/detect.d.ts +70 -0
- package/dist/detect.d.ts.map +1 -0
- package/dist/detect.js +228 -0
- package/dist/detect.js.map +1 -0
- package/dist/evaluate.d.ts +98 -0
- package/dist/evaluate.d.ts.map +1 -0
- package/dist/evaluate.js +110 -0
- package/dist/evaluate.js.map +1 -0
- package/dist/extract.d.ts +81 -0
- package/dist/extract.d.ts.map +1 -0
- package/dist/extract.js +280 -0
- package/dist/extract.js.map +1 -0
- package/dist/gcp-auth.d.ts +58 -0
- package/dist/gcp-auth.d.ts.map +1 -0
- package/dist/gcp-auth.js +113 -0
- package/dist/gcp-auth.js.map +1 -0
- package/dist/glob.d.ts +49 -0
- package/dist/glob.d.ts.map +1 -0
- package/dist/glob.js +154 -0
- package/dist/glob.js.map +1 -0
- package/dist/host.d.ts +30 -0
- package/dist/host.d.ts.map +1 -0
- package/dist/host.js +69 -0
- package/dist/host.js.map +1 -0
- package/dist/i18n/en.d.ts +4 -0
- package/dist/i18n/en.d.ts.map +1 -0
- package/dist/i18n/en.js +168 -0
- package/dist/i18n/en.js.map +1 -0
- package/dist/i18n/es.d.ts +4 -0
- package/dist/i18n/es.d.ts.map +1 -0
- package/dist/i18n/es.js +168 -0
- package/dist/i18n/es.js.map +1 -0
- package/dist/i18n/index.d.ts +36 -0
- package/dist/i18n/index.d.ts.map +1 -0
- package/dist/i18n/index.js +50 -0
- package/dist/i18n/index.js.map +1 -0
- package/dist/i18n/types.d.ts +180 -0
- package/dist/i18n/types.d.ts.map +1 -0
- package/dist/i18n/types.js +11 -0
- package/dist/i18n/types.js.map +1 -0
- package/dist/index.d.ts +66 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +69 -0
- package/dist/index.js.map +1 -0
- package/dist/llm.d.ts +226 -0
- package/dist/llm.d.ts.map +1 -0
- package/dist/llm.js +485 -0
- package/dist/llm.js.map +1 -0
- package/dist/nearest.d.ts +20 -0
- package/dist/nearest.d.ts.map +1 -0
- package/dist/nearest.js +54 -0
- package/dist/nearest.js.map +1 -0
- package/dist/net.d.ts +90 -0
- package/dist/net.d.ts.map +1 -0
- package/dist/net.js +203 -0
- package/dist/net.js.map +1 -0
- package/dist/node.d.ts +32 -0
- package/dist/node.d.ts.map +1 -0
- package/dist/node.js +33 -0
- package/dist/node.js.map +1 -0
- package/dist/openrouter.d.ts +25 -0
- package/dist/openrouter.d.ts.map +1 -0
- package/dist/openrouter.js +72 -0
- package/dist/openrouter.js.map +1 -0
- package/dist/optimize.d.ts +38 -0
- package/dist/optimize.d.ts.map +1 -0
- package/dist/optimize.js +183 -0
- package/dist/optimize.js.map +1 -0
- package/dist/otlp.d.ts +91 -0
- package/dist/otlp.d.ts.map +1 -0
- package/dist/otlp.js +102 -0
- package/dist/otlp.js.map +1 -0
- package/dist/phrases.d.ts +169 -0
- package/dist/phrases.d.ts.map +1 -0
- package/dist/phrases.js +939 -0
- package/dist/phrases.js.map +1 -0
- package/dist/pricing-overlay.d.ts +55 -0
- package/dist/pricing-overlay.d.ts.map +1 -0
- package/dist/pricing-overlay.js +241 -0
- package/dist/pricing-overlay.js.map +1 -0
- package/dist/pricing.d.ts +115 -0
- package/dist/pricing.d.ts.map +1 -0
- package/dist/pricing.js +400 -0
- package/dist/pricing.js.map +1 -0
- package/dist/profile.d.ts +71 -0
- package/dist/profile.d.ts.map +1 -0
- package/dist/profile.js +55 -0
- package/dist/profile.js.map +1 -0
- package/dist/promptfoo.d.ts +58 -0
- package/dist/promptfoo.d.ts.map +1 -0
- package/dist/promptfoo.js +149 -0
- package/dist/promptfoo.js.map +1 -0
- package/dist/prune.d.ts +91 -0
- package/dist/prune.d.ts.map +1 -0
- package/dist/prune.js +110 -0
- package/dist/prune.js.map +1 -0
- package/dist/reorder.d.ts +82 -0
- package/dist/reorder.d.ts.map +1 -0
- package/dist/reorder.js +215 -0
- package/dist/reorder.js.map +1 -0
- package/dist/review.d.ts +54 -0
- package/dist/review.d.ts.map +1 -0
- package/dist/review.js +131 -0
- package/dist/review.js.map +1 -0
- package/dist/rules.d.ts +5 -0
- package/dist/rules.d.ts.map +1 -0
- package/dist/rules.js +279 -0
- package/dist/rules.js.map +1 -0
- package/dist/savings.d.ts +36 -0
- package/dist/savings.d.ts.map +1 -0
- package/dist/savings.js +83 -0
- package/dist/savings.js.map +1 -0
- package/dist/segment.d.ts +8 -0
- package/dist/segment.d.ts.map +1 -0
- package/dist/segment.js +74 -0
- package/dist/segment.js.map +1 -0
- package/dist/shared-prefix.d.ts +63 -0
- package/dist/shared-prefix.d.ts.map +1 -0
- package/dist/shared-prefix.js +151 -0
- package/dist/shared-prefix.js.map +1 -0
- package/dist/similarity.d.ts +13 -0
- package/dist/similarity.d.ts.map +1 -0
- package/dist/similarity.js +30 -0
- package/dist/similarity.js.map +1 -0
- package/dist/structure.d.ts +144 -0
- package/dist/structure.d.ts.map +1 -0
- package/dist/structure.js +455 -0
- package/dist/structure.js.map +1 -0
- package/dist/suggest.d.ts +100 -0
- package/dist/suggest.d.ts.map +1 -0
- package/dist/suggest.js +151 -0
- package/dist/suggest.js.map +1 -0
- package/dist/tokenizer.d.ts +57 -0
- package/dist/tokenizer.d.ts.map +1 -0
- package/dist/tokenizer.js +157 -0
- package/dist/tokenizer.js.map +1 -0
- package/dist/types.d.ts +296 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +3 -0
- package/dist/types.js.map +1 -0
- package/dist/walk.d.ts +40 -0
- package/dist/walk.d.ts.map +1 -0
- package/dist/walk.js +95 -0
- package/dist/walk.js.map +1 -0
- package/package.json +56 -0
- package/src/advisories.ts +431 -0
- package/src/aws-sigv4.ts +174 -0
- package/src/baseline.ts +390 -0
- package/src/cache.ts +54 -0
- package/src/changes.ts +158 -0
- package/src/compare.ts +131 -0
- package/src/config-schema.ts +451 -0
- package/src/config.ts +161 -0
- package/src/detect.ts +312 -0
- package/src/evaluate.ts +188 -0
- package/src/extract.ts +336 -0
- package/src/gcp-auth.ts +166 -0
- package/src/glob.ts +160 -0
- package/src/host.ts +90 -0
- package/src/i18n/en.ts +236 -0
- package/src/i18n/es.ts +236 -0
- package/src/i18n/index.ts +68 -0
- package/src/i18n/types.ts +230 -0
- package/src/index.ts +228 -0
- package/src/llm.ts +708 -0
- package/src/nearest.ts +61 -0
- package/src/net.ts +233 -0
- package/src/node.ts +63 -0
- package/src/openrouter.ts +125 -0
- package/src/optimize.ts +228 -0
- package/src/otlp.ts +179 -0
- package/src/phrases.ts +1047 -0
- package/src/pricing-overlay.ts +319 -0
- package/src/pricing.ts +468 -0
- package/src/profile.ts +124 -0
- package/src/promptfoo.ts +213 -0
- package/src/prune.ts +211 -0
- package/src/reorder.ts +307 -0
- package/src/review.ts +180 -0
- package/src/rules.ts +324 -0
- package/src/savings.ts +121 -0
- package/src/segment.ts +106 -0
- package/src/shared-prefix.ts +198 -0
- package/src/similarity.ts +28 -0
- package/src/structure.ts +652 -0
- package/src/suggest.ts +254 -0
- package/src/tokenizer.ts +190 -0
- package/src/types.ts +323 -0
- package/src/walk.ts +117 -0
|
@@ -0,0 +1,431 @@
|
|
|
1
|
+
import { analyzeCachePrefix } from './cache.js';
|
|
2
|
+
import { getMessages } from './i18n/index.js';
|
|
3
|
+
import type { Locale } from './i18n/types.js';
|
|
4
|
+
import { COMPLEX_SIGNALS, SIMPLE_SIGNALS } from './phrases.js';
|
|
5
|
+
import { BUNDLED_CATALOGUE, effectivePricing, modelFrom, multipliersFor } from './pricing.js';
|
|
6
|
+
import type { PricingCatalogue } from './pricing.js';
|
|
7
|
+
import { formatUsd } from './savings.js';
|
|
8
|
+
import { analyzeExamples, findContradictions, findMovableSchema,
|
|
9
|
+
findRestatedFormat } from './structure.js';
|
|
10
|
+
import { estimateTokens } from './tokenizer.js';
|
|
11
|
+
import type { Advisory, ModelPricing, TokenCounter, UsageProfile } from './types.js';
|
|
12
|
+
|
|
13
|
+
function countSignals(haystack: string, signals: readonly string[]): number {
|
|
14
|
+
let count = 0;
|
|
15
|
+
for (const signal of signals) if (haystack.includes(signal)) count++;
|
|
16
|
+
return count;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Estimates the capability tier the prompt needs.
|
|
21
|
+
*
|
|
22
|
+
* This is a keyword-and-size heuristic, not a judgement about answer quality.
|
|
23
|
+
* Treat it as a hypothesis to validate with your own evaluations before
|
|
24
|
+
* moving down a tier in production.
|
|
25
|
+
*/
|
|
26
|
+
export function recommendTier(prompt: string, tokens: number): ModelPricing['tier'] {
|
|
27
|
+
const haystack = prompt.toLowerCase();
|
|
28
|
+
let score =
|
|
29
|
+
countSignals(haystack, COMPLEX_SIGNALS) * 2 - countSignals(haystack, SIMPLE_SIGNALS) * 2;
|
|
30
|
+
|
|
31
|
+
if (tokens > 4000) score += 2;
|
|
32
|
+
else if (tokens > 1500) score += 1;
|
|
33
|
+
else if (tokens < 300) score -= 1;
|
|
34
|
+
|
|
35
|
+
if (/```|~~~/.test(prompt)) score += 1;
|
|
36
|
+
|
|
37
|
+
if (score >= 3) return 'opus';
|
|
38
|
+
if (score >= 0) return 'sonnet';
|
|
39
|
+
return 'haiku';
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const TIER_ORDER: Record<ModelPricing['tier'], number> = {
|
|
43
|
+
/**
|
|
44
|
+
* Below every real tier, so no tier is ever "less capable than unknown".
|
|
45
|
+
*
|
|
46
|
+
* The value carries the intent rather than the guard below carrying it alone.
|
|
47
|
+
* Set the other way — above everything — the comparison
|
|
48
|
+
* `TIER_ORDER[suggested] < TIER_ORDER['unknown']` is true for every prompt,
|
|
49
|
+
* and the only thing left standing between that and a recommendation is an
|
|
50
|
+
* unrelated provider filter. Mutation testing found exactly that: deleting
|
|
51
|
+
* the guard changed no test result, because a second accident was covering
|
|
52
|
+
* for it. Two accidents in a row is not a design.
|
|
53
|
+
*/
|
|
54
|
+
unknown: Number.NEGATIVE_INFINITY,
|
|
55
|
+
haiku: 0,
|
|
56
|
+
sonnet: 1,
|
|
57
|
+
opus: 2,
|
|
58
|
+
frontier: 3,
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Cheapest model of a capability tier, using today's effective input price.
|
|
63
|
+
*
|
|
64
|
+
* **Within the same provider**, which is a product decision rather than an
|
|
65
|
+
* implementation detail. Dropping from Opus to Sonnet is a one-line change;
|
|
66
|
+
* moving to another vendor is a different API, different behaviour and a
|
|
67
|
+
* migration. This advisory is already caveated as a keyword heuristic rather
|
|
68
|
+
* than a judgement about answer quality, and a keyword heuristic has no business
|
|
69
|
+
* recommending that somebody change supplier.
|
|
70
|
+
*
|
|
71
|
+
* A model with no provider recorded only matches others with none, so an overlay
|
|
72
|
+
* that adds a bare model cannot pull a switch out of thin air.
|
|
73
|
+
*/
|
|
74
|
+
function cheapestInTier(
|
|
75
|
+
tier: ModelPricing['tier'],
|
|
76
|
+
on: Date,
|
|
77
|
+
pricing: PricingCatalogue,
|
|
78
|
+
provider: string | undefined,
|
|
79
|
+
): ModelPricing | undefined {
|
|
80
|
+
const candidates = pricing.models.filter(
|
|
81
|
+
(m) =>
|
|
82
|
+
m.tier === tier &&
|
|
83
|
+
m.provider === provider &&
|
|
84
|
+
// Not generally available: recommending a model the reader cannot call is
|
|
85
|
+
// worse than recommending nothing.
|
|
86
|
+
m.recommendable !== false,
|
|
87
|
+
);
|
|
88
|
+
if (candidates.length === 0) return undefined;
|
|
89
|
+
return candidates.reduce((best, m) =>
|
|
90
|
+
effectivePricing(m, on).inputPerMTok < effectivePricing(best, on).inputPerMTok ? m : best,
|
|
91
|
+
);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export interface AdvisoryOptions {
|
|
95
|
+
/** Reference date, used to decide whether a promotional price is live. */
|
|
96
|
+
on?: Date;
|
|
97
|
+
/** Token counter, so the cache-prefix analysis matches the caller's. */
|
|
98
|
+
count?: TokenCounter;
|
|
99
|
+
locale?: Locale;
|
|
100
|
+
/** Prices to work from. Defaults to the catalogue bundled with this release. */
|
|
101
|
+
pricing?: PricingCatalogue;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** Builds the advisories that do not modify the prompt but do move the bill. */
|
|
105
|
+
export function buildAdvisories(
|
|
106
|
+
optimizedPrompt: string,
|
|
107
|
+
tokensAfter: number,
|
|
108
|
+
usage: UsageProfile,
|
|
109
|
+
options: AdvisoryOptions = {},
|
|
110
|
+
): Advisory[] {
|
|
111
|
+
const { on = new Date(), count = estimateTokens, locale, pricing = BUNDLED_CATALOGUE } = options;
|
|
112
|
+
const t = getMessages(locale);
|
|
113
|
+
|
|
114
|
+
const advisories: Advisory[] = [];
|
|
115
|
+
const model = modelFrom(pricing, usage.model);
|
|
116
|
+
const { inputPerMTok, outputPerMTok, promoApplied } = effectivePricing(model, on);
|
|
117
|
+
// Per model, not global. A cache read is ~10% of input on Anthropic and about
|
|
118
|
+
// half on OpenAI; one constant for both overstates an OpenAI caching saving
|
|
119
|
+
// fivefold, which is an invented saving rather than an imprecise one.
|
|
120
|
+
const rates = multipliersFor(model);
|
|
121
|
+
const batchFactor = usage.batchEligible ? (rates.batch ?? 1) : 1;
|
|
122
|
+
|
|
123
|
+
const monthlyInputUsd =
|
|
124
|
+
(tokensAfter / 1_000_000) * inputPerMTok * usage.callsPerMonth * batchFactor;
|
|
125
|
+
const monthlyOutputUsd =
|
|
126
|
+
(usage.avgOutputTokens / 1_000_000) * outputPerMTok * usage.callsPerMonth * batchFactor;
|
|
127
|
+
|
|
128
|
+
// --- Context window ---
|
|
129
|
+
if (tokensAfter > model.contextWindow) {
|
|
130
|
+
advisories.push({
|
|
131
|
+
id: 'context-overflow',
|
|
132
|
+
severity: 'warning',
|
|
133
|
+
...t.advisories.contextOverflow({
|
|
134
|
+
tokens: tokensAfter,
|
|
135
|
+
modelName: model.displayName,
|
|
136
|
+
contextWindow: model.contextWindow,
|
|
137
|
+
}),
|
|
138
|
+
estimatedMonthlyUsd: null,
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// --- Prompt caching ---
|
|
143
|
+
// Caching is a prefix match: in a template with placeholders, only what
|
|
144
|
+
// precedes the first placeholder is cached. Costing the whole prompt would
|
|
145
|
+
// be a lie the moment {{x}} takes a different value between calls.
|
|
146
|
+
const cache = analyzeCachePrefix(optimizedPrompt, count);
|
|
147
|
+
// A provider with no prompt caching gets no caching advice. Without this the
|
|
148
|
+
// zero minimum satisfies `0 >= 0` and Trazum offers a saving that cannot be
|
|
149
|
+
// bought at any price — the exact failure that moving the multipliers onto the
|
|
150
|
+
// model was meant to prevent, reintroduced one field along.
|
|
151
|
+
/**
|
|
152
|
+
* Three ways to have nothing useful to say about caching, and only one of
|
|
153
|
+
* them used to be handled.
|
|
154
|
+
*
|
|
155
|
+
* `none` is a fact: the provider does not cache, so advice would be a saving
|
|
156
|
+
* nobody can buy. `unknown` is the absence of a fact — a catalogue built from
|
|
157
|
+
* a live price feed knows what a model costs and not how it caches — and it
|
|
158
|
+
* has to decline for the opposite reason: not because the answer is no, but
|
|
159
|
+
* because nobody asked anyone. Guessing either way is a number in somebody's
|
|
160
|
+
* budget that came from nowhere.
|
|
161
|
+
*/
|
|
162
|
+
const cachingKnown = model.caching !== 'none' && model.caching !== 'unknown' && model.cacheMinTokens !== null;
|
|
163
|
+
|
|
164
|
+
if (usage.callsPerMonth > 1 && cachingKnown) {
|
|
165
|
+
const prefixShare = tokensAfter > 0 ? cache.stablePrefixTokens / tokensAfter : 0;
|
|
166
|
+
const monthlyPrefixUsd = monthlyInputUsd * Math.min(1, prefixShare);
|
|
167
|
+
const hitRate = Math.min(Math.max(usage.cacheHitRate, 0), 1);
|
|
168
|
+
const factor = (1 - hitRate) * rates.cacheWrite5m + hitRate * rates.cacheRead;
|
|
169
|
+
|
|
170
|
+
const minTokens = model.cacheMinTokens ?? 0;
|
|
171
|
+
|
|
172
|
+
if (cache.stablePrefixTokens >= minTokens) {
|
|
173
|
+
const saving = monthlyPrefixUsd * (1 - factor);
|
|
174
|
+
if (saving > 0) {
|
|
175
|
+
advisories.push({
|
|
176
|
+
id: 'prompt-caching',
|
|
177
|
+
severity: 'opportunity',
|
|
178
|
+
...t.advisories.promptCaching({
|
|
179
|
+
placeholder: cache.firstPlaceholder,
|
|
180
|
+
prefixTokens: cache.stablePrefixTokens,
|
|
181
|
+
totalTokens: tokensAfter,
|
|
182
|
+
minTokens,
|
|
183
|
+
modelName: model.displayName,
|
|
184
|
+
hitRatePct: Math.round(hitRate * 100),
|
|
185
|
+
readPct: Math.round(rates.cacheRead * 100),
|
|
186
|
+
writePct: Math.round(rates.cacheWrite5m * 100),
|
|
187
|
+
explicit: (model.caching ?? 'explicit') === 'explicit',
|
|
188
|
+
}),
|
|
189
|
+
estimatedMonthlyUsd: saving,
|
|
190
|
+
});
|
|
191
|
+
} else {
|
|
192
|
+
advisories.push({
|
|
193
|
+
id: 'prompt-caching-not-worth-it',
|
|
194
|
+
severity: 'info',
|
|
195
|
+
...t.advisories.promptCachingNotWorthIt(),
|
|
196
|
+
estimatedMonthlyUsd: null,
|
|
197
|
+
});
|
|
198
|
+
}
|
|
199
|
+
} else {
|
|
200
|
+
advisories.push({
|
|
201
|
+
id: 'below-cache-minimum',
|
|
202
|
+
severity: 'info',
|
|
203
|
+
...t.advisories.belowCacheMinimum({
|
|
204
|
+
modelName: model.displayName,
|
|
205
|
+
minTokens,
|
|
206
|
+
placeholder: cache.firstPlaceholder,
|
|
207
|
+
prefixTokens: cache.stablePrefixTokens,
|
|
208
|
+
totalTokens: tokensAfter,
|
|
209
|
+
mentionLowerMinimum: minTokens > 512,
|
|
210
|
+
}),
|
|
211
|
+
estimatedMonthlyUsd: null,
|
|
212
|
+
});
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
// Stable content placed AFTER the first placeholder: never cached today,
|
|
216
|
+
// but moving it in front would make it cacheable.
|
|
217
|
+
if (
|
|
218
|
+
cache.firstPlaceholder &&
|
|
219
|
+
cache.staticTokensAfter >= 200 &&
|
|
220
|
+
cache.staticTokensAfter >= tokensAfter * 0.3
|
|
221
|
+
) {
|
|
222
|
+
const movableShare = tokensAfter > 0 ? cache.staticTokensAfter / tokensAfter : 0;
|
|
223
|
+
const saving = monthlyInputUsd * movableShare * Math.max(0, 1 - factor);
|
|
224
|
+
advisories.push({
|
|
225
|
+
id: 'cache-prefix-reorder',
|
|
226
|
+
severity: 'opportunity',
|
|
227
|
+
...t.advisories.cachePrefixReorder({
|
|
228
|
+
staticTokensAfter: cache.staticTokensAfter,
|
|
229
|
+
sharePct: Math.round(movableShare * 100),
|
|
230
|
+
placeholder: cache.firstPlaceholder,
|
|
231
|
+
}),
|
|
232
|
+
estimatedMonthlyUsd: saving > 0 ? saving : null,
|
|
233
|
+
});
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// --- Batch API ---
|
|
238
|
+
// `rates.batch === null` means the provider has no batch API at all, which is
|
|
239
|
+
// different from not having said: advising a discount that cannot be bought
|
|
240
|
+
// is worse than staying quiet.
|
|
241
|
+
if (!usage.batchEligible && usage.callsPerMonth > 1 && rates.batch !== null) {
|
|
242
|
+
const saving = (monthlyInputUsd + monthlyOutputUsd) * (1 - rates.batch);
|
|
243
|
+
advisories.push({
|
|
244
|
+
id: 'batch-api',
|
|
245
|
+
severity: 'opportunity',
|
|
246
|
+
...t.advisories.batchApi(),
|
|
247
|
+
estimatedMonthlyUsd: saving,
|
|
248
|
+
});
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
// --- Recommended model ---
|
|
252
|
+
const suggestedTier = recommendTier(optimizedPrompt, tokensAfter);
|
|
253
|
+
/**
|
|
254
|
+
* A model whose capability nobody recorded is never told it is overpowered.
|
|
255
|
+
*
|
|
256
|
+
* Part of the condition rather than an early return, which is how the first
|
|
257
|
+
* version of this was wrong: `return advisories` here skipped every advisory
|
|
258
|
+
* *after* the model check — output-dominated, contradictory instructions —
|
|
259
|
+
* none of which has anything to do with what tier the model is. An unknown
|
|
260
|
+
* capability is a reason to say nothing about capability, not a reason to
|
|
261
|
+
* stop reading the prompt.
|
|
262
|
+
*
|
|
263
|
+
* **Deliberately redundant**, and the one surviving mutant in this change.
|
|
264
|
+
* `TIER_ORDER.unknown` is `-Infinity`, so the comparison is already false and
|
|
265
|
+
* deleting this clause changes no behaviour and no test. It stays because the
|
|
266
|
+
* two express the same rule in different places: an ordering that forgets it
|
|
267
|
+
* and a condition that forgets it both have to happen before a model of
|
|
268
|
+
* unrecorded capability is told it is overpowered.
|
|
269
|
+
*/
|
|
270
|
+
if (model.tier !== 'unknown' && TIER_ORDER[suggestedTier] < TIER_ORDER[model.tier]) {
|
|
271
|
+
const candidate = cheapestInTier(suggestedTier, on, pricing, model.provider);
|
|
272
|
+
if (candidate) {
|
|
273
|
+
const candidatePricing = effectivePricing(candidate, on);
|
|
274
|
+
const candidateMonthly =
|
|
275
|
+
((tokensAfter / 1_000_000) * candidatePricing.inputPerMTok +
|
|
276
|
+
(usage.avgOutputTokens / 1_000_000) * candidatePricing.outputPerMTok) *
|
|
277
|
+
usage.callsPerMonth *
|
|
278
|
+
(usage.batchEligible ? 0.5 : 1);
|
|
279
|
+
const saving = monthlyInputUsd + monthlyOutputUsd - candidateMonthly;
|
|
280
|
+
if (saving > 0) {
|
|
281
|
+
advisories.push({
|
|
282
|
+
id: 'model-downgrade',
|
|
283
|
+
severity: 'opportunity',
|
|
284
|
+
...t.advisories.modelDowngrade({
|
|
285
|
+
modelName: model.displayName,
|
|
286
|
+
tier: suggestedTier,
|
|
287
|
+
candidateName: candidate.displayName,
|
|
288
|
+
currentUsd: formatUsd(monthlyInputUsd + monthlyOutputUsd),
|
|
289
|
+
candidateUsd: formatUsd(candidateMonthly),
|
|
290
|
+
}),
|
|
291
|
+
estimatedMonthlyUsd: saving,
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
// --- Where the money actually is ---
|
|
298
|
+
if (monthlyOutputUsd > monthlyInputUsd * 2 && usage.avgOutputTokens > 0) {
|
|
299
|
+
advisories.push({
|
|
300
|
+
id: 'output-dominated',
|
|
301
|
+
severity: 'info',
|
|
302
|
+
...t.advisories.outputDominated({
|
|
303
|
+
outputUsd: formatUsd(monthlyOutputUsd),
|
|
304
|
+
inputUsd: formatUsd(monthlyInputUsd),
|
|
305
|
+
}),
|
|
306
|
+
estimatedMonthlyUsd: null,
|
|
307
|
+
});
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
// --- Instructions that fight each other ---
|
|
311
|
+
// Not a saving: a contradiction is a correctness problem that happens to
|
|
312
|
+
// cost tokens, so it carries no dollar figure and sorts on severity instead.
|
|
313
|
+
const contradictions = findContradictions(optimizedPrompt);
|
|
314
|
+
const worst = contradictions[0];
|
|
315
|
+
if (worst) {
|
|
316
|
+
advisories.push({
|
|
317
|
+
id: 'contradictory-instructions',
|
|
318
|
+
severity: 'warning',
|
|
319
|
+
...t.advisories.contradictoryInstructions({
|
|
320
|
+
axis: t.contradictionAxes[worst.axis],
|
|
321
|
+
firstValue: t.contradictionValues[worst.a.value],
|
|
322
|
+
firstSnippet: worst.a.snippet,
|
|
323
|
+
secondValue: t.contradictionValues[worst.b.value],
|
|
324
|
+
secondSnippet: worst.b.snippet,
|
|
325
|
+
otherCount: contradictions.length - 1,
|
|
326
|
+
}),
|
|
327
|
+
estimatedMonthlyUsd: null,
|
|
328
|
+
});
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
// --- Few-shot examples that repeat each other ---
|
|
332
|
+
const examples = analyzeExamples(optimizedPrompt, count);
|
|
333
|
+
if (examples.redundant.length > 0 && examples.redundantTokens > 0) {
|
|
334
|
+
const saving =
|
|
335
|
+
tokensAfter > 0
|
|
336
|
+
? monthlyInputUsd * Math.min(1, examples.redundantTokens / tokensAfter)
|
|
337
|
+
: 0;
|
|
338
|
+
const topSimilarity = Math.max(...examples.redundant.map((r) => r.similarity));
|
|
339
|
+
advisories.push({
|
|
340
|
+
id: 'redundant-examples',
|
|
341
|
+
severity: 'opportunity',
|
|
342
|
+
...t.advisories.redundantExamples({
|
|
343
|
+
redundantCount: examples.redundant.length,
|
|
344
|
+
totalCount: examples.examples.length,
|
|
345
|
+
redundantTokens: examples.redundantTokens,
|
|
346
|
+
topSimilarityPct: Math.round(topSimilarity * 100),
|
|
347
|
+
}),
|
|
348
|
+
estimatedMonthlyUsd: saving > 0 ? saving : null,
|
|
349
|
+
});
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
// --- Output format written out twice ---
|
|
353
|
+
const restated = findRestatedFormat(optimizedPrompt, count);
|
|
354
|
+
const movable = findMovableSchema(optimizedPrompt, count);
|
|
355
|
+
if (movable) {
|
|
356
|
+
/**
|
|
357
|
+
* Priced from the tokens, which are real and reproducible, with the
|
|
358
|
+
* uncertainty in the text rather than in the figure.
|
|
359
|
+
*
|
|
360
|
+
* What Trazum knows is how many tokens the block holds; what it cannot know
|
|
361
|
+
* from here is whether the provider accepts a response schema. Withholding
|
|
362
|
+
* the figure for that reason would be the wrong trade — the number is right
|
|
363
|
+
* *if* the move is available, and the advisory says plainly that it does not
|
|
364
|
+
* check. The same posture as `model-downgrade`, which carries a figure and
|
|
365
|
+
* says out loud that it is a keyword heuristic.
|
|
366
|
+
*/
|
|
367
|
+
const saving =
|
|
368
|
+
tokensAfter > 0 ? monthlyInputUsd * Math.min(1, movable.tokens / tokensAfter) : 0;
|
|
369
|
+
advisories.push({
|
|
370
|
+
id: 'movable-output-schema',
|
|
371
|
+
severity: 'opportunity',
|
|
372
|
+
...t.advisories.movableSchema({
|
|
373
|
+
blocks: movable.blocks,
|
|
374
|
+
tokens: movable.tokens,
|
|
375
|
+
keyList: movable.keys.map((k) => `\`${k}\``).join(', '),
|
|
376
|
+
cue: movable.cue,
|
|
377
|
+
}),
|
|
378
|
+
estimatedMonthlyUsd: saving > 0 ? saving : null,
|
|
379
|
+
});
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
if (restated) {
|
|
383
|
+
const saving =
|
|
384
|
+
tokensAfter > 0
|
|
385
|
+
? monthlyInputUsd * Math.min(1, restated.restatedTokens / tokensAfter)
|
|
386
|
+
: 0;
|
|
387
|
+
advisories.push({
|
|
388
|
+
id: 'restated-output-format',
|
|
389
|
+
severity: 'opportunity',
|
|
390
|
+
...t.advisories.restatedOutputFormat({
|
|
391
|
+
restatedCount: restated.restatedKeys.length,
|
|
392
|
+
totalCount: restated.keys.length,
|
|
393
|
+
restatedTokens: restated.restatedTokens,
|
|
394
|
+
keyList: restated.restatedKeys.map((k) => `\`${k}\``).join(', '),
|
|
395
|
+
}),
|
|
396
|
+
estimatedMonthlyUsd: saving > 0 ? saving : null,
|
|
397
|
+
});
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
if (promoApplied && model.promo) {
|
|
401
|
+
advisories.push({
|
|
402
|
+
id: 'promo-pricing',
|
|
403
|
+
severity: 'warning',
|
|
404
|
+
...t.advisories.promoPricing({
|
|
405
|
+
modelName: model.displayName,
|
|
406
|
+
promoInput: model.promo.inputPerMTok,
|
|
407
|
+
promoOutput: model.promo.outputPerMTok,
|
|
408
|
+
until: model.promo.until,
|
|
409
|
+
listInput: model.inputPerMTok,
|
|
410
|
+
listOutput: model.outputPerMTok,
|
|
411
|
+
}),
|
|
412
|
+
estimatedMonthlyUsd: null,
|
|
413
|
+
});
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
// Warnings first, then by money. Sorting purely on the dollar figure buried
|
|
417
|
+
// things that are wrong — an overflowing context window, two instructions
|
|
418
|
+
// that contradict each other — underneath a saving of a few dollars, because
|
|
419
|
+
// being wrong carries no price tag.
|
|
420
|
+
const SEVERITY_ORDER: Record<Advisory['severity'], number> = {
|
|
421
|
+
warning: 0,
|
|
422
|
+
opportunity: 1,
|
|
423
|
+
info: 2,
|
|
424
|
+
};
|
|
425
|
+
|
|
426
|
+
return advisories.sort(
|
|
427
|
+
(a, b) =>
|
|
428
|
+
SEVERITY_ORDER[a.severity] - SEVERITY_ORDER[b.severity] ||
|
|
429
|
+
(b.estimatedMonthlyUsd ?? 0) - (a.estimatedMonthlyUsd ?? 0),
|
|
430
|
+
);
|
|
431
|
+
}
|
package/src/aws-sigv4.ts
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AWS Signature Version 4, by hand.
|
|
3
|
+
*
|
|
4
|
+
* **Why by hand.** `@trazum/core` has zero runtime dependencies and a test that
|
|
5
|
+
* fails the build if one appears. That is a security property rather than a
|
|
6
|
+
* packaging preference: this library reads people's prompts, and every
|
|
7
|
+
* dependency is somebody else's code reading them too. The AWS SDK is roughly
|
|
8
|
+
* a hundred packages to sign one request.
|
|
9
|
+
*
|
|
10
|
+
* **WebCrypto, not `node:crypto`.** The browser-safe entry point cannot reach a
|
|
11
|
+
* Node builtin — `apps/web` bundles this library, and a single `node:` import
|
|
12
|
+
* anywhere in that graph fails the build. `crypto.subtle` exists in both, so
|
|
13
|
+
* signing works in a browser and in the CLI without a second implementation.
|
|
14
|
+
* Everything here is therefore async, which HMAC-SHA256 does not need to be and
|
|
15
|
+
* `crypto.subtle` insists on anyway.
|
|
16
|
+
*
|
|
17
|
+
* **What is not asserted, and it matters.** There is no AWS-published
|
|
18
|
+
* known-answer vector in the tests. This environment cannot reach the internet
|
|
19
|
+
* to fetch one and cannot reach AWS to try a real call, so the tests check the
|
|
20
|
+
* canonical strings this builds — which are derivable from the specification by
|
|
21
|
+
* reading — plus the cryptographic properties that any correct signer has. A
|
|
22
|
+
* systematically wrong canonicalisation that is wrong *consistently* would pass
|
|
23
|
+
* all of them. The first real request against Bedrock is the proof, and until
|
|
24
|
+
* somebody makes one this is careful code rather than verified code.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
const ALGORITHM = 'AWS4-HMAC-SHA256';
|
|
28
|
+
|
|
29
|
+
const encoder = new TextEncoder();
|
|
30
|
+
|
|
31
|
+
function hex(bytes: ArrayBuffer): string {
|
|
32
|
+
return [...new Uint8Array(bytes)].map((byte) => byte.toString(16).padStart(2, '0')).join('');
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
async function sha256Hex(text: string): Promise<string> {
|
|
36
|
+
return hex(await crypto.subtle.digest('SHA-256', encoder.encode(text)));
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async function hmac(key: ArrayBuffer | Uint8Array, message: string): Promise<ArrayBuffer> {
|
|
40
|
+
const imported = await crypto.subtle.importKey(
|
|
41
|
+
'raw',
|
|
42
|
+
key as ArrayBuffer,
|
|
43
|
+
{ name: 'HMAC', hash: 'SHA-256' },
|
|
44
|
+
false,
|
|
45
|
+
['sign'],
|
|
46
|
+
);
|
|
47
|
+
return crypto.subtle.sign('HMAC', imported, encoder.encode(message));
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* `20260810T140102Z` and `20260810`, from one instant.
|
|
52
|
+
*
|
|
53
|
+
* Both derive from the same `Date` rather than being formatted twice. The
|
|
54
|
+
* classic SigV4 bug is a request signed at 23:59:59.9 whose date stamp comes
|
|
55
|
+
* from the next day: the credential scope and the timestamp disagree by one
|
|
56
|
+
* day, AWS rejects it, and it happens roughly once per thousand requests at
|
|
57
|
+
* midnight UTC and never in a test.
|
|
58
|
+
*/
|
|
59
|
+
export function amzDates(now: Date): { amzDate: string; dateStamp: string } {
|
|
60
|
+
const amzDate = `${now.toISOString().replace(/[:-]|\.\d{3}/g, '')}`;
|
|
61
|
+
return { amzDate, dateStamp: amzDate.slice(0, 8) };
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* The signing key: four chained HMACs, each keyed by the last.
|
|
66
|
+
*
|
|
67
|
+
* The chain is what scopes a signature to one day, one region and one service,
|
|
68
|
+
* so a leaked signature is not a leaked credential. `AWS4` prefixes the secret
|
|
69
|
+
* and is part of the specification rather than decoration.
|
|
70
|
+
*/
|
|
71
|
+
export async function signingKey(
|
|
72
|
+
secretAccessKey: string,
|
|
73
|
+
dateStamp: string,
|
|
74
|
+
region: string,
|
|
75
|
+
service: string,
|
|
76
|
+
): Promise<ArrayBuffer> {
|
|
77
|
+
const kDate = await hmac(encoder.encode(`AWS4${secretAccessKey}`), dateStamp);
|
|
78
|
+
const kRegion = await hmac(kDate, region);
|
|
79
|
+
const kService = await hmac(kRegion, service);
|
|
80
|
+
return hmac(kService, 'aws4_request');
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export interface SignInput {
|
|
84
|
+
method: string;
|
|
85
|
+
/** Already-encoded path, e.g. `/model/anthropic.claude-v2/invoke`. */
|
|
86
|
+
path: string;
|
|
87
|
+
host: string;
|
|
88
|
+
region: string;
|
|
89
|
+
service: string;
|
|
90
|
+
body: string;
|
|
91
|
+
accessKeyId: string;
|
|
92
|
+
secretAccessKey: string;
|
|
93
|
+
/** Present for temporary credentials, and signed when it is. */
|
|
94
|
+
sessionToken?: string;
|
|
95
|
+
now: Date;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* The headers a signed request carries.
|
|
100
|
+
*
|
|
101
|
+
* Returned rather than mutated onto a request, so this is a pure function of its
|
|
102
|
+
* input and a test can read every byte of what it produced.
|
|
103
|
+
*/
|
|
104
|
+
export interface SignedHeaders {
|
|
105
|
+
authorization: string;
|
|
106
|
+
'x-amz-date': string;
|
|
107
|
+
'x-amz-content-sha256': string;
|
|
108
|
+
'x-amz-security-token'?: string;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Builds the canonical request, exactly as the specification orders it.
|
|
113
|
+
*
|
|
114
|
+
* Exported for the tests, because this string is where a signer goes wrong.
|
|
115
|
+
* Every field is joined with a newline, headers are lowercased and sorted, and
|
|
116
|
+
* the trailing newline after the header block is required — omit it and every
|
|
117
|
+
* signature is wrong in a way whose only symptom is `403 SignatureDoesNotMatch`.
|
|
118
|
+
*/
|
|
119
|
+
export async function canonicalRequest(input: SignInput): Promise<{
|
|
120
|
+
canonical: string;
|
|
121
|
+
signedHeaderNames: string;
|
|
122
|
+
payloadHash: string;
|
|
123
|
+
}> {
|
|
124
|
+
const { amzDate } = amzDates(input.now);
|
|
125
|
+
const payloadHash = await sha256Hex(input.body);
|
|
126
|
+
|
|
127
|
+
// `host` and the two `x-amz-` headers are what Bedrock requires signed. Sorted
|
|
128
|
+
// by name because the specification says sorted, not because it looks tidier.
|
|
129
|
+
const headers: Array<[string, string]> = [
|
|
130
|
+
['host', input.host],
|
|
131
|
+
['x-amz-content-sha256', payloadHash],
|
|
132
|
+
['x-amz-date', amzDate],
|
|
133
|
+
];
|
|
134
|
+
if (input.sessionToken) headers.push(['x-amz-security-token', input.sessionToken]);
|
|
135
|
+
headers.sort((a, b) => (a[0] < b[0] ? -1 : 1));
|
|
136
|
+
|
|
137
|
+
const canonicalHeaders = headers.map(([name, value]) => `${name}:${value.trim()}\n`).join('');
|
|
138
|
+
const signedHeaderNames = headers.map(([name]) => name).join(';');
|
|
139
|
+
|
|
140
|
+
const canonical = [
|
|
141
|
+
input.method,
|
|
142
|
+
input.path,
|
|
143
|
+
// No query string on any request this signs. Present and empty, because the
|
|
144
|
+
// field is positional: dropping it shifts everything below it up a line.
|
|
145
|
+
'',
|
|
146
|
+
canonicalHeaders,
|
|
147
|
+
signedHeaderNames,
|
|
148
|
+
payloadHash,
|
|
149
|
+
].join('\n');
|
|
150
|
+
|
|
151
|
+
return { canonical, signedHeaderNames, payloadHash };
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** The full `Authorization` header value, and the headers that go with it. */
|
|
155
|
+
export async function signRequest(input: SignInput): Promise<SignedHeaders> {
|
|
156
|
+
const { amzDate, dateStamp } = amzDates(input.now);
|
|
157
|
+
const { canonical, signedHeaderNames, payloadHash } = await canonicalRequest(input);
|
|
158
|
+
|
|
159
|
+
const scope = `${dateStamp}/${input.region}/${input.service}/aws4_request`;
|
|
160
|
+
const stringToSign = [ALGORITHM, amzDate, scope, await sha256Hex(canonical)].join('\n');
|
|
161
|
+
|
|
162
|
+
const key = await signingKey(input.secretAccessKey, dateStamp, input.region, input.service);
|
|
163
|
+
const signature = hex(await hmac(key, stringToSign));
|
|
164
|
+
|
|
165
|
+
const headers: SignedHeaders = {
|
|
166
|
+
authorization:
|
|
167
|
+
`${ALGORITHM} Credential=${input.accessKeyId}/${scope}, ` +
|
|
168
|
+
`SignedHeaders=${signedHeaderNames}, Signature=${signature}`,
|
|
169
|
+
'x-amz-date': amzDate,
|
|
170
|
+
'x-amz-content-sha256': payloadHash,
|
|
171
|
+
};
|
|
172
|
+
if (input.sessionToken) headers['x-amz-security-token'] = input.sessionToken;
|
|
173
|
+
return headers;
|
|
174
|
+
}
|