@warmdrift/kgauto-check 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +50 -11
- package/dist/bin.js +19 -38
- package/dist/chunk-6ZBEABKR.js +4636 -0
- package/dist/index.d.ts +260 -14
- package/dist/index.js +13 -1
- package/package.json +5 -4
- package/dist/chunk-NHTVNYK2.js +0 -477
package/dist/chunk-NHTVNYK2.js
DELETED
|
@@ -1,477 +0,0 @@
|
|
|
1
|
-
// src/extract.ts
|
|
2
|
-
import { z } from "zod";
|
|
3
|
-
var CallSiteSchema = z.object({
|
|
4
|
-
/** Agent-chosen stable label, e.g. "api/summarize.ts:42" or "hunt loop". */
|
|
5
|
-
id: z.string().min(1),
|
|
6
|
-
/** Model id exactly as configured in the project. */
|
|
7
|
-
model: z.string().min(1),
|
|
8
|
-
/** How the model id is selected at this site. */
|
|
9
|
-
modelSelection: z.enum(["hardcoded", "config", "env", "dynamic"]).optional(),
|
|
10
|
-
/**
|
|
11
|
-
* Best-guess intent archetype from kgauto's dialect (ask, classify,
|
|
12
|
-
* critique, extract, generate, hunt, plan, summarize, transform).
|
|
13
|
-
* Free-form here; validated softly against the roster's dialect.
|
|
14
|
-
*/
|
|
15
|
-
archetype: z.string().optional(),
|
|
16
|
-
/** Typical input size in tokens (estimate is fine; ~4 chars/token). */
|
|
17
|
-
estInputTokens: z.number().nonnegative().optional(),
|
|
18
|
-
/** Worst-case input size in tokens, if larger than typical. */
|
|
19
|
-
maxInputTokens: z.number().nonnegative().optional(),
|
|
20
|
-
/** Typical output size in tokens. */
|
|
21
|
-
estOutputTokens: z.number().nonnegative().optional(),
|
|
22
|
-
/** Number of tool definitions sent with the call. */
|
|
23
|
-
toolCount: z.number().int().nonnegative().optional(),
|
|
24
|
-
/** Does the site rely on the model making parallel tool calls? */
|
|
25
|
-
usesParallelToolCalls: z.boolean().optional(),
|
|
26
|
-
/** Does the site parse the response as JSON / expect schema output? */
|
|
27
|
-
expectsStructuredOutput: z.boolean().optional(),
|
|
28
|
-
streaming: z.boolean().optional(),
|
|
29
|
-
/** What happens when the call fails. */
|
|
30
|
-
retry: z.enum(["none", "same-model", "fallback-chain", "unknown"]).optional(),
|
|
31
|
-
/** Fallback model ids, when retry is 'fallback-chain'. */
|
|
32
|
-
fallbackModels: z.array(z.string()).optional(),
|
|
33
|
-
/** Rough call volume, if inferable (cron cadence, traffic, logs). */
|
|
34
|
-
estCallsPerMonth: z.number().positive().optional(),
|
|
35
|
-
/** Anything structural the agent thinks matters. Never prompt text. */
|
|
36
|
-
notes: z.string().max(500).optional()
|
|
37
|
-
});
|
|
38
|
-
var ExtractSchema = z.object({
|
|
39
|
-
project: z.object({
|
|
40
|
-
name: z.string().min(1),
|
|
41
|
-
description: z.string().max(500).optional()
|
|
42
|
-
}),
|
|
43
|
-
callSites: z.array(CallSiteSchema).min(1),
|
|
44
|
-
notes: z.string().max(1e3).optional()
|
|
45
|
-
});
|
|
46
|
-
|
|
47
|
-
// src/version.ts
|
|
48
|
-
import { readFileSync } from "fs";
|
|
49
|
-
import { fileURLToPath } from "url";
|
|
50
|
-
import { dirname, join } from "path";
|
|
51
|
-
var CHECK_VERSION = (() => {
|
|
52
|
-
try {
|
|
53
|
-
const here = dirname(fileURLToPath(import.meta.url));
|
|
54
|
-
return JSON.parse(readFileSync(join(here, "../package.json"), "utf8")).version;
|
|
55
|
-
} catch {
|
|
56
|
-
return "0.0.0-unknown";
|
|
57
|
-
}
|
|
58
|
-
})();
|
|
59
|
-
|
|
60
|
-
// src/engine.ts
|
|
61
|
-
import {
|
|
62
|
-
allProfiles,
|
|
63
|
-
tryGetProfile,
|
|
64
|
-
resolveModelAlias,
|
|
65
|
-
isArchetype,
|
|
66
|
-
LIBRARY_VERSION
|
|
67
|
-
} from "@warmdrift/kgauto-compiler";
|
|
68
|
-
var DESTINATION_SENTENCE = "This check ran locally against kgauto\u2019s bundled model catalog. Nothing was uploaded; no rows were written anywhere.";
|
|
69
|
-
var defaultResolve = (id) => {
|
|
70
|
-
const canonical = resolveModelAlias(id);
|
|
71
|
-
return { canonical, profile: tryGetProfile(id) };
|
|
72
|
-
};
|
|
73
|
-
function perCallCostUsd(p, inTok, outTok) {
|
|
74
|
-
return inTok / 1e6 * p.costInputPer1m + outTok / 1e6 * p.costOutputPer1m;
|
|
75
|
-
}
|
|
76
|
-
function runCheck(extract, opts = {}) {
|
|
77
|
-
const resolve = opts.resolve ?? defaultResolve;
|
|
78
|
-
const roster = opts.roster ?? allProfiles;
|
|
79
|
-
const findings = [];
|
|
80
|
-
let n = 0;
|
|
81
|
-
const add = (f) => {
|
|
82
|
-
findings.push({ id: `F${++n}`, ...f });
|
|
83
|
-
};
|
|
84
|
-
const unknownModels = /* @__PURE__ */ new Set();
|
|
85
|
-
for (const site of extract.callSites) {
|
|
86
|
-
const { canonical, profile } = resolve(site.model);
|
|
87
|
-
const at = [site.id];
|
|
88
|
-
if (canonical !== site.model) {
|
|
89
|
-
add({
|
|
90
|
-
severity: "yellow",
|
|
91
|
-
confidence: "certain",
|
|
92
|
-
title: `\`${site.model}\` is a provider compat alias for \`${canonical}\``,
|
|
93
|
-
detail: `The provider silently serves a different model than the id you configured. Aliases hide capability and price drift \u2014 the served model's context window and prices can differ by 10\u201350\xD7 from what the alias name suggests, with no error.`,
|
|
94
|
-
basis: "kgauto alias table (verified against provider docs)",
|
|
95
|
-
callSites: at
|
|
96
|
-
});
|
|
97
|
-
}
|
|
98
|
-
if (!profile) {
|
|
99
|
-
unknownModels.add(site.model);
|
|
100
|
-
add({
|
|
101
|
-
severity: "info",
|
|
102
|
-
confidence: "certain",
|
|
103
|
-
title: `\`${site.model}\` is not in kgauto's verified catalog`,
|
|
104
|
-
detail: `kgauto has no measured evidence for this model and will not fabricate a verdict on it. Checks below skip this site. (If it's a new release, the catalog gains models through doc-verified onboarding, never from training-data guesses.)`,
|
|
105
|
-
basis: "catalog lookup",
|
|
106
|
-
callSites: at
|
|
107
|
-
});
|
|
108
|
-
continue;
|
|
109
|
-
}
|
|
110
|
-
if (profile.status === "legacy") {
|
|
111
|
-
add({
|
|
112
|
-
severity: "yellow",
|
|
113
|
-
confidence: "certain",
|
|
114
|
-
title: `\`${canonical}\` is a legacy model`,
|
|
115
|
-
detail: `The provider lists it as superseded. Legacy models are the deprecation-risk tier: they can start 404ing for whole account cohorts with no code change on your side. A current-generation successor exists in the same family.`,
|
|
116
|
-
basis: `catalog status: legacy${profile.verifiedAgainstDocs ? ` (verified ${profile.verifiedAgainstDocs})` : ""}`,
|
|
117
|
-
callSites: at
|
|
118
|
-
});
|
|
119
|
-
} else if (profile.status === "preview") {
|
|
120
|
-
add({
|
|
121
|
-
severity: "info",
|
|
122
|
-
confidence: "certain",
|
|
123
|
-
title: `\`${canonical}\` is a preview model`,
|
|
124
|
-
detail: `Preview pricing and limits are not yet doc-verified; both can change at GA.`,
|
|
125
|
-
basis: "catalog status: preview",
|
|
126
|
-
callSites: at
|
|
127
|
-
});
|
|
128
|
-
}
|
|
129
|
-
const worstIn = site.maxInputTokens ?? site.estInputTokens;
|
|
130
|
-
if (worstIn !== void 0 && worstIn > profile.maxContextTokens) {
|
|
131
|
-
add({
|
|
132
|
-
severity: "red",
|
|
133
|
-
confidence: site.maxInputTokens !== void 0 ? "certain" : "likely",
|
|
134
|
-
title: `input can exceed \`${canonical}\`'s context window`,
|
|
135
|
-
detail: `Your ${site.maxInputTokens !== void 0 ? "worst-case" : "typical"} input estimate (${worstIn.toLocaleString()} tokens) exceeds the model's ${profile.maxContextTokens.toLocaleString()}-token window. Those calls fail outright or silently truncate, depending on the provider.`,
|
|
136
|
-
basis: `your estimate vs catalog maxContextTokens=${profile.maxContextTokens}`,
|
|
137
|
-
callSites: at
|
|
138
|
-
});
|
|
139
|
-
}
|
|
140
|
-
if (site.estOutputTokens !== void 0 && site.estOutputTokens > profile.maxOutputTokens) {
|
|
141
|
-
add({
|
|
142
|
-
severity: "red",
|
|
143
|
-
confidence: "likely",
|
|
144
|
-
title: `expected output exceeds \`${canonical}\`'s output cap`,
|
|
145
|
-
detail: `You expect ~${site.estOutputTokens.toLocaleString()} output tokens; the model caps at ${profile.maxOutputTokens.toLocaleString()}. Responses will be cut mid-generation \u2014 on JSON output this is the classic unparseable-tail failure.`,
|
|
146
|
-
basis: `your estimate vs catalog maxOutputTokens=${profile.maxOutputTokens}`,
|
|
147
|
-
callSites: at
|
|
148
|
-
});
|
|
149
|
-
}
|
|
150
|
-
if (site.toolCount !== void 0 && site.toolCount > profile.maxTools) {
|
|
151
|
-
add({
|
|
152
|
-
severity: "red",
|
|
153
|
-
confidence: "certain",
|
|
154
|
-
title: `${site.toolCount} tools exceeds \`${canonical}\`'s ceiling of ${profile.maxTools}`,
|
|
155
|
-
detail: `Calls with more tool definitions than the model supports are rejected or degrade hard.`,
|
|
156
|
-
basis: `extract toolCount vs catalog maxTools=${profile.maxTools}`,
|
|
157
|
-
callSites: at
|
|
158
|
-
});
|
|
159
|
-
}
|
|
160
|
-
if (site.usesParallelToolCalls && !profile.parallelToolCalls) {
|
|
161
|
-
add({
|
|
162
|
-
severity: "red",
|
|
163
|
-
confidence: "certain",
|
|
164
|
-
title: `\`${canonical}\` does not make parallel tool calls`,
|
|
165
|
-
detail: `This site relies on parallel tool use, but the model calls tools sequentially \u2014 measured at roughly 10\xD7 fewer operations per step on agentic workloads. The work still "succeeds", just slowly and shallowly, which is why this rarely shows up as an error.`,
|
|
166
|
-
basis: "catalog parallelToolCalls=false (measured, not vendor-claimed)",
|
|
167
|
-
callSites: at
|
|
168
|
-
});
|
|
169
|
-
}
|
|
170
|
-
if (site.expectsStructuredOutput && profile.structuredOutput === "none") {
|
|
171
|
-
add({
|
|
172
|
-
severity: "red",
|
|
173
|
-
confidence: "certain",
|
|
174
|
-
title: `\`${canonical}\` has no structured-output support, but this site parses JSON`,
|
|
175
|
-
detail: `Output-shape failures here land on your parser as intermittent exceptions. Models with native or grammar-constrained output move that failure from runtime to the request.`,
|
|
176
|
-
basis: "catalog structuredOutput=none",
|
|
177
|
-
callSites: at
|
|
178
|
-
});
|
|
179
|
-
}
|
|
180
|
-
const archetype = site.archetype && isArchetype(site.archetype) ? site.archetype : void 0;
|
|
181
|
-
for (const cliff of profile.cliffs) {
|
|
182
|
-
if (cliff.whenIntent && cliff.whenIntent !== archetype) continue;
|
|
183
|
-
const metricValue = cliff.metric === "input_tokens" ? worstIn : cliff.metric === "tool_count" ? site.toolCount : void 0;
|
|
184
|
-
if (metricValue === void 0 || metricValue < cliff.threshold) continue;
|
|
185
|
-
add({
|
|
186
|
-
severity: "yellow",
|
|
187
|
-
confidence: "likely",
|
|
188
|
-
title: `\`${canonical}\` has a known ${cliff.metric.replaceAll("_", " ")} cliff at ${cliff.threshold.toLocaleString()} \u2014 this site crosses it`,
|
|
189
|
-
detail: `Above this threshold${cliff.whenIntent ? ` on ${cliff.whenIntent} work` : ""}, quality degrades in a measured, repeatable way (compiler action when routed through kgauto: ${cliff.action.replaceAll("_", " ")}). Your estimate of ${metricValue.toLocaleString()} sits over the line.`,
|
|
190
|
-
basis: `catalog cliff rule vs your estimate; the same rule the kgauto compiler executes`,
|
|
191
|
-
callSites: at
|
|
192
|
-
});
|
|
193
|
-
}
|
|
194
|
-
if (site.retry === "none") {
|
|
195
|
-
add({
|
|
196
|
-
severity: "yellow",
|
|
197
|
-
confidence: "certain",
|
|
198
|
-
title: `no fallback path \u2014 a single ${profile.provider} incident takes this surface to raw errors`,
|
|
199
|
-
detail: `Provider outages are routine and undramatic: billing exhaustion, quota, silent deprecations. With no cross-provider fallback, every one of them reaches your users directly.`,
|
|
200
|
-
basis: "extract retry=none",
|
|
201
|
-
callSites: at
|
|
202
|
-
});
|
|
203
|
-
}
|
|
204
|
-
if (site.estInputTokens !== void 0 && site.estOutputTokens !== void 0 && site.estInputTokens + site.estOutputTokens > 0) {
|
|
205
|
-
const perCall = perCallCostUsd(profile, site.estInputTokens, site.estOutputTokens);
|
|
206
|
-
const monthly = site.estCallsPerMonth !== void 0 ? perCall * site.estCallsPerMonth : void 0;
|
|
207
|
-
const inputShare = site.estInputTokens / (site.estInputTokens + site.estOutputTokens);
|
|
208
|
-
add({
|
|
209
|
-
severity: "info",
|
|
210
|
-
confidence: "likely",
|
|
211
|
-
title: `~$${perCall.toFixed(4)}/call on \`${canonical}\`${monthly !== void 0 ? ` (~$${monthly.toFixed(2)}/month at your volume)` : ""}`,
|
|
212
|
-
detail: `From your token estimates and doc-verified prices ($${profile.costInputPer1m}/M in, $${profile.costOutputPer1m}/M out${profile.verifiedAgainstDocs ? `, verified ${profile.verifiedAgainstDocs}` : ""}).`,
|
|
213
|
-
basis: "arithmetic on your estimates \xD7 verified catalog prices",
|
|
214
|
-
callSites: at
|
|
215
|
-
});
|
|
216
|
-
if (inputShare > 0.85) {
|
|
217
|
-
add({
|
|
218
|
-
severity: "yellow",
|
|
219
|
-
confidence: "likely",
|
|
220
|
-
title: `${Math.round(inputShare * 100)}% of this site's tokens are input`,
|
|
221
|
-
detail: `Above ~85% input share, spend is dominated by context rather than answers \u2014 the single strongest signal of a leaking prompt. What's actually IN that context, and how much of it each call needed, is measurable, not guessable.`,
|
|
222
|
-
basis: "your token estimates; the >85% threshold is a measured portfolio heuristic",
|
|
223
|
-
callSites: at
|
|
224
|
-
});
|
|
225
|
-
}
|
|
226
|
-
if (archetype) {
|
|
227
|
-
const cheaper = roster().filter(
|
|
228
|
-
(p) => p.status === "current" && p.id !== canonical && perCallCostUsd(p, site.estInputTokens, site.estOutputTokens) < perCall * 0.5
|
|
229
|
-
).sort(
|
|
230
|
-
(a, b) => perCallCostUsd(a, site.estInputTokens, site.estOutputTokens) - perCallCostUsd(b, site.estInputTokens, site.estOutputTokens)
|
|
231
|
-
);
|
|
232
|
-
const cheapest = cheaper[0];
|
|
233
|
-
if (cheapest) {
|
|
234
|
-
add({
|
|
235
|
-
severity: "info",
|
|
236
|
-
confidence: "measurable-only",
|
|
237
|
-
title: `${cheaper.length} current model(s) price under half of \`${canonical}\` on ${archetype} work`,
|
|
238
|
-
detail: `Cheapest: \`${cheapest.id}\` at ~$${perCallCostUsd(cheapest, site.estInputTokens, site.estOutputTokens).toFixed(4)}/call. The price is arithmetic; whether any of them holds YOUR quality bar on ${archetype} work is exactly what a one-day observe run plus kgauto's judged eval measures. kgauto never promotes a model on price alone.`,
|
|
239
|
-
basis: "verified catalog prices; quality deliberately unclaimed",
|
|
240
|
-
callSites: at
|
|
241
|
-
});
|
|
242
|
-
}
|
|
243
|
-
}
|
|
244
|
-
}
|
|
245
|
-
}
|
|
246
|
-
add({
|
|
247
|
-
severity: "info",
|
|
248
|
-
confidence: "measurable-only",
|
|
249
|
-
title: "what a code read cannot tell you (and a one-day observe run can)",
|
|
250
|
-
detail: `Your actual failure rate per model (walkaway failures look like green health checks); your actual input-waste ratio per call; which call sites dominate spend; whether a cheaper model holds quality on your real traffic. These need one day of observed calls \u2014 observe-only, your keys, no routing changes.`,
|
|
251
|
-
basis: "the honest limit of static analysis",
|
|
252
|
-
callSites: extract.callSites.map((s) => s.id)
|
|
253
|
-
});
|
|
254
|
-
const merged = [];
|
|
255
|
-
const byKey = /* @__PURE__ */ new Map();
|
|
256
|
-
for (const f of findings) {
|
|
257
|
-
const key = `${f.severity}|${f.confidence}|${f.title}`;
|
|
258
|
-
const existing = byKey.get(key);
|
|
259
|
-
if (existing) {
|
|
260
|
-
for (const cs of f.callSites) {
|
|
261
|
-
if (!existing.callSites.includes(cs)) existing.callSites.push(cs);
|
|
262
|
-
}
|
|
263
|
-
} else {
|
|
264
|
-
const copy = { ...f, callSites: [...f.callSites] };
|
|
265
|
-
byKey.set(key, copy);
|
|
266
|
-
merged.push(copy);
|
|
267
|
-
}
|
|
268
|
-
}
|
|
269
|
-
return {
|
|
270
|
-
findings: merged,
|
|
271
|
-
card: renderCard(extract, merged, unknownModels),
|
|
272
|
-
catalogVersion: LIBRARY_VERSION
|
|
273
|
-
};
|
|
274
|
-
}
|
|
275
|
-
var TIER_ORDER = ["certain", "likely", "measurable-only"];
|
|
276
|
-
var TIER_HEADING = {
|
|
277
|
-
certain: "Certain \u2014 read from your code and kgauto\u2019s verified catalog",
|
|
278
|
-
likely: "Likely \u2014 priced from your estimates",
|
|
279
|
-
"measurable-only": "Only measurable \u2014 what the observe day answers"
|
|
280
|
-
};
|
|
281
|
-
var SEV_MARK = { red: "\u{1F534}", yellow: "\u{1F7E1}", info: "\u2139\uFE0F" };
|
|
282
|
-
function renderCard(extract, findings, unknownModels = /* @__PURE__ */ new Set()) {
|
|
283
|
-
const lines = [];
|
|
284
|
-
lines.push(`# kgauto check \u2014 ${extract.project.name}`);
|
|
285
|
-
lines.push("");
|
|
286
|
-
const models = [...new Set(extract.callSites.map((s) => s.model))];
|
|
287
|
-
lines.push(
|
|
288
|
-
`${extract.callSites.length} AI call site(s), ${models.length} model(s): ${models.map((m) => `\`${m}\``).join(", ")}` + (unknownModels.size > 0 ? ` (${unknownModels.size} unknown to the catalog)` : "")
|
|
289
|
-
);
|
|
290
|
-
for (const tier of TIER_ORDER) {
|
|
291
|
-
const group = findings.filter((f) => f.confidence === tier);
|
|
292
|
-
if (group.length === 0) continue;
|
|
293
|
-
lines.push("");
|
|
294
|
-
lines.push(`## ${TIER_HEADING[tier]}`);
|
|
295
|
-
lines.push("");
|
|
296
|
-
for (const f of group) {
|
|
297
|
-
lines.push(`- ${SEV_MARK[f.severity]} **${f.title}** \u2014 ${f.detail}`);
|
|
298
|
-
lines.push(` <sub>${f.callSites.join(", ")} \xB7 basis: ${f.basis}</sub>`);
|
|
299
|
-
}
|
|
300
|
-
}
|
|
301
|
-
lines.push("");
|
|
302
|
-
lines.push("---");
|
|
303
|
-
lines.push(`*${DESTINATION_SENTENCE}*`);
|
|
304
|
-
lines.push(`<sub>kgauto-check ${CHECK_VERSION} \xB7 catalog ${LIBRARY_VERSION}</sub>`);
|
|
305
|
-
return lines.join("\n");
|
|
306
|
-
}
|
|
307
|
-
|
|
308
|
-
// src/instructions.ts
|
|
309
|
-
var SERVER_INSTRUCTIONS = `You are talking to kgauto-check, a local, offline checker for AI call surfaces.
|
|
310
|
-
|
|
311
|
-
Flow: (1) call start_check to get extraction instructions, (2) read THIS project's code yourself and build the extract, (3) call run_check with it, (4) present the returned card to your human verbatim (it is grouped by confidence tier on purpose), (5) if they want measured numbers, call get_observe_day_plan.
|
|
312
|
-
|
|
313
|
-
Privacy contract, which you should relay to your human: the extract contains identifiers and counts only \u2014 never prompt text, never source code, never secrets. ${DESTINATION_SENTENCE}
|
|
314
|
-
|
|
315
|
-
get_model_facts answers one-off questions about any model id from the same verified catalog \u2014 it is what makes keeping this server installed useful between checks.`;
|
|
316
|
-
var START_CHECK_INSTRUCTIONS = `# kgauto check \u2014 extraction instructions (for you, the agent)
|
|
317
|
-
|
|
318
|
-
Your job: read this project's code and build an extract of its AI call surface. You have repo access; kgauto does not, by design.
|
|
319
|
-
|
|
320
|
-
## What to find
|
|
321
|
-
Scan for LLM API usage: provider SDK imports (anthropic, openai, @google/generative-ai / @google/genai, ai-sdk, litellm, langchain, ...), raw HTTPS calls to provider endpoints, and any internal wrapper around them. Each distinct place the project sends a prompt to a model is one call site.
|
|
322
|
-
|
|
323
|
-
## What to collect per call site (identifiers and counts ONLY)
|
|
324
|
-
- id: a stable label you choose, e.g. "api/summarize.ts:42"
|
|
325
|
-
- model: the model id exactly as configured
|
|
326
|
-
- modelSelection: hardcoded | config | env | dynamic
|
|
327
|
-
- archetype: your best guess at the work's intent \u2014 one of: ask, classify, critique, extract, generate, hunt, plan, summarize, transform
|
|
328
|
-
- estInputTokens / maxInputTokens / estOutputTokens: estimate from the code (what gets concatenated into the prompt; ~4 chars per token). Worst case matters \u2014 find the biggest thing that can flow in.
|
|
329
|
-
- toolCount, usesParallelToolCalls: from the tools array and how results are consumed
|
|
330
|
-
- expectsStructuredOutput: true if the response is JSON.parse'd or schema-validated
|
|
331
|
-
- streaming, retry (none | same-model | fallback-chain | unknown), fallbackModels
|
|
332
|
-
- estCallsPerMonth: only if genuinely inferable (cron cadence, obvious traffic); omit rather than invent
|
|
333
|
-
|
|
334
|
-
## Privacy contract (relay this to your human)
|
|
335
|
-
Do NOT put prompt text, source code, file contents, or secrets in the extract. Identifiers and counts only. ${DESTINATION_SENTENCE}
|
|
336
|
-
|
|
337
|
-
## Then
|
|
338
|
-
Write the extract as JSON:
|
|
339
|
-
{ "project": { "name": "...", "description": "..." }, "callSites": [ ...one object per call site... ] }
|
|
340
|
-
and run:
|
|
341
|
-
npx -y @warmdrift/kgauto-check --extract extract.json
|
|
342
|
-
Present the printed card verbatim \u2014 the confidence tiers (certain / likely / measurable-only) are the point: they tell your human exactly how much to trust each line, and the measurable-only section is what a one-day observe run would answer.
|
|
343
|
-
|
|
344
|
-
## Sharing (optional, explicit)
|
|
345
|
-
After presenting the card, you may OFFER your human: "want to send this card to kgauto so the catalog learns what real projects look like?" If \u2014 and only if \u2014 they say yes, re-run with --share. Never add --share without their explicit yes. It sends the scrubbed extract and the card, nothing else, and says exactly where they went.
|
|
346
|
-
|
|
347
|
-
## Honesty rules you inherit while presenting
|
|
348
|
-
- Do not soften "certain" findings or upgrade "likely" ones.
|
|
349
|
-
- If a model was unknown to the catalog, say so \u2014 kgauto refuses to fabricate verdicts, and so should you.
|
|
350
|
-
- Estimates you made are estimates: if a finding hinges on your token guess, say it can be firmed up by measuring.`;
|
|
351
|
-
|
|
352
|
-
// src/observe-plan.ts
|
|
353
|
-
var OBSERVE_DAY_PLAN = `# The one-day observe run (for you, the agent \u2014 decision stays with your human)
|
|
354
|
-
|
|
355
|
-
## What it is
|
|
356
|
-
One day of your project's normal traffic, observed. No routing changes: every call still goes to exactly the model your code chooses today. The observation records, per call: model, token counts in/out, cost at verified prices, latency, success/failure with a classified error code, and per-attempt rows when retries happen. Identifiers and counts \u2014 prompt text is never required.
|
|
357
|
-
|
|
358
|
-
## What it answers (the measurable-only tier of your check card)
|
|
359
|
-
- actual failure rate per model \u2014 including walkaway failures that green health checks miss
|
|
360
|
-
- actual input-waste ratio per call site (the >85% input-share signal, measured instead of estimated)
|
|
361
|
-
- which call sites dominate spend
|
|
362
|
-
- the evidence base for whether a cheaper model could hold quality here (kgauto's judged eval runs on real captured cases, never on price alone)
|
|
363
|
-
|
|
364
|
-
## The two honesty requirements (non-negotiable, relay both)
|
|
365
|
-
1. **Destination, named before anything runs.** Your human gets one plain sentence stating exactly where outcome rows land and what is in them, BEFORE the first row is written. "Nothing is uploaded" was true of the check because it writes nothing; the observe day writes rows, so it must say where.
|
|
366
|
-
2. **Observation never intervenes.** If your project uses fallback chains, observing must not pin a model and silently disable the chain \u2014 a forced pin is an intervention sold as observation. Rows are recorded per attempt, so the chain's real behaviour is what gets measured.
|
|
367
|
-
|
|
368
|
-
## How to proceed
|
|
369
|
-
The integration is small (the kgauto compiler's call/record path with your existing keys), but because rows now land somewhere, the wiring starts with that destination sentence and your human's explicit yes. Next step: have your human reply to whoever pointed them at kgauto \u2014 the observe day is set up per-project, together.`;
|
|
370
|
-
|
|
371
|
-
// src/model-facts.ts
|
|
372
|
-
import { tryGetProfile as tryGetProfile2, resolveModelAlias as resolveModelAlias2, LIBRARY_VERSION as LIBRARY_VERSION2 } from "@warmdrift/kgauto-compiler";
|
|
373
|
-
function renderModelFacts(model) {
|
|
374
|
-
const canonical = resolveModelAlias2(model);
|
|
375
|
-
const p = tryGetProfile2(model);
|
|
376
|
-
if (!p) {
|
|
377
|
-
return `\`${model}\` is not in kgauto's verified catalog (lib ${LIBRARY_VERSION2}). No measured evidence exists for it here, and kgauto does not fabricate verdicts. New models enter the catalog through doc-verified onboarding.`;
|
|
378
|
-
}
|
|
379
|
-
return [
|
|
380
|
-
`# ${canonical}${canonical !== model ? ` (you asked for \`${model}\`, a provider alias)` : ""}`,
|
|
381
|
-
"",
|
|
382
|
-
`- provider: ${p.provider} \xB7 status: **${p.status}**${p.verifiedAgainstDocs ? ` \xB7 capability data verified against provider docs ${p.verifiedAgainstDocs}` : ""}`,
|
|
383
|
-
`- price: $${p.costInputPer1m}/M input \xB7 $${p.costOutputPer1m}/M output`,
|
|
384
|
-
`- limits: ${p.maxContextTokens.toLocaleString()} context \xB7 ${p.maxOutputTokens.toLocaleString()} max output \xB7 ${p.maxTools} tools`,
|
|
385
|
-
`- capabilities: parallel tool calls ${p.parallelToolCalls ? "yes" : "NO (sequential \u2014 ~10\xD7 fewer ops/step on agentic work)"} \xB7 structured output: ${p.structuredOutput} \xB7 streaming: ${p.streaming ? "yes" : "no"}`,
|
|
386
|
-
p.cliffs.length > 0 ? `- known cliffs (executable in the kgauto compiler):
|
|
387
|
-
${p.cliffs.map(
|
|
388
|
-
(c) => ` - ${c.metric.replaceAll("_", " ")} \u2265 ${c.threshold.toLocaleString()}${c.whenIntent ? ` on ${c.whenIntent}` : ""} \u2192 ${c.action.replaceAll("_", " ")}`
|
|
389
|
-
).join("\n")}` : p.probedLimits?.length ? "- known cliffs: none recorded \u2014 and see the probed limits below, which is why that is a finding rather than a blank" : "- known cliffs: none recorded (nothing measured here yet \u2014 kgauto does not invent one)",
|
|
390
|
-
// alpha.95 — measured NEGATIVES. Without these, a frontier model reads
|
|
391
|
-
// "cliffs: none recorded", which a stranger correctly hears as "you know
|
|
392
|
-
// nothing about this one" — and it was the honest reading, because every
|
|
393
|
-
// probe result landed brain-only while this lookup runs offline against
|
|
394
|
-
// the bundle. A probed limit is the one claim here that a general-purpose
|
|
395
|
-
// model cannot produce: it required running the call.
|
|
396
|
-
...p.probedLimits?.length ? [
|
|
397
|
-
`- probed limits (we ran the call; ${p.probedLimits.length === 1 ? "this is a measurement" : "these are measurements"}, not a spec sheet):
|
|
398
|
-
${p.probedLimits.map(
|
|
399
|
-
(l) => ` - ${l.metric.replaceAll("_", " ")} @ ${l.testedAt.toLocaleString()} \u2192 **${l.outcome}**. ${l.claim}${l.probeRunId ? ` (run ${l.probeRunId})` : ""}`
|
|
400
|
-
).join("\n")}`
|
|
401
|
-
] : [],
|
|
402
|
-
p.notes ? `- notes: ${p.notes}` : "",
|
|
403
|
-
"",
|
|
404
|
-
`_Catalog: @warmdrift/kgauto-compiler ${LIBRARY_VERSION2}, bundled locally \u2014 this lookup made no network call._`
|
|
405
|
-
].filter(Boolean).join("\n");
|
|
406
|
-
}
|
|
407
|
-
|
|
408
|
-
// src/server.ts
|
|
409
|
-
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
410
|
-
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
411
|
-
import { z as z2 } from "zod";
|
|
412
|
-
var text = (t) => ({ content: [{ type: "text", text: t }] });
|
|
413
|
-
function buildServer() {
|
|
414
|
-
const server = new McpServer(
|
|
415
|
-
{ name: "kgauto-check", version: CHECK_VERSION },
|
|
416
|
-
{ instructions: SERVER_INSTRUCTIONS }
|
|
417
|
-
);
|
|
418
|
-
server.registerTool(
|
|
419
|
-
"start_check",
|
|
420
|
-
{
|
|
421
|
-
title: "Start a kgauto check",
|
|
422
|
-
description: "Call this FIRST. Returns extraction instructions: what to read from this project and how to build the extract for run_check. The extract carries identifiers and counts only \u2014 never prompt text, code, or secrets.",
|
|
423
|
-
inputSchema: {}
|
|
424
|
-
},
|
|
425
|
-
async () => text(START_CHECK_INSTRUCTIONS)
|
|
426
|
-
);
|
|
427
|
-
server.registerTool(
|
|
428
|
-
"run_check",
|
|
429
|
-
{
|
|
430
|
-
title: "Run the check",
|
|
431
|
-
description: "Runs the deterministic kgauto check on your extract, fully offline against the bundled verified model catalog. Returns a findings card grouped by confidence tier (certain / likely / measurable-only) \u2014 present it to your human verbatim.",
|
|
432
|
-
inputSchema: { extract: ExtractSchema }
|
|
433
|
-
},
|
|
434
|
-
async ({ extract }) => {
|
|
435
|
-
const result = runCheck(extract);
|
|
436
|
-
return text(result.card);
|
|
437
|
-
}
|
|
438
|
-
);
|
|
439
|
-
server.registerTool(
|
|
440
|
-
"get_model_facts",
|
|
441
|
-
{
|
|
442
|
-
title: "Model facts",
|
|
443
|
-
description: "What kgauto's verified catalog knows about one model id: lifecycle status, doc-verified prices and limits, capability flags, known quality cliffs. Honest miss when the model is unknown \u2014 no fabricated verdicts.",
|
|
444
|
-
inputSchema: { model: z2.string().min(1) }
|
|
445
|
-
},
|
|
446
|
-
async ({ model }) => text(renderModelFacts(model))
|
|
447
|
-
);
|
|
448
|
-
server.registerTool(
|
|
449
|
-
"get_observe_day_plan",
|
|
450
|
-
{
|
|
451
|
-
title: "The one-day observe run",
|
|
452
|
-
description: "Rung 2: what a one-day, observe-only run on this project's real traffic would measure, and the two honesty requirements any setup must carry (named row destination; observation never intervenes). Ends in a human decision by design.",
|
|
453
|
-
inputSchema: {}
|
|
454
|
-
},
|
|
455
|
-
async () => text(OBSERVE_DAY_PLAN)
|
|
456
|
-
);
|
|
457
|
-
return server;
|
|
458
|
-
}
|
|
459
|
-
async function startStdio() {
|
|
460
|
-
const server = buildServer();
|
|
461
|
-
await server.connect(new StdioServerTransport());
|
|
462
|
-
}
|
|
463
|
-
|
|
464
|
-
export {
|
|
465
|
-
CallSiteSchema,
|
|
466
|
-
ExtractSchema,
|
|
467
|
-
CHECK_VERSION,
|
|
468
|
-
DESTINATION_SENTENCE,
|
|
469
|
-
runCheck,
|
|
470
|
-
renderCard,
|
|
471
|
-
SERVER_INSTRUCTIONS,
|
|
472
|
-
START_CHECK_INSTRUCTIONS,
|
|
473
|
-
OBSERVE_DAY_PLAN,
|
|
474
|
-
renderModelFacts,
|
|
475
|
-
buildServer,
|
|
476
|
-
startStdio
|
|
477
|
-
};
|