modelrig 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +125 -0
- package/README.md +54 -4
- package/dist/adapters/anthropic.d.ts.map +1 -1
- package/dist/adapters/anthropic.js +36 -7
- package/dist/adapters/attempt-meta-map.d.ts +63 -0
- package/dist/adapters/attempt-meta-map.d.ts.map +1 -0
- package/dist/adapters/attempt-meta-map.js +154 -0
- package/dist/adapters/deepseek.d.ts.map +1 -1
- package/dist/adapters/deepseek.js +25 -2
- package/dist/adapters/gemini.d.ts.map +1 -1
- package/dist/adapters/gemini.js +29 -2
- package/dist/adapters/grok.d.ts.map +1 -1
- package/dist/adapters/grok.js +25 -2
- package/dist/adapters/openai-compat.d.ts.map +1 -1
- package/dist/adapters/openai-compat.js +25 -2
- package/dist/adapters/openai-stream.d.ts +2 -0
- package/dist/adapters/openai-stream.d.ts.map +1 -1
- package/dist/adapters/openai-stream.js +23 -3
- package/dist/adapters/openai.d.ts.map +1 -1
- package/dist/adapters/openai.js +36 -6
- package/dist/artifacts/artifact.d.ts.map +1 -1
- package/dist/artifacts/artifact.js +24 -5
- package/dist/artifacts/classification.d.ts +27 -0
- package/dist/artifacts/classification.d.ts.map +1 -0
- package/dist/artifacts/classification.js +38 -0
- package/dist/artifacts/index.d.ts +3 -1
- package/dist/artifacts/index.d.ts.map +1 -1
- package/dist/artifacts/index.js +10 -1
- package/dist/artifacts/run-context.d.ts +25 -3
- package/dist/artifacts/run-context.d.ts.map +1 -1
- package/dist/artifacts/run-context.js +61 -0
- package/dist/artifacts/scrub.d.ts +26 -0
- package/dist/artifacts/scrub.d.ts.map +1 -1
- package/dist/artifacts/scrub.js +53 -0
- package/dist/artifacts/step.d.ts +9 -0
- package/dist/artifacts/step.d.ts.map +1 -1
- package/dist/artifacts/step.js +9 -0
- package/dist/bakeoff/engine.d.ts +46 -4
- package/dist/bakeoff/engine.d.ts.map +1 -1
- package/dist/bakeoff/engine.js +269 -19
- package/dist/bakeoff/eval-cases.d.ts +5 -0
- package/dist/bakeoff/eval-cases.d.ts.map +1 -1
- package/dist/bakeoff/eval-cases.js +3 -0
- package/dist/bakeoff/judge.d.ts +39 -0
- package/dist/bakeoff/judge.d.ts.map +1 -0
- package/dist/bakeoff/judge.js +102 -0
- package/dist/bakeoff/quality.d.ts +44 -0
- package/dist/bakeoff/quality.d.ts.map +1 -0
- package/dist/bakeoff/quality.js +126 -0
- package/dist/bakeoff/reference.d.ts +34 -0
- package/dist/bakeoff/reference.d.ts.map +1 -0
- package/dist/bakeoff/reference.js +68 -0
- package/dist/bakeoff/replay.d.ts +25 -0
- package/dist/bakeoff/replay.d.ts.map +1 -1
- package/dist/bakeoff/replay.js +48 -1
- package/dist/bakeoff/report.d.ts +8 -1
- package/dist/bakeoff/report.d.ts.map +1 -1
- package/dist/bakeoff/report.js +83 -12
- package/dist/bakeoff/samples.d.ts +35 -2
- package/dist/bakeoff/samples.d.ts.map +1 -1
- package/dist/bakeoff/samples.js +67 -6
- package/dist/bakeoff/stats.d.ts +17 -1
- package/dist/bakeoff/stats.d.ts.map +1 -1
- package/dist/bakeoff/stats.js +67 -0
- package/dist/bakeoff/types.d.ts +76 -5
- package/dist/bakeoff/types.d.ts.map +1 -1
- package/dist/bakeoff/types.js +4 -3
- package/dist/cli-query.d.ts +14 -0
- package/dist/cli-query.d.ts.map +1 -0
- package/dist/cli-query.js +59 -0
- package/dist/cli.d.ts +2 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +26 -23
- package/dist/config.d.ts +6 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +14 -0
- package/dist/detect/detector.d.ts +82 -0
- package/dist/detect/detector.d.ts.map +1 -0
- package/dist/detect/detector.js +23 -0
- package/dist/index.d.ts +19 -14
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +38 -11
- package/dist/query/analytics.d.ts +84 -0
- package/dist/query/analytics.d.ts.map +1 -0
- package/dist/query/analytics.js +280 -0
- package/dist/query/columns.d.ts +59 -0
- package/dist/query/columns.d.ts.map +1 -0
- package/dist/query/columns.js +89 -0
- package/dist/query/grammar.d.ts +74 -7
- package/dist/query/grammar.d.ts.map +1 -1
- package/dist/query/grammar.js +179 -71
- package/dist/query/saved.d.ts +26 -0
- package/dist/query/saved.d.ts.map +1 -0
- package/dist/query/saved.js +51 -0
- package/dist/routes/loader.js +15 -10
- package/dist/routes/objective-loader.d.ts +35 -0
- package/dist/routes/objective-loader.d.ts.map +1 -0
- package/dist/routes/objective-loader.js +286 -0
- package/dist/run/accept.d.ts +43 -0
- package/dist/run/accept.d.ts.map +1 -0
- package/dist/run/accept.js +65 -0
- package/dist/run/cost.d.ts +13 -0
- package/dist/run/cost.d.ts.map +1 -1
- package/dist/run/cost.js +27 -0
- package/dist/run/dispatch.d.ts +24 -1
- package/dist/run/dispatch.d.ts.map +1 -1
- package/dist/run/dispatch.js +19 -0
- package/dist/run/ladder-key.d.ts +4 -0
- package/dist/run/ladder-key.d.ts.map +1 -1
- package/dist/run/ladder-key.js +3 -0
- package/dist/run/quality-gate.d.ts +39 -0
- package/dist/run/quality-gate.d.ts.map +1 -0
- package/dist/run/quality-gate.js +70 -0
- package/dist/run/raw.d.ts.map +1 -1
- package/dist/run/raw.js +22 -3
- package/dist/run/record-attempt.d.ts.map +1 -1
- package/dist/run/record-attempt.js +22 -1
- package/dist/run/routing-docs.d.ts.map +1 -1
- package/dist/run/routing-docs.js +11 -4
- package/dist/run/run.d.ts.map +1 -1
- package/dist/run/run.js +46 -39
- package/dist/settings/index.d.ts +1 -1
- package/dist/settings/index.d.ts.map +1 -1
- package/dist/settings/index.js +2 -1
- package/dist/settings/task-settings.d.ts +10 -0
- package/dist/settings/task-settings.d.ts.map +1 -1
- package/dist/settings/task-settings.js +17 -0
- package/dist/telemetry/attempt-meta.d.ts +178 -0
- package/dist/telemetry/attempt-meta.d.ts.map +1 -0
- package/dist/telemetry/attempt-meta.js +319 -0
- package/dist/telemetry/exporter.d.ts.map +1 -1
- package/dist/telemetry/exporter.js +6 -0
- package/dist/telemetry/ingest-protocol.d.ts +11 -0
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
- package/dist/telemetry/ingest-protocol.js +3 -0
- package/dist/telemetry/otel-map.d.ts +10 -0
- package/dist/telemetry/otel-map.d.ts.map +1 -1
- package/dist/telemetry/otel-map.js +64 -0
- package/dist/telemetry/store-migrations.d.ts.map +1 -1
- package/dist/telemetry/store-migrations.js +15 -0
- package/dist/telemetry/store.d.ts.map +1 -1
- package/dist/telemetry/store.js +25 -7
- package/dist/types/failures-teaching.d.ts.map +1 -1
- package/dist/types/failures-teaching.js +20 -0
- package/dist/types/failures.d.ts +1 -1
- package/dist/types/failures.d.ts.map +1 -1
- package/dist/types/failures.js +1 -0
- package/dist/types/provider.d.ts +9 -0
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/quality.d.ts +41 -0
- package/dist/types/quality.d.ts.map +1 -0
- package/dist/types/quality.js +12 -0
- package/dist/types/route.d.ts +89 -0
- package/dist/types/route.d.ts.map +1 -1
- package/dist/types/telemetry.d.ts +13 -0
- package/dist/types/telemetry.d.ts.map +1 -1
- package/dist/validate-cli.d.ts.map +1 -1
- package/dist/validate-cli.js +10 -0
- package/package.json +1 -1
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* The quality-gate seam (Q1, quality-gate spec §4.1). Extracted from run.ts —
|
|
4
|
+
* which sits at the 700-LOC cap — so the run loop gets ONE compact call site
|
|
5
|
+
* between `outcome.valid` and the accept branch.
|
|
6
|
+
*
|
|
7
|
+
* The gate evaluates a builder predicate on a schema-VALID output. It handles the
|
|
8
|
+
* awkward bits so the loop does not: awaiting an async verdict, normalizing an
|
|
9
|
+
* optional score/reason, and rejecting fail-closed when the predicate THROWS
|
|
10
|
+
* (spec §4.1 step 4, flag F2 — reject + warn ONCE per process). The loop keeps
|
|
11
|
+
* the control flow: a passing verdict takes the existing accept path (tagged),
|
|
12
|
+
* a failing one records `quality_rejected` and falls through on its own budget,
|
|
13
|
+
* never touching the repair rung.
|
|
14
|
+
*/
|
|
15
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
|
+
exports.resetQualityGateWarningForTest = resetQualityGateWarningForTest;
|
|
17
|
+
exports.evaluateQualityGate = evaluateQualityGate;
|
|
18
|
+
exports.qualityTags = qualityTags;
|
|
19
|
+
exports.qualityRejectedMessage = qualityRejectedMessage;
|
|
20
|
+
/** One warning per process when a predicate throws (F2). Module-level so it
|
|
21
|
+
* latches across runs; `resetQualityGateWarningForTest` clears it under test. */
|
|
22
|
+
let warnedOnThrow = false;
|
|
23
|
+
/** Test-only: reset the once-per-process throw-warning latch so a test can
|
|
24
|
+
* assert the single-warning behavior deterministically. */
|
|
25
|
+
function resetQualityGateWarningForTest() {
|
|
26
|
+
warnedOnThrow = false;
|
|
27
|
+
}
|
|
28
|
+
function finiteOrNull(n) {
|
|
29
|
+
return typeof n === "number" && Number.isFinite(n) ? n : null;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Evaluate a quality gate on a schema-valid output. Awaits an async verdict; a
|
|
33
|
+
* throwing predicate rejects fail-closed with a `qualityGate threw: …` reason
|
|
34
|
+
* and warns exactly once per process.
|
|
35
|
+
*/
|
|
36
|
+
async function evaluateQualityGate(gate, output, meta) {
|
|
37
|
+
try {
|
|
38
|
+
const verdict = await gate(output, meta);
|
|
39
|
+
return { pass: verdict.ok === true, score: finiteOrNull(verdict.score), reason: verdict.reason ?? null };
|
|
40
|
+
}
|
|
41
|
+
catch (err) {
|
|
42
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
43
|
+
if (!warnedOnThrow) {
|
|
44
|
+
warnedOnThrow = true;
|
|
45
|
+
console.warn(`[modelrig] qualityGate threw — treating the output as REJECTED (fail-closed): ${message}. ` +
|
|
46
|
+
"A predicate must return { ok } rather than throw; this warning fires once per process.");
|
|
47
|
+
}
|
|
48
|
+
return { pass: false, score: null, reason: `qualityGate threw: ${message}` };
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
/** Render a score for a tag / message — a plain decimal string, or "" when null. */
|
|
52
|
+
function scoreText(score) {
|
|
53
|
+
return score === null ? "" : String(score);
|
|
54
|
+
}
|
|
55
|
+
/** SDK-owned telemetry tags for the attempt row (spec §2): `quality` = pass|fail
|
|
56
|
+
* and, when a score exists, `quality_score` as a decimal string. Never carries
|
|
57
|
+
* `rung` — the recorder owns that and stamps it last. */
|
|
58
|
+
function qualityTags(outcome) {
|
|
59
|
+
const tags = { quality: outcome.pass ? "pass" : "fail" };
|
|
60
|
+
if (outcome.score !== null)
|
|
61
|
+
tags.quality_score = scoreText(outcome.score);
|
|
62
|
+
return tags;
|
|
63
|
+
}
|
|
64
|
+
/** The `quality_rejected` failure message — carries the reason and score so a
|
|
65
|
+
* stuck engineer (and the exhausted-ladder error) sees WHY it was rejected. */
|
|
66
|
+
function qualityRejectedMessage(outcome) {
|
|
67
|
+
const scorePart = outcome.score === null ? "" : ` (score ${scoreText(outcome.score)})`;
|
|
68
|
+
const reasonPart = outcome.reason === null ? "" : `: ${outcome.reason}`;
|
|
69
|
+
return `output met the schema but not the quality bar${scorePart}${reasonPart}`;
|
|
70
|
+
}
|
package/dist/run/raw.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"raw.d.ts","sourceRoot":"","sources":["../../src/run/raw.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;
|
|
1
|
+
{"version":3,"file":"raw.d.ts","sourceRoot":"","sources":["../../src/run/raw.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AAwBH,OAAO,EAAiC,KAAK,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACrF,OAAO,KAAK,EACV,cAAc,EAGd,YAAY,EAGZ,UAAU,EACV,eAAe,EACf,WAAW,EACX,UAAU,EACV,cAAc,EACf,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAW,SAAS,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAGlE,OAAO,KAAK,EAAkB,eAAe,EAAE,MAAM,UAAU,CAAC;AAChE,OAAO,KAAK,EAA+B,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAEtF;;;sCAGsC;AACtC,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;iFAC6E;IAC7E,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC;IACxB;;;;;iFAK6E;IAC7E,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAClC;;;6EAGyE;IACzE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAC3C,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC;;;;;;;0BAOsB;IACtB,QAAQ,CAAC,KAAK,CAAC,EAAE;QAAE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACxE;;;;;;;iDAO6C;IAC7C,QAAQ,CAAC,SAAS,CAAC,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;KAAE,CAAC;IACjD;;;;mFAI+E;IAC/E,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;IACnC;;;;qFAIiF;IACjF,QAAQ,CAAC,SAAS,CAAC,EAAE,eAAe,CAAC;IACrC;;;;;;;;sEAQkE;IAClE,QAAQ,CAAC,cAAc,CAAC,EAAE,aAAa,CAAC;CACzC;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,SAAS,EAAE,cAAc,CAAC;IACnC;;+EAE2E;IAC3E,QAAQ,CAAC,sBAAsB,CAAC,EAAE,MAAM,CAAC;IACzC;;sEAEkE;IAClE,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;CACxC;AAiED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,WAAW,GAAG,IAAI,CAiDhE;AAED,4EAA4E;AAC5E,wBAAgB,eAAe,CAAC,SAAS,EAAE,YAAY,EAAE,KAAK,EAAE,WAAW,GAAG,cAAc,CAsC3F;AA2PD;wDACwD;AACxD,wBAAsB,MAAM,CAAC,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC,CAYrF;AAcD;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,GAAG,eAAe,CAiDlF"}
|
package/dist/run/raw.js
CHANGED
|
@@ -61,6 +61,7 @@ const raw_support_1 = require("./raw-support");
|
|
|
61
61
|
const failures_1 = require("../types/failures");
|
|
62
62
|
const run_context_1 = require("../artifacts/run-context");
|
|
63
63
|
const manifest_1 = require("../routes/manifest");
|
|
64
|
+
const attempt_meta_1 = require("../telemetry/attempt-meta");
|
|
64
65
|
/**
|
|
65
66
|
* R1 — resolve the effective raw-lane timeout. An explicit `timeoutMs` always
|
|
66
67
|
* wins; absent, the fallback is TIER-AWARE (`RAW_TIMEOUT_DEFAULT_MS`), because a
|
|
@@ -267,6 +268,7 @@ function recordRawRow(telemetry, p) {
|
|
|
267
268
|
callId: p.callId,
|
|
268
269
|
attemptSeq: 1,
|
|
269
270
|
candidateIndex: 0,
|
|
271
|
+
meta: p.meta,
|
|
270
272
|
};
|
|
271
273
|
telemetry.record(row);
|
|
272
274
|
}
|
|
@@ -279,8 +281,9 @@ function recordRawRow(telemetry, p) {
|
|
|
279
281
|
* step is ever recorded; the dispatch is byte-identical to pre-G1.
|
|
280
282
|
*/
|
|
281
283
|
function prepareRawStep(deps, input) {
|
|
282
|
-
(0, manifest_1.warnMissingRequiredDimensions)(deps.manifest ?? null, input.tags ?? {});
|
|
283
284
|
const runCtx = (0, run_context_1.getActiveRunContext)();
|
|
285
|
+
// Warn on the EFFECTIVE tags: an auto-stamped run_id (RC-1) is not missing.
|
|
286
|
+
(0, manifest_1.warnMissingRequiredDimensions)(deps.manifest ?? null, (0, run_context_1.withRunIdTag)(input.tags ?? {}, runCtx));
|
|
284
287
|
if (runCtx === null)
|
|
285
288
|
return null;
|
|
286
289
|
const rawModelId = `${input.provider}/${input.model}`;
|
|
@@ -306,8 +309,22 @@ function finalizeRaw(deps, input, result, latencyMs, step) {
|
|
|
306
309
|
const callId = (0, node_crypto_1.randomUUID)();
|
|
307
310
|
// CE-2 F2: the declared tier is reported truthfully on the row AND the meta.
|
|
308
311
|
const requestedTier = input.serviceTier ?? "standard";
|
|
309
|
-
//
|
|
310
|
-
|
|
312
|
+
// RC-1: `run_id` from the ambient run (caller's value wins); lane is SDK-owned
|
|
313
|
+
// and cannot be overridden by a caller-supplied tag.
|
|
314
|
+
const tags = { ...(0, run_context_1.withRunIdTag)(input.tags ?? {}, step?.runCtx ?? null), lane: "raw" };
|
|
315
|
+
// WI-1 (spec §2, §6.3 lane parity): the attempt envelope, built by the SAME
|
|
316
|
+
// producer + rendered-request canonicalization the routed lane uses. The raw
|
|
317
|
+
// lane never sends an output schema (unvalidated by design), so schema = null.
|
|
318
|
+
const declaredToolNames = input.tools?.map((t) => t.function.name);
|
|
319
|
+
const rawMeta = (usage) => (0, attempt_meta_1.summarizeAttemptMeta)({
|
|
320
|
+
...(result.meta !== undefined ? { adapterMeta: result.meta } : {}),
|
|
321
|
+
renderedRequest: (0, attempt_meta_1.renderRequestForHash)({ system: input.systemPrompt, user: input.userPrompt, schema: null }),
|
|
322
|
+
outputText: result.ok ? result.text : null,
|
|
323
|
+
...(declaredToolNames !== undefined ? { declaredToolNames } : {}),
|
|
324
|
+
provider: input.provider,
|
|
325
|
+
model: input.model,
|
|
326
|
+
...(usage !== undefined ? { usage } : {}),
|
|
327
|
+
});
|
|
311
328
|
// G1: mirror this raw inference into a ground-truth step (parity with the
|
|
312
329
|
// routed lane's record-attempt). task = tags.step ?? provider/model (spec F1).
|
|
313
330
|
// Fail-open is handled inside emitGroundTruthStep (buffer errors are swallowed).
|
|
@@ -345,6 +362,7 @@ function finalizeRaw(deps, input, result, latencyMs, step) {
|
|
|
345
362
|
tags,
|
|
346
363
|
pricingMissing,
|
|
347
364
|
callId,
|
|
365
|
+
meta: rawMeta(result.usage),
|
|
348
366
|
...(input.cache !== undefined ? { cacheKey: input.cache.key } : {}),
|
|
349
367
|
});
|
|
350
368
|
emitStep(failId, result.failure.class);
|
|
@@ -374,6 +392,7 @@ function finalizeRaw(deps, input, result, latencyMs, step) {
|
|
|
374
392
|
tags,
|
|
375
393
|
pricingMissing,
|
|
376
394
|
callId,
|
|
395
|
+
meta: rawMeta(result.usage),
|
|
377
396
|
...(input.cache !== undefined ? { cacheKey: input.cache.key } : {}),
|
|
378
397
|
});
|
|
379
398
|
emitStep(id, null);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"record-attempt.d.ts","sourceRoot":"","sources":["../../src/run/record-attempt.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAE3C,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACvE,OAAO,KAAK,EAAgB,aAAa,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAGtF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"record-attempt.d.ts","sourceRoot":"","sources":["../../src/run/record-attempt.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAE3C,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACvE,OAAO,KAAK,EAAgB,aAAa,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAGtF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAGpD,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,SAAS,EAAE,cAAc,CAAC;IACnC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B;;;gDAG4C;IAC5C,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAC;IACjC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,kFAAkF;IAClF,QAAQ,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACzC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC;;qFAEiF;IACjF,QAAQ,CAAC,KAAK,CAAC,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC;IACrC;;;wEAGoE;IACpE,QAAQ,CAAC,kBAAkB,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CAClD;AAED;;;;;GAKG;AACH,wBAAgB,MAAM,CACpB,OAAO,EAAE,gBAAgB,EACzB,WAAW,EAAE,OAAO,EACpB,cAAc,EAAE,MAAM,GAAG,IAAI,EAC7B,iBAAiB,EAAE,GAAG,CAAC,MAAM,CAAC,GAC7B,aAAa,CAUf;AAED;;yCAEyC;AACzC,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,iBAAiB,GAAG,CAAC,OAAO,EAAE,gBAAgB,KAAK,MAAM,CAmHlG"}
|
|
@@ -17,6 +17,7 @@ const node_crypto_1 = require("node:crypto");
|
|
|
17
17
|
const failures_1 = require("../types/failures");
|
|
18
18
|
const cache_stamp_1 = require("./cache-stamp");
|
|
19
19
|
const cache_handle_1 = require("./cache-handle");
|
|
20
|
+
const attempt_meta_1 = require("../telemetry/attempt-meta");
|
|
20
21
|
/**
|
|
21
22
|
* Classify a recorded row's ladder rung (R1 review, F1). `primaryDispatched`
|
|
22
23
|
* carries the candidate indices already given a primary dispatch on this call —
|
|
@@ -91,6 +92,23 @@ function createRecordAttempt(deps) {
|
|
|
91
92
|
// pre-dispatch terminal failure, then primary-vs-retry by whether this
|
|
92
93
|
// candidate index was already dispatched on this call.
|
|
93
94
|
const rung = rungOf(attempt, preDispatch, ladderKey.candidateIndex, primaryDispatched);
|
|
95
|
+
// WI-1 (spec §2): the metadata-safe attempt envelope. The single producer
|
|
96
|
+
// hashes the rendered request/output, computes cost components + context
|
|
97
|
+
// bucket, and enforces the classification rule; it never throws (a failure
|
|
98
|
+
// returns null — the row is byte-identical). Built from the adapter's native
|
|
99
|
+
// meta (metaInputs) plus THIS row's provider/model/usage.
|
|
100
|
+
const meta = (0, attempt_meta_1.summarizeAttemptMeta)({
|
|
101
|
+
...(attempt.metaInputs ?? {}),
|
|
102
|
+
provider: attempt.candidate.provider,
|
|
103
|
+
model: attempt.candidate.model,
|
|
104
|
+
usage: {
|
|
105
|
+
tokensIn: attempt.tokensIn,
|
|
106
|
+
tokensOut: attempt.tokensOut,
|
|
107
|
+
tokensCached: attempt.tokensCached,
|
|
108
|
+
...(attempt.tokensCacheWrite !== undefined ? { tokensCacheWrite: attempt.tokensCacheWrite } : {}),
|
|
109
|
+
...(attempt.tokensCacheWrite1h !== undefined ? { tokensCacheWrite1h: attempt.tokensCacheWrite1h } : {}),
|
|
110
|
+
},
|
|
111
|
+
});
|
|
94
112
|
const row = {
|
|
95
113
|
id: (0, node_crypto_1.randomUUID)(),
|
|
96
114
|
ts: new Date().toISOString(),
|
|
@@ -113,7 +131,9 @@ function createRecordAttempt(deps) {
|
|
|
113
131
|
ttfbMs: attempt.ttfbMs ?? null,
|
|
114
132
|
failureClass: attempt.failure?.class ?? null,
|
|
115
133
|
attemptsJson: JSON.stringify(ledger.snapshot()),
|
|
116
|
-
|
|
134
|
+
// `rung` is stamped LAST so a caller's extraTags (Q1 quality tags) can
|
|
135
|
+
// never override the recorder-owned rung.
|
|
136
|
+
tagsJson: JSON.stringify({ ...effectiveTags, ...attempt.extraTags, rung }),
|
|
117
137
|
fingerprintCell: route.fingerprintCell,
|
|
118
138
|
servedVariant,
|
|
119
139
|
repairedBy: attempt.repairedBy ?? null,
|
|
@@ -121,6 +141,7 @@ function createRecordAttempt(deps) {
|
|
|
121
141
|
callId: ladderKey.callId,
|
|
122
142
|
attemptSeq: ladderKey.attemptSeq,
|
|
123
143
|
candidateIndex: ladderKey.candidateIndex,
|
|
144
|
+
meta,
|
|
124
145
|
};
|
|
125
146
|
telemetry.record(row);
|
|
126
147
|
// Mirror this inference into a ground-truth step attempt (fail-open — the
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"routing-docs.d.ts","sourceRoot":"","sources":["../../src/run/routing-docs.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAItD,MAAM,MAAM,cAAc,GAAG,WAAW,GAAG,UAAU,GAAG,QAAQ,GAAG,WAAW,CAAC;AAE/E,MAAM,WAAW,gBAAgB;IAC/B,sEAAsE;IACtE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,6DAA6D;IAC7D,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,yEAAyE;IACzE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,aAAa;IAC5B,6EAA6E;IAC7E,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,kBAAkB;IAClB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,uDAAuD;IACvD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,kDAAkD;IAClD,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,CAAC,EAAE,gBAAgB,GAAG;QAAE,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAC;CAC3E;AAED,yCAAyC;AACzC,eAAO,MAAM,qBAAqB,EAAE,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,MAAM,CAAC,CAK1E,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,gBAAgB,EAAE,SAAS,aAAa,
|
|
1
|
+
{"version":3,"file":"routing-docs.d.ts","sourceRoot":"","sources":["../../src/run/routing-docs.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAItD,MAAM,MAAM,cAAc,GAAG,WAAW,GAAG,UAAU,GAAG,QAAQ,GAAG,WAAW,CAAC;AAE/E,MAAM,WAAW,gBAAgB;IAC/B,sEAAsE;IACtE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,6DAA6D;IAC7D,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,yEAAyE;IACzE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,aAAa;IAC5B,6EAA6E;IAC7E,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,kBAAkB;IAClB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,uDAAuD;IACvD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,kDAAkD;IAClD,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,CAAC,EAAE,gBAAgB,GAAG;QAAE,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAC;CAC3E;AAED,yCAAyC;AACzC,eAAO,MAAM,qBAAqB,EAAE,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,MAAM,CAAC,CAK1E,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,gBAAgB,EAAE,SAAS,aAAa,EAiGpD,CAAC;AAwBF;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CAAC,SAAS,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,GAAG,MAAM,CAoB7E;AAED;;;;+DAI+D;AAC/D,wBAAgB,iBAAiB,IAAI,SAAS,YAAY,EAAE,CAE3D;AA0BD;;;;;GAKG;AACH,wBAAgB,oBAAoB,IAAI,MAAM,CAW7C;AAED,gFAAgF;AAChF,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,YAAY,GAAG,MAAM,CAE5D"}
|
package/dist/run/routing-docs.js
CHANGED
|
@@ -38,10 +38,17 @@ exports.ROUTING_BOUNDARY = [
|
|
|
38
38
|
{
|
|
39
39
|
id: "B1",
|
|
40
40
|
status: "not_built",
|
|
41
|
-
capability: "
|
|
42
|
-
statement: "
|
|
43
|
-
tracking: "customer request on file (2026-08-21);
|
|
44
|
-
|
|
41
|
+
capability: "Route-declared judge (hosted lane)",
|
|
42
|
+
statement: "The SDK-lane `qualityGate` predicate (it rejects a schema-valid output as `quality_rejected`) ships; a route-DECLARED judge that runs on the hosted lane — a `judge:` block in the bundle with a rubric — is not built, and the bake-off-scoped `bakeoff.judge` (offline scoring) (which only measures value accuracy during a bake-off) is a separate SDK feature that never serves the hosted lane.",
|
|
43
|
+
tracking: "Route-declared judge — customer request on file (2026-08-21); follows the SDK predicate",
|
|
44
|
+
// The SDK predicate flipped this entry: adding `qualityGate` to route.ts would
|
|
45
|
+
// fire the old /qualityGate|"judged"/ tripwire, so it watches for the
|
|
46
|
+
// HOSTED declared-judge shape — a `judge?: JudgePolicy` field or a judge block
|
|
47
|
+
// carrying a `rubric` (spec §3). RB-1 added `bakeoff.judge?: { model,
|
|
48
|
+
// maxSamples }` (no rubric, nested in BakeoffConfig) — deliberately NOT the
|
|
49
|
+
// hosted judge, so the regex requires the rubric/JudgePolicy shape to avoid a
|
|
50
|
+
// false positive. `judgedBy:` and prose mentions of "judge" still do not match.
|
|
51
|
+
tripwire: { file: "src/types/route.ts", mustNotMatch: /\bjudge\??\s*:\s*JudgePolicy\b|\brubric\s*:/ },
|
|
45
52
|
},
|
|
46
53
|
{
|
|
47
54
|
id: "B2",
|
package/dist/run/run.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/run/run.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAMrD,OAAO,KAAK,EAAE,eAAe,EAAgB,MAAM,mBAAmB,CAAC;AAGvE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/run/run.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAMrD,OAAO,KAAK,EAAE,eAAe,EAAgB,MAAM,mBAAmB,CAAC;AAGvE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAS3D,OAAO,KAAK,EAAE,cAAc,EAAoC,MAAM,aAAa,CAAC;AAWpF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAOlD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAEtE,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IACxD,QAAQ,CAAC,SAAS,EAAE,cAAc,CAAC;IACnC,QAAQ,CAAC,YAAY,EAAE,gBAAgB,CAAC;IACxC,+EAA+E;IAC/E,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IACpC;;gDAE4C;IAC5C,QAAQ,CAAC,sBAAsB,CAAC,EAAE,MAAM,CAAC;IACzC,0EAA0E;IAC1E,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;IACvC;2EACuE;IACvE,QAAQ,CAAC,QAAQ,CAAC,EAAE,YAAY,GAAG,IAAI,CAAC;IACxC;;0BAEsB;IACtB,QAAQ,CAAC,MAAM,CAAC,EAAE,cAAc,GAAG,IAAI,CAAC;IACxC,qDAAqD;IACrD,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C;;;mFAG+E;IAC/E,QAAQ,CAAC,aAAa,CAAC,EAAE,eAAe,CAAC;IACzC;;;;;uDAKmD;IACnD,QAAQ,CAAC,QAAQ,CAAC,EAAE,oBAAoB,GAAG,IAAI,CAAC;IAChD;;;;2EAIuE;IACvE,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC;CACzB;AAsBD,wBAAsB,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CAwkBhG"}
|
package/dist/run/run.js
CHANGED
|
@@ -26,6 +26,8 @@ const validate_1 = require("./validate");
|
|
|
26
26
|
const dispatch_1 = require("./dispatch");
|
|
27
27
|
const record_attempt_1 = require("./record-attempt");
|
|
28
28
|
const ladder_key_1 = require("./ladder-key");
|
|
29
|
+
const quality_gate_1 = require("./quality-gate");
|
|
30
|
+
const accept_1 = require("./accept");
|
|
29
31
|
const run_context_1 = require("../artifacts/run-context");
|
|
30
32
|
const capture_write_1 = require("./capture-write");
|
|
31
33
|
/** Phase 1 prompt convention: the bundle's system template carries the whole
|
|
@@ -50,9 +52,13 @@ async function run(deps, routeName, opts) {
|
|
|
50
52
|
const variant = (0, variant_1.resolveVariant)(baseRoute, opts.variant, mode);
|
|
51
53
|
const servedVariant = variant?.name ?? null;
|
|
52
54
|
const route = (0, resolve_1.applyRunRequirements)((0, variant_1.effectiveRoute)(baseRoute, variant), opts.zeroRetention === true);
|
|
53
|
-
|
|
55
|
+
// RC-1: inside a run context the SDK stamps `run_id` (episodeKey, else run id)
|
|
56
|
+
// when the caller passed none — the bridge between tags and the run graph.
|
|
57
|
+
const runCtx = (0, run_context_1.getActiveRunContext)();
|
|
58
|
+
const runTags = (0, run_context_1.withRunIdTag)(opts.tags, runCtx);
|
|
59
|
+
const effectiveTags = route.require.includes("zero_retention") ? { ...runTags, zdr_enforced: "true" } : runTags;
|
|
54
60
|
// Data-hygiene warning (spec WS5): once per process per dimension.
|
|
55
|
-
(0, manifest_1.warnMissingRequiredDimensions)(deps.manifest ?? null,
|
|
61
|
+
(0, manifest_1.warnMissingRequiredDimensions)(deps.manifest ?? null, effectiveTags);
|
|
56
62
|
const ledger = (0, retry_1.createRetryLedger)(route.policy);
|
|
57
63
|
const sleep = deps.sleep ?? DEFAULT_SLEEP;
|
|
58
64
|
const guard = opts.budget
|
|
@@ -68,7 +74,6 @@ async function run(deps, routeName, opts) {
|
|
|
68
74
|
// Auto-step (E2 chunk A, spec §1.4): with an active run context, each attempt
|
|
69
75
|
// exit that records an inference also records a ground-truth step referencing
|
|
70
76
|
// it (attempt-per-row, ruling c). Inert without a run.start on this path.
|
|
71
|
-
const runCtx = (0, run_context_1.getActiveRunContext)();
|
|
72
77
|
const autoStepKey = runCtx !== null ? (0, run_context_1.resolveStepKey)(opts.stepKey, opts.tags.step, () => runCtx.nextStepKey(route.route)) : null;
|
|
73
78
|
// Per-call ladder key (spec §2.1 R0): one call_id, a monotonic attempt_seq per
|
|
74
79
|
// row, and the current candidate_index (advanced by nextCandidate below).
|
|
@@ -171,6 +176,7 @@ async function run(deps, routeName, opts) {
|
|
|
171
176
|
failure: result.failure,
|
|
172
177
|
repairedBy,
|
|
173
178
|
pricingMissing: repairFailurePricingMissing,
|
|
179
|
+
metaInputs: (0, dispatch_1.attemptMetaInputs)({ systemPrompt, userDirective: REPAIR_DIRECTIVE, outputSchema: route.outputSchema, result }),
|
|
174
180
|
});
|
|
175
181
|
if (failures_1.TERMINAL_CLASSES.has(result.failure.class)) {
|
|
176
182
|
throw new failures_1.RigFailureError(result.failure);
|
|
@@ -202,6 +208,7 @@ async function run(deps, routeName, opts) {
|
|
|
202
208
|
failure,
|
|
203
209
|
repairedBy,
|
|
204
210
|
pricingMissing: repairPricingMissing,
|
|
211
|
+
metaInputs: (0, dispatch_1.attemptMetaInputs)({ systemPrompt, userDirective: REPAIR_DIRECTIVE, outputSchema: route.outputSchema, result }),
|
|
205
212
|
});
|
|
206
213
|
captureAttempt(inferenceId, result.text);
|
|
207
214
|
if (!outcome.valid) {
|
|
@@ -386,47 +393,45 @@ async function run(deps, routeName, opts) {
|
|
|
386
393
|
servedVariant,
|
|
387
394
|
pricingMissing,
|
|
388
395
|
callId: ladder.callId,
|
|
396
|
+
metaInputs: (0, dispatch_1.attemptMetaInputs)({ systemPrompt, userDirective: USER_DIRECTIVE, outputSchema: route.outputSchema, result, tools: opts.tools }),
|
|
389
397
|
});
|
|
390
398
|
}
|
|
391
399
|
const outcome = (0, validate_1.validateOutput)(route, result.text);
|
|
400
|
+
const metaInputs = (0, dispatch_1.attemptMetaInputs)({ systemPrompt, userDirective: USER_DIRECTIVE, outputSchema: route.outputSchema, result, tools: opts.tools });
|
|
392
401
|
if (outcome.valid) {
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
402
|
+
// Q1 quality gate (spec §4.1): a schema-valid output must still clear the
|
|
403
|
+
// builder's bar before it is accepted. Tool-call turns never reach here;
|
|
404
|
+
// a `schema: null` route gates on the parsed JSON just the same. Gate
|
|
405
|
+
// logic (async await, fail-closed throw, tag/message shaping) lives in
|
|
406
|
+
// ./quality-gate.ts so this stays ONE seam.
|
|
407
|
+
const gateMeta = { route: route.route, provider: candidate.provider, model: candidate.model,
|
|
408
|
+
candidateIndex: iter.index, attemptSeq: ladder.peekNextSeq(), servedVariant };
|
|
409
|
+
const gate = opts.qualityGate === undefined
|
|
410
|
+
? null
|
|
411
|
+
: await (0, quality_gate_1.evaluateQualityGate)(opts.qualityGate, outcome.parsed, gateMeta);
|
|
412
|
+
if (gate !== null && !gate.pass) {
|
|
413
|
+
// Rejected on quality: its OWN class + budget; the repair rung is NEVER
|
|
414
|
+
// entered from here (spec §4.1 step 3). The rejected text is captured
|
|
415
|
+
// so it stays replayable.
|
|
416
|
+
const failure = { class: "quality_rejected",
|
|
417
|
+
message: (0, quality_gate_1.qualityRejectedMessage)(gate), provider: candidate.provider, model: candidate.model };
|
|
418
|
+
const rejectedId = recordAttempt((0, accept_1.successAttemptRow)(candidate, result.usage, costUsd, latencyMs, pricingMissing, failure, metaInputs, (0, quality_gate_1.qualityTags)(gate)));
|
|
419
|
+
captureAttempt(rejectedId, result.text);
|
|
420
|
+
lastFailure = failure;
|
|
421
|
+
if (ledger.consume("quality_rejected"))
|
|
422
|
+
continue; // re-sample the SAME candidate
|
|
423
|
+
candidate = nextCandidate();
|
|
424
|
+
continue;
|
|
425
|
+
}
|
|
426
|
+
const acceptTags = gate !== null ? (0, quality_gate_1.qualityTags)(gate) : undefined;
|
|
427
|
+
const inferenceId = recordAttempt((0, accept_1.successAttemptRow)(candidate, result.usage, costUsd, latencyMs, pricingMissing, null, metaInputs, acceptTags));
|
|
407
428
|
captureAttempt(inferenceId, result.text);
|
|
408
|
-
const meta = {
|
|
409
|
-
inferenceId,
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
requestedTier: route.policy.tier,
|
|
415
|
-
servedTier: result.usage.servedTier,
|
|
416
|
-
tokensIn: result.usage.tokensIn,
|
|
417
|
-
tokensOut: result.usage.tokensOut,
|
|
418
|
-
tokensCached: result.usage.tokensCached,
|
|
419
|
-
costEstimateUsd: costUsd,
|
|
420
|
-
latencyMs,
|
|
421
|
-
validated: true,
|
|
422
|
-
attemptsByClass: ledger.snapshot(),
|
|
423
|
-
servedVariant,
|
|
424
|
-
repairedBy: null,
|
|
425
|
-
callId: ladder.callId,
|
|
426
|
-
...(injectedSources !== null
|
|
427
|
-
? { citations: (0, grounding_1.extractCitations)(result.text, injectedSources) }
|
|
428
|
-
: {}),
|
|
429
|
-
};
|
|
429
|
+
const meta = (0, accept_1.buildSuccessMeta)({
|
|
430
|
+
inferenceId, route, candidate, usage: result.usage, costUsd, latencyMs,
|
|
431
|
+
attemptsByClass: ledger.snapshot(), servedVariant, callId: ladder.callId,
|
|
432
|
+
...(injectedSources !== null ? { citations: (0, grounding_1.extractCitations)(result.text, injectedSources) } : {}),
|
|
433
|
+
...(gate !== null ? { quality: { score: gate.score, verdict: "pass", judgedBy: null } } : {}),
|
|
434
|
+
});
|
|
430
435
|
return { output: outcome.parsed, meta };
|
|
431
436
|
}
|
|
432
437
|
// Schema-invalid output — a real (billed) attempt that failed.
|
|
@@ -449,6 +454,7 @@ async function run(deps, routeName, opts) {
|
|
|
449
454
|
...(result.usage.ttfbMs !== undefined ? { ttfbMs: result.usage.ttfbMs } : {}),
|
|
450
455
|
failure,
|
|
451
456
|
pricingMissing,
|
|
457
|
+
metaInputs, // computed once above (same `result`) — reused across valid/invalid
|
|
452
458
|
});
|
|
453
459
|
captureAttempt(invalidId, result.text);
|
|
454
460
|
lastFailure = failure;
|
|
@@ -501,6 +507,7 @@ async function run(deps, routeName, opts) {
|
|
|
501
507
|
...(result.usage?.ttfbMs !== undefined ? { ttfbMs: result.usage.ttfbMs } : {}),
|
|
502
508
|
failure,
|
|
503
509
|
pricingMissing: failurePricingMissing,
|
|
510
|
+
metaInputs: (0, dispatch_1.attemptMetaInputs)({ systemPrompt, userDirective: USER_DIRECTIVE, outputSchema: route.outputSchema, result, tools: opts.tools }),
|
|
504
511
|
});
|
|
505
512
|
if (failures_1.TERMINAL_CLASSES.has(failure.class)) {
|
|
506
513
|
throw new failures_1.RigFailureError(failure);
|
package/dist/settings/index.d.ts
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* Re-exported from the SDK entrypoint so `index.ts` stays under the LOC cap and
|
|
4
4
|
* the settings surface has one home.
|
|
5
5
|
*/
|
|
6
|
-
export { createTaskSettingsResolver, effectiveCapture, DEFAULT_TASK_SETTINGS, } from "./task-settings";
|
|
6
|
+
export { createTaskSettingsResolver, effectiveCapture, effectiveScrubMode, DEFAULT_TASK_SETTINGS, } from "./task-settings";
|
|
7
7
|
export type { TaskSettings, TaskSettingsResolver, TaskSettingsFetcher } from "./task-settings";
|
|
8
8
|
export { createApiTaskSettingsFetcher, startApiSettings } from "./api-fetcher";
|
|
9
9
|
export type { ApiTaskSettingsFetcherConfig } from "./api-fetcher";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/settings/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EACL,0BAA0B,EAC1B,gBAAgB,EAChB,qBAAqB,GACtB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,YAAY,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAC/F,OAAO,EAAE,4BAA4B,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAC/E,YAAY,EAAE,4BAA4B,EAAE,MAAM,eAAe,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/settings/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EACL,0BAA0B,EAC1B,gBAAgB,EAChB,kBAAkB,EAClB,qBAAqB,GACtB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,YAAY,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAC/F,OAAO,EAAE,4BAA4B,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAC/E,YAAY,EAAE,4BAA4B,EAAE,MAAM,eAAe,CAAC"}
|
package/dist/settings/index.js
CHANGED
|
@@ -5,10 +5,11 @@
|
|
|
5
5
|
* the settings surface has one home.
|
|
6
6
|
*/
|
|
7
7
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8
|
-
exports.startApiSettings = exports.createApiTaskSettingsFetcher = exports.DEFAULT_TASK_SETTINGS = exports.effectiveCapture = exports.createTaskSettingsResolver = void 0;
|
|
8
|
+
exports.startApiSettings = exports.createApiTaskSettingsFetcher = exports.DEFAULT_TASK_SETTINGS = exports.effectiveScrubMode = exports.effectiveCapture = exports.createTaskSettingsResolver = void 0;
|
|
9
9
|
var task_settings_1 = require("./task-settings");
|
|
10
10
|
Object.defineProperty(exports, "createTaskSettingsResolver", { enumerable: true, get: function () { return task_settings_1.createTaskSettingsResolver; } });
|
|
11
11
|
Object.defineProperty(exports, "effectiveCapture", { enumerable: true, get: function () { return task_settings_1.effectiveCapture; } });
|
|
12
|
+
Object.defineProperty(exports, "effectiveScrubMode", { enumerable: true, get: function () { return task_settings_1.effectiveScrubMode; } });
|
|
12
13
|
Object.defineProperty(exports, "DEFAULT_TASK_SETTINGS", { enumerable: true, get: function () { return task_settings_1.DEFAULT_TASK_SETTINGS; } });
|
|
13
14
|
var api_fetcher_1 = require("./api-fetcher");
|
|
14
15
|
Object.defineProperty(exports, "createApiTaskSettingsFetcher", { enumerable: true, get: function () { return api_fetcher_1.createApiTaskSettingsFetcher; } });
|
|
@@ -88,4 +88,14 @@ export declare function createTaskSettingsResolver(config: TaskSettingsResolverC
|
|
|
88
88
|
* because there is no control-plane row to resolve.
|
|
89
89
|
*/
|
|
90
90
|
export declare function effectiveCapture(resolver: TaskSettingsResolver | null, task: string | undefined, yamlCapture: boolean): boolean;
|
|
91
|
+
/**
|
|
92
|
+
* The EFFECTIVE scrub mode for an artifact, given the task's configured mode and
|
|
93
|
+
* the artifact's `data_classification` (screening SC-0 S8-c). A `pii`/`phi`
|
|
94
|
+
* artifact resolves a FLOOR of `redact` — content carrying PII/PHI is never
|
|
95
|
+
* uploaded verbatim just because the task's mode was `off`/`detect`. A stronger
|
|
96
|
+
* configured mode wins: `block` stays `block` (block ≥ redact — it refuses the
|
|
97
|
+
* upload entirely). A non-sensitive (or absent) classification leaves the
|
|
98
|
+
* configured mode untouched. Pure; the single place the floor is spelled.
|
|
99
|
+
*/
|
|
100
|
+
export declare function effectiveScrubMode(configured: ScrubMode, classification: string | null | undefined): ScrubMode;
|
|
91
101
|
//# sourceMappingURL=task-settings.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"task-settings.d.ts","sourceRoot":"","sources":["../../src/settings/task-settings.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"task-settings.d.ts","sourceRoot":"","sources":["../../src/settings/task-settings.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAG3C;0EAC0E;AAC1E,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,2EAA2E;IAC3E,QAAQ,CAAC,iBAAiB,EAAE,OAAO,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,gBAAgB,EAAE,OAAO,CAAC;IACnC,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;CAC3B;AAED;;;uCAGuC;AACvC,eAAO,MAAM,qBAAqB,EAAE,YAKnC,CAAC;AAEF;;;2EAG2E;AAC3E,MAAM,WAAW,mBAAmB;IAClC,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;CACnD;AAED,MAAM,WAAW,oBAAoB;IACnC;sFACkF;IAClF,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY,CAAC;IAChC;oEACgE;IAChE,QAAQ,IAAI,WAAW,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IAC9C;;gEAE4D;IAC5D,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B;2EACuE;IACvE,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACzB,4CAA4C;IAC5C,KAAK,IAAI,IAAI,CAAC;IACd,2CAA2C;IAC3C,IAAI,IAAI,IAAI,CAAC;CACd;AAED,MAAM,WAAW,0BAA0B;IACzC,QAAQ,CAAC,OAAO,EAAE,mBAAmB,CAAC;IACtC;gFAC4E;IAC5E,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,yDAAyD;IACzD,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,IAAI,EAAE,EAAE,EAAE,MAAM,KAAK,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC;IACtF,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC,MAAM,EAAE,UAAU,CAAC,OAAO,WAAW,CAAC,KAAK,IAAI,CAAC;CAC3E;AAED;;;;;GAKG;AACH,wBAAgB,0BAA0B,CACxC,MAAM,EAAE,0BAA0B,GACjC,oBAAoB,CA4DtB;AAED;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAC9B,QAAQ,EAAE,oBAAoB,GAAG,IAAI,EACrC,IAAI,EAAE,MAAM,GAAG,SAAS,EACxB,WAAW,EAAE,OAAO,GACnB,OAAO,CAGT;AAED;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAChC,UAAU,EAAE,SAAS,EACrB,cAAc,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GACxC,SAAS,CAIX"}
|
|
@@ -27,6 +27,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
27
27
|
exports.DEFAULT_TASK_SETTINGS = void 0;
|
|
28
28
|
exports.createTaskSettingsResolver = createTaskSettingsResolver;
|
|
29
29
|
exports.effectiveCapture = effectiveCapture;
|
|
30
|
+
exports.effectiveScrubMode = effectiveScrubMode;
|
|
31
|
+
const classification_1 = require("../artifacts/classification");
|
|
30
32
|
/** The safe floor for a task the control plane has not answered for yet: capture
|
|
31
33
|
* OFF, artifacts inherit the default, content OFF, scrub the schema default. A
|
|
32
34
|
* run that reaches the cache before the first fetch lands sees this — never a
|
|
@@ -114,3 +116,18 @@ function effectiveCapture(resolver, task, yamlCapture) {
|
|
|
114
116
|
return yamlCapture;
|
|
115
117
|
return resolver.get(task).capture;
|
|
116
118
|
}
|
|
119
|
+
/**
|
|
120
|
+
* The EFFECTIVE scrub mode for an artifact, given the task's configured mode and
|
|
121
|
+
* the artifact's `data_classification` (screening SC-0 S8-c). A `pii`/`phi`
|
|
122
|
+
* artifact resolves a FLOOR of `redact` — content carrying PII/PHI is never
|
|
123
|
+
* uploaded verbatim just because the task's mode was `off`/`detect`. A stronger
|
|
124
|
+
* configured mode wins: `block` stays `block` (block ≥ redact — it refuses the
|
|
125
|
+
* upload entirely). A non-sensitive (or absent) classification leaves the
|
|
126
|
+
* configured mode untouched. Pure; the single place the floor is spelled.
|
|
127
|
+
*/
|
|
128
|
+
function effectiveScrubMode(configured, classification) {
|
|
129
|
+
if (classification == null || !classification_1.SENSITIVE_CLASSIFICATIONS.has(classification))
|
|
130
|
+
return configured;
|
|
131
|
+
// Sensitive tier: never below `redact`. `block` is stronger and stays.
|
|
132
|
+
return configured === "block" ? "block" : "redact";
|
|
133
|
+
}
|