modelrig 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +125 -0
- package/README.md +54 -4
- package/dist/adapters/anthropic.d.ts.map +1 -1
- package/dist/adapters/anthropic.js +36 -7
- package/dist/adapters/attempt-meta-map.d.ts +63 -0
- package/dist/adapters/attempt-meta-map.d.ts.map +1 -0
- package/dist/adapters/attempt-meta-map.js +154 -0
- package/dist/adapters/deepseek.d.ts.map +1 -1
- package/dist/adapters/deepseek.js +25 -2
- package/dist/adapters/gemini.d.ts.map +1 -1
- package/dist/adapters/gemini.js +29 -2
- package/dist/adapters/grok.d.ts.map +1 -1
- package/dist/adapters/grok.js +25 -2
- package/dist/adapters/openai-compat.d.ts.map +1 -1
- package/dist/adapters/openai-compat.js +25 -2
- package/dist/adapters/openai-stream.d.ts +2 -0
- package/dist/adapters/openai-stream.d.ts.map +1 -1
- package/dist/adapters/openai-stream.js +23 -3
- package/dist/adapters/openai.d.ts.map +1 -1
- package/dist/adapters/openai.js +36 -6
- package/dist/artifacts/artifact.d.ts.map +1 -1
- package/dist/artifacts/artifact.js +24 -5
- package/dist/artifacts/classification.d.ts +27 -0
- package/dist/artifacts/classification.d.ts.map +1 -0
- package/dist/artifacts/classification.js +38 -0
- package/dist/artifacts/index.d.ts +3 -1
- package/dist/artifacts/index.d.ts.map +1 -1
- package/dist/artifacts/index.js +10 -1
- package/dist/artifacts/run-context.d.ts +25 -3
- package/dist/artifacts/run-context.d.ts.map +1 -1
- package/dist/artifacts/run-context.js +61 -0
- package/dist/artifacts/scrub.d.ts +26 -0
- package/dist/artifacts/scrub.d.ts.map +1 -1
- package/dist/artifacts/scrub.js +53 -0
- package/dist/artifacts/step.d.ts +9 -0
- package/dist/artifacts/step.d.ts.map +1 -1
- package/dist/artifacts/step.js +9 -0
- package/dist/bakeoff/engine.d.ts +46 -4
- package/dist/bakeoff/engine.d.ts.map +1 -1
- package/dist/bakeoff/engine.js +269 -19
- package/dist/bakeoff/eval-cases.d.ts +5 -0
- package/dist/bakeoff/eval-cases.d.ts.map +1 -1
- package/dist/bakeoff/eval-cases.js +3 -0
- package/dist/bakeoff/judge.d.ts +39 -0
- package/dist/bakeoff/judge.d.ts.map +1 -0
- package/dist/bakeoff/judge.js +102 -0
- package/dist/bakeoff/quality.d.ts +44 -0
- package/dist/bakeoff/quality.d.ts.map +1 -0
- package/dist/bakeoff/quality.js +126 -0
- package/dist/bakeoff/reference.d.ts +34 -0
- package/dist/bakeoff/reference.d.ts.map +1 -0
- package/dist/bakeoff/reference.js +68 -0
- package/dist/bakeoff/replay.d.ts +25 -0
- package/dist/bakeoff/replay.d.ts.map +1 -1
- package/dist/bakeoff/replay.js +48 -1
- package/dist/bakeoff/report.d.ts +8 -1
- package/dist/bakeoff/report.d.ts.map +1 -1
- package/dist/bakeoff/report.js +83 -12
- package/dist/bakeoff/samples.d.ts +35 -2
- package/dist/bakeoff/samples.d.ts.map +1 -1
- package/dist/bakeoff/samples.js +67 -6
- package/dist/bakeoff/stats.d.ts +17 -1
- package/dist/bakeoff/stats.d.ts.map +1 -1
- package/dist/bakeoff/stats.js +67 -0
- package/dist/bakeoff/types.d.ts +76 -5
- package/dist/bakeoff/types.d.ts.map +1 -1
- package/dist/bakeoff/types.js +4 -3
- package/dist/cli-query.d.ts +14 -0
- package/dist/cli-query.d.ts.map +1 -0
- package/dist/cli-query.js +59 -0
- package/dist/cli.d.ts +2 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +26 -23
- package/dist/config.d.ts +6 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +14 -0
- package/dist/detect/detector.d.ts +82 -0
- package/dist/detect/detector.d.ts.map +1 -0
- package/dist/detect/detector.js +23 -0
- package/dist/index.d.ts +19 -14
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +38 -11
- package/dist/query/analytics.d.ts +84 -0
- package/dist/query/analytics.d.ts.map +1 -0
- package/dist/query/analytics.js +280 -0
- package/dist/query/columns.d.ts +59 -0
- package/dist/query/columns.d.ts.map +1 -0
- package/dist/query/columns.js +89 -0
- package/dist/query/grammar.d.ts +74 -7
- package/dist/query/grammar.d.ts.map +1 -1
- package/dist/query/grammar.js +179 -71
- package/dist/query/saved.d.ts +26 -0
- package/dist/query/saved.d.ts.map +1 -0
- package/dist/query/saved.js +51 -0
- package/dist/routes/loader.js +15 -10
- package/dist/routes/objective-loader.d.ts +35 -0
- package/dist/routes/objective-loader.d.ts.map +1 -0
- package/dist/routes/objective-loader.js +286 -0
- package/dist/run/accept.d.ts +43 -0
- package/dist/run/accept.d.ts.map +1 -0
- package/dist/run/accept.js +65 -0
- package/dist/run/cost.d.ts +13 -0
- package/dist/run/cost.d.ts.map +1 -1
- package/dist/run/cost.js +27 -0
- package/dist/run/dispatch.d.ts +24 -1
- package/dist/run/dispatch.d.ts.map +1 -1
- package/dist/run/dispatch.js +19 -0
- package/dist/run/ladder-key.d.ts +4 -0
- package/dist/run/ladder-key.d.ts.map +1 -1
- package/dist/run/ladder-key.js +3 -0
- package/dist/run/quality-gate.d.ts +39 -0
- package/dist/run/quality-gate.d.ts.map +1 -0
- package/dist/run/quality-gate.js +70 -0
- package/dist/run/raw.d.ts.map +1 -1
- package/dist/run/raw.js +22 -3
- package/dist/run/record-attempt.d.ts.map +1 -1
- package/dist/run/record-attempt.js +22 -1
- package/dist/run/routing-docs.d.ts.map +1 -1
- package/dist/run/routing-docs.js +11 -4
- package/dist/run/run.d.ts.map +1 -1
- package/dist/run/run.js +46 -39
- package/dist/settings/index.d.ts +1 -1
- package/dist/settings/index.d.ts.map +1 -1
- package/dist/settings/index.js +2 -1
- package/dist/settings/task-settings.d.ts +10 -0
- package/dist/settings/task-settings.d.ts.map +1 -1
- package/dist/settings/task-settings.js +17 -0
- package/dist/telemetry/attempt-meta.d.ts +178 -0
- package/dist/telemetry/attempt-meta.d.ts.map +1 -0
- package/dist/telemetry/attempt-meta.js +319 -0
- package/dist/telemetry/exporter.d.ts.map +1 -1
- package/dist/telemetry/exporter.js +6 -0
- package/dist/telemetry/ingest-protocol.d.ts +11 -0
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
- package/dist/telemetry/ingest-protocol.js +3 -0
- package/dist/telemetry/otel-map.d.ts +10 -0
- package/dist/telemetry/otel-map.d.ts.map +1 -1
- package/dist/telemetry/otel-map.js +64 -0
- package/dist/telemetry/store-migrations.d.ts.map +1 -1
- package/dist/telemetry/store-migrations.js +15 -0
- package/dist/telemetry/store.d.ts.map +1 -1
- package/dist/telemetry/store.js +25 -7
- package/dist/types/failures-teaching.d.ts.map +1 -1
- package/dist/types/failures-teaching.js +20 -0
- package/dist/types/failures.d.ts +1 -1
- package/dist/types/failures.d.ts.map +1 -1
- package/dist/types/failures.js +1 -0
- package/dist/types/provider.d.ts +9 -0
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/quality.d.ts +41 -0
- package/dist/types/quality.d.ts.map +1 -0
- package/dist/types/quality.js +12 -0
- package/dist/types/route.d.ts +89 -0
- package/dist/types/route.d.ts.map +1 -1
- package/dist/types/telemetry.d.ts +13 -0
- package/dist/types/telemetry.d.ts.map +1 -1
- package/dist/validate-cli.d.ts.map +1 -1
- package/dist/validate-cli.js +10 -0
- package/package.json +1 -1
package/dist/bakeoff/samples.js
CHANGED
|
@@ -9,6 +9,15 @@
|
|
|
9
9
|
* key; read server-side only; deny-anon RLS. NEVER the anon staging posture.
|
|
10
10
|
*
|
|
11
11
|
* Transport is raw PostgREST over fetch (same pattern as the exporter).
|
|
12
|
+
*
|
|
13
|
+
* ── SCRUB BEFORE PERSIST (S7, screening B0-a) ───────────────────────────────
|
|
14
|
+
* `bakeoff_samples` is the largest server-side customer-content path, so the
|
|
15
|
+
* task's scrub setting — the SAME `ScrubMode` the content-custody + capture-write
|
|
16
|
+
* paths honor — runs on `inputText` AND `outputText` BEFORE any row is built or
|
|
17
|
+
* POSTed (mirrors src/run/capture-write.ts). off/detect ⇒ verbatim (detect still
|
|
18
|
+
* COUNTS findings onto the row's `arm_meta.scrub`, never values); redact ⇒ masked
|
|
19
|
+
* copy; block ⇒ a sample with any finding becomes metadata-only (input/output
|
|
20
|
+
* dropped) and counted. Findings are `{type,count}` only — never a matched value.
|
|
12
21
|
*/
|
|
13
22
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
23
|
exports.OUTPUT_MAX_BYTES = void 0;
|
|
@@ -18,6 +27,8 @@ exports.retentionExpiry = retentionExpiry;
|
|
|
18
27
|
exports.buildBakeoffSamples = buildBakeoffSamples;
|
|
19
28
|
exports.createBakeoffSamplesClient = createBakeoffSamplesClient;
|
|
20
29
|
exports.buildDirectSamplesSink = buildDirectSamplesSink;
|
|
30
|
+
exports.resolveSampleScrubMode = resolveSampleScrubMode;
|
|
31
|
+
const scrub_1 = require("../artifacts/scrub");
|
|
21
32
|
/** 256KB per output, per the architect's caps; TOAST compresses at rest. */
|
|
22
33
|
exports.OUTPUT_MAX_BYTES = 256 * 1024;
|
|
23
34
|
/** Default persists incumbent + winner only; "all" persists every arm. */
|
|
@@ -44,18 +55,53 @@ function truncateOutput(text) {
|
|
|
44
55
|
function retentionExpiry(nowMs, days = 90) {
|
|
45
56
|
return new Date(nowMs + days * 24 * 3600 * 1000).toISOString();
|
|
46
57
|
}
|
|
47
|
-
|
|
58
|
+
/** Sum two finding lists by type (counts only), sorted by type — the scrub
|
|
59
|
+
* invariant: a finding is `{type, count}`, never a value. */
|
|
60
|
+
function mergeFindings(a, b) {
|
|
61
|
+
const counts = new Map();
|
|
62
|
+
for (const f of [...a, ...b])
|
|
63
|
+
counts.set(f.type, (counts.get(f.type) ?? 0) + f.count);
|
|
64
|
+
return [...counts.entries()]
|
|
65
|
+
.map(([type, count]) => ({ type, count }))
|
|
66
|
+
.sort((x, y) => (x.type < y.type ? -1 : x.type > y.type ? 1 : 0));
|
|
67
|
+
}
|
|
68
|
+
/** One sample's input + output run through scrub `mode` (S7). off ⇒ verbatim,
|
|
69
|
+
* no findings; detect ⇒ verbatim, findings counted; redact ⇒ masked; block ⇒ a
|
|
70
|
+
* sample with any finding is dropped to metadata-only (both texts null). Pure. */
|
|
71
|
+
function scrubSample(inputText, outputText, mode) {
|
|
72
|
+
if (mode === "off")
|
|
73
|
+
return { inputText, outputText, findings: [] };
|
|
74
|
+
const inScrub = (0, scrub_1.scrubContent)(inputText, mode);
|
|
75
|
+
const outScrub = outputText === null ? null : (0, scrub_1.scrubContent)(outputText, mode);
|
|
76
|
+
const findings = mergeFindings(inScrub.findings, outScrub?.findings ?? []);
|
|
77
|
+
// block: ANY finding on either side ⇒ the whole sample is metadata-only.
|
|
78
|
+
if (mode === "block" && (inScrub.blocked || (outScrub?.blocked ?? false))) {
|
|
79
|
+
return { inputText: null, outputText: null, findings };
|
|
80
|
+
}
|
|
81
|
+
return { inputText: inScrub.text, outputText: outScrub === null ? null : outScrub.text, findings };
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Flatten arms×samples into rows, scrubbing input + output with the task's
|
|
85
|
+
* `mode` (S7, default `off` ⇒ verbatim, byte-identical). Scrub findings ride the
|
|
86
|
+
* row's `arm_meta.scrub` as counts-by-type (never values); a `block`-mode sample
|
|
87
|
+
* with any finding is stored metadata-only (input/output dropped).
|
|
88
|
+
*/
|
|
89
|
+
function buildBakeoffSamples(bakeoffId, route, arms, expiresAt, mode = "off") {
|
|
48
90
|
const rows = [];
|
|
49
91
|
for (const arm of arms) {
|
|
50
92
|
for (const s of arm.samples) {
|
|
93
|
+
const scrubbed = scrubSample(s.inputText, s.outputText, mode);
|
|
94
|
+
const armMeta = scrubbed.findings.length > 0
|
|
95
|
+
? { ...(arm.armMeta ?? {}), scrub: { mode, findings: scrubbed.findings } }
|
|
96
|
+
: arm.armMeta;
|
|
51
97
|
rows.push({
|
|
52
98
|
bakeoffId,
|
|
53
99
|
route,
|
|
54
100
|
arm: arm.arm,
|
|
55
|
-
armMeta
|
|
101
|
+
armMeta,
|
|
56
102
|
inputRef: s.inputRef,
|
|
57
|
-
inputText:
|
|
58
|
-
outputText: truncateOutput(
|
|
103
|
+
inputText: scrubbed.inputText,
|
|
104
|
+
outputText: truncateOutput(scrubbed.outputText),
|
|
59
105
|
conform: s.conform,
|
|
60
106
|
costUsd: s.costUsd,
|
|
61
107
|
latencyMs: s.latencyMs,
|
|
@@ -104,7 +150,11 @@ function createBakeoffSamplesClient(config) {
|
|
|
104
150
|
arm: s.arm,
|
|
105
151
|
arm_meta: s.armMeta,
|
|
106
152
|
input_ref: s.inputRef,
|
|
107
|
-
input_text
|
|
153
|
+
// `input_text` is NOT NULL (00008); a block-mode drop persists ""
|
|
154
|
+
// (metadata-only — zero content), never SQL null. `output_text` is
|
|
155
|
+
// nullable and rides as-is. (Spec §6/B0-a asks for null on both; the
|
|
156
|
+
// live column disallows it and this mission is ZERO-DDL — see PR body.)
|
|
157
|
+
input_text: s.inputText ?? "",
|
|
108
158
|
output_text: s.outputText,
|
|
109
159
|
conform: s.conform,
|
|
110
160
|
cost_usd: s.costUsd,
|
|
@@ -139,10 +189,21 @@ function createBakeoffSamplesClient(config) {
|
|
|
139
189
|
*/
|
|
140
190
|
function buildDirectSamplesSink(cfg) {
|
|
141
191
|
return async ({ bakeoffId, route, arms }) => {
|
|
142
|
-
const
|
|
192
|
+
const mode = resolveSampleScrubMode(cfg.resolver ?? null, route, cfg.defaultScrub ?? "off");
|
|
193
|
+
const rows = buildBakeoffSamples(bakeoffId, route, arms, retentionExpiry(Date.now()), mode);
|
|
143
194
|
await createBakeoffSamplesClient({
|
|
144
195
|
supabaseUrl: cfg.supabaseUrl,
|
|
145
196
|
serviceRoleKey: cfg.serviceRoleKey,
|
|
146
197
|
}).insert(rows);
|
|
147
198
|
};
|
|
148
199
|
}
|
|
200
|
+
/**
|
|
201
|
+
* The effective scrub mode for a bake-off sample write — resolved EXACTLY as
|
|
202
|
+
* capture-write.ts does: a configured control plane WINS (`resolver.get(route)`),
|
|
203
|
+
* else the rig's config default. A local-only / direct-mode rig has no resolver,
|
|
204
|
+
* so it falls to `configDefault`. Pure and synchronous (the resolver read is
|
|
205
|
+
* cache-only, never a network hop).
|
|
206
|
+
*/
|
|
207
|
+
function resolveSampleScrubMode(resolver, route, configDefault) {
|
|
208
|
+
return resolver !== null ? resolver.get(route).scrub : configDefault;
|
|
209
|
+
}
|
package/dist/bakeoff/stats.d.ts
CHANGED
|
@@ -7,7 +7,23 @@
|
|
|
7
7
|
* (base + retries + repair + search, F1-corrected) ÷ conformant count × 1000.
|
|
8
8
|
*/
|
|
9
9
|
import type { ReplaySampleResult } from "./replay";
|
|
10
|
-
import type { VariantMetrics } from "./types";
|
|
10
|
+
import type { RegretStats, VariantMetrics } from "./types";
|
|
11
|
+
/** Seeded so a bake-off's regret CI is reproducible given the same sample set
|
|
12
|
+
* (spec §4.2 — "seeded for determinism"). */
|
|
13
|
+
export declare const REGRET_BOOTSTRAP_REPS = 2000;
|
|
14
|
+
export declare const REGRET_BOOTSTRAP_SEED = 20260823;
|
|
15
|
+
/** Per-sample quality (spec §2): value accuracy when a golden/deterministic
|
|
16
|
+
* check/judge scored it, else 1/0 conformance. Always defined — so paired regret
|
|
17
|
+
* is always computable. */
|
|
18
|
+
export declare function sampleQuality(s: ReplaySampleResult): number;
|
|
11
19
|
export declare function wilson95(successes: number, n: number): readonly [number, number];
|
|
12
20
|
export declare function aggregateVariantMetrics(variant: string, samples: readonly ReplaySampleResult[]): VariantMetrics;
|
|
21
|
+
/**
|
|
22
|
+
* Paired regret of an arm vs the reference (spec §4.2). Pairs samples by
|
|
23
|
+
* captureId, takes per-sample quality differences `q(reference) − q(arm)`, and
|
|
24
|
+
* reports the mean, a seeded paired-bootstrap 95% CI, and n_paired. Samples
|
|
25
|
+
* present on only one side are excluded (and, by omission, counted: n_paired vs
|
|
26
|
+
* each arm's own sample count). Deterministic given the sample set + seed.
|
|
27
|
+
*/
|
|
28
|
+
export declare function pairedRegret(armSamples: readonly ReplaySampleResult[], refSamples: readonly ReplaySampleResult[], seed?: number, reps?: number): RegretStats;
|
|
13
29
|
//# sourceMappingURL=stats.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stats.d.ts","sourceRoot":"","sources":["../../src/bakeoff/stats.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;
|
|
1
|
+
{"version":3,"file":"stats.d.ts","sourceRoot":"","sources":["../../src/bakeoff/stats.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AACnD,OAAO,KAAK,EAAiB,WAAW,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAE1E;6CAC6C;AAC7C,eAAO,MAAM,qBAAqB,OAAO,CAAC;AAC1C,eAAO,MAAM,qBAAqB,WAAW,CAAC;AAE9C;;2BAE2B;AAC3B,wBAAgB,aAAa,CAAC,CAAC,EAAE,kBAAkB,GAAG,MAAM,CAE3D;AAED,wBAAgB,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAShF;AAMD,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,SAAS,kBAAkB,EAAE,GACrC,cAAc,CA0ChB;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAC1B,UAAU,EAAE,SAAS,kBAAkB,EAAE,EACzC,UAAU,EAAE,SAAS,kBAAkB,EAAE,EACzC,IAAI,GAAE,MAA8B,EACpC,IAAI,GAAE,MAA8B,GACnC,WAAW,CAwBb"}
|
package/dist/bakeoff/stats.js
CHANGED
|
@@ -8,8 +8,22 @@
|
|
|
8
8
|
* (base + retries + repair + search, F1-corrected) ÷ conformant count × 1000.
|
|
9
9
|
*/
|
|
10
10
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
11
|
+
exports.REGRET_BOOTSTRAP_SEED = exports.REGRET_BOOTSTRAP_REPS = void 0;
|
|
12
|
+
exports.sampleQuality = sampleQuality;
|
|
11
13
|
exports.wilson95 = wilson95;
|
|
12
14
|
exports.aggregateVariantMetrics = aggregateVariantMetrics;
|
|
15
|
+
exports.pairedRegret = pairedRegret;
|
|
16
|
+
const bootstrap_1 = require("../rigindex/bootstrap");
|
|
17
|
+
/** Seeded so a bake-off's regret CI is reproducible given the same sample set
|
|
18
|
+
* (spec §4.2 — "seeded for determinism"). */
|
|
19
|
+
exports.REGRET_BOOTSTRAP_REPS = 2000;
|
|
20
|
+
exports.REGRET_BOOTSTRAP_SEED = 20260823;
|
|
21
|
+
/** Per-sample quality (spec §2): value accuracy when a golden/deterministic
|
|
22
|
+
* check/judge scored it, else 1/0 conformance. Always defined — so paired regret
|
|
23
|
+
* is always computable. */
|
|
24
|
+
function sampleQuality(s) {
|
|
25
|
+
return s.valueAccuracy ?? (s.conform ? 1 : 0);
|
|
26
|
+
}
|
|
13
27
|
function wilson95(successes, n) {
|
|
14
28
|
if (n === 0)
|
|
15
29
|
return [0, 1];
|
|
@@ -32,6 +46,16 @@ function aggregateVariantMetrics(variant, samples) {
|
|
|
32
46
|
.map((s) => s.valueAccuracy)
|
|
33
47
|
.filter((a) => a !== null);
|
|
34
48
|
const totalCostUsd = samples.reduce((a, s) => a + s.costUsd, 0);
|
|
49
|
+
// RB-1: the metric is value_accuracy iff at least one sample was scored on
|
|
50
|
+
// value (eval case / value check / judge / Q1 gate); otherwise the arm's
|
|
51
|
+
// quality is 1/0 conformance. qualityMean averages per-sample quality either way.
|
|
52
|
+
const qualityMetric = accuracies.length > 0 ? "value_accuracy" : "conformance";
|
|
53
|
+
// Q1 quality pass rate (spec §4.2): a variant is "gated" once ANY sample
|
|
54
|
+
// carried a boolean verdict; a failed (unscored) sample counts against the
|
|
55
|
+
// pass rate as a non-pass, exactly as a conformance miss counts against
|
|
56
|
+
// conformance. null when no gate ran, so computeProposal can tell.
|
|
57
|
+
const gated = samples.some((s) => s.qualityPassed !== null && s.qualityPassed !== undefined);
|
|
58
|
+
const passing = samples.filter((s) => s.qualityPassed === true).length;
|
|
35
59
|
return {
|
|
36
60
|
variant,
|
|
37
61
|
samples: n,
|
|
@@ -43,6 +67,49 @@ function aggregateVariantMetrics(variant, samples) {
|
|
|
43
67
|
meanLatencyMs: mean(samples.map((s) => s.latencyMs)),
|
|
44
68
|
meanCostUsd: n === 0 ? 0 : totalCostUsd / n,
|
|
45
69
|
effectiveUsdPer1kConformant: conformant.length === 0 ? Number.POSITIVE_INFINITY : (totalCostUsd / conformant.length) * 1000,
|
|
70
|
+
qualityPassCi95: gated ? wilson95(passing, n) : null,
|
|
71
|
+
effectiveUsdPer1kPassing: !gated
|
|
72
|
+
? null
|
|
73
|
+
: passing === 0
|
|
74
|
+
? Number.POSITIVE_INFINITY
|
|
75
|
+
: (totalCostUsd / passing) * 1000,
|
|
46
76
|
pricingMissing: samples.some((s) => s.pricingMissing),
|
|
77
|
+
qualityMean: n === 0 ? null : mean(samples.map(sampleQuality)),
|
|
78
|
+
qualityMetric,
|
|
47
79
|
};
|
|
48
80
|
}
|
|
81
|
+
/**
|
|
82
|
+
* Paired regret of an arm vs the reference (spec §4.2). Pairs samples by
|
|
83
|
+
* captureId, takes per-sample quality differences `q(reference) − q(arm)`, and
|
|
84
|
+
* reports the mean, a seeded paired-bootstrap 95% CI, and n_paired. Samples
|
|
85
|
+
* present on only one side are excluded (and, by omission, counted: n_paired vs
|
|
86
|
+
* each arm's own sample count). Deterministic given the sample set + seed.
|
|
87
|
+
*/
|
|
88
|
+
function pairedRegret(armSamples, refSamples, seed = exports.REGRET_BOOTSTRAP_SEED, reps = exports.REGRET_BOOTSTRAP_REPS) {
|
|
89
|
+
const refByCapture = new Map(refSamples.map((s) => [s.captureId, s]));
|
|
90
|
+
const diffs = [];
|
|
91
|
+
for (const arm of armSamples) {
|
|
92
|
+
const ref = refByCapture.get(arm.captureId);
|
|
93
|
+
if (ref === undefined)
|
|
94
|
+
continue;
|
|
95
|
+
diffs.push(sampleQuality(ref) - sampleQuality(arm));
|
|
96
|
+
}
|
|
97
|
+
const nPaired = diffs.length;
|
|
98
|
+
if (nPaired === 0)
|
|
99
|
+
return { regretMean: 0, regretCi95: [0, 0], nPaired: 0 };
|
|
100
|
+
const regretMean = mean(diffs);
|
|
101
|
+
// Paired bootstrap: resample the DIFFERENCES with replacement (the pairing is
|
|
102
|
+
// already fixed), take each resample's mean, then the 2.5/97.5 percentiles.
|
|
103
|
+
const rng = (0, bootstrap_1.mulberry32)(seed);
|
|
104
|
+
const means = [];
|
|
105
|
+
for (let b = 0; b < reps; b += 1) {
|
|
106
|
+
let sum = 0;
|
|
107
|
+
for (let i = 0; i < nPaired; i += 1)
|
|
108
|
+
sum += diffs[Math.floor(rng() * nPaired)];
|
|
109
|
+
means.push(sum / nPaired);
|
|
110
|
+
}
|
|
111
|
+
means.sort((a, b) => a - b);
|
|
112
|
+
const lo = means[Math.floor(0.025 * reps)];
|
|
113
|
+
const hi = means[Math.min(reps - 1, Math.floor(0.975 * reps))];
|
|
114
|
+
return { regretMean, regretCi95: [lo, hi], nPaired };
|
|
115
|
+
}
|
package/dist/bakeoff/types.d.ts
CHANGED
|
@@ -1,8 +1,22 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Conformance bake-off contracts — Phase 3 spec WS5 (closed core)
|
|
3
|
-
*
|
|
4
|
-
*
|
|
2
|
+
* Conformance bake-off contracts — Phase 3 spec WS5 (closed core), extended for
|
|
3
|
+
* RB-1 regret-bounded objectives (near-optimal spec §4). The headline metric is
|
|
4
|
+
* effective $ per 1K conformant outputs (F1-corrected); RB-1 adds per-sample
|
|
5
|
+
* value accuracy, paired regret vs a reference arm, and an objective-aware gate.
|
|
5
6
|
*/
|
|
7
|
+
import type { ObjectiveMode } from "../types/route";
|
|
8
|
+
/** Which quality metric an arm's numbers are on (spec §2): value accuracy when a
|
|
9
|
+
* golden / value check / judge scored it, else schema conformance. */
|
|
10
|
+
export type QualityMetric = "value_accuracy" | "conformance";
|
|
11
|
+
/** An arm's quality gap to the reference on the SAME paired samples (spec §2).
|
|
12
|
+
* 0 = matches the reference; negative = beats it. CI is a seeded paired
|
|
13
|
+
* bootstrap of the per-sample differences. */
|
|
14
|
+
export interface RegretStats {
|
|
15
|
+
readonly regretMean: number;
|
|
16
|
+
readonly regretCi95: readonly [number, number];
|
|
17
|
+
/** Samples present on BOTH the arm and the reference; the CI's n. */
|
|
18
|
+
readonly nPaired: number;
|
|
19
|
+
}
|
|
6
20
|
export interface VariantMetrics {
|
|
7
21
|
readonly variant: string;
|
|
8
22
|
readonly samples: number;
|
|
@@ -16,9 +30,58 @@ export interface VariantMetrics {
|
|
|
16
30
|
readonly meanCostUsd: number;
|
|
17
31
|
/** (base + repair + search costs) / conformant outputs × 1000 — the number. */
|
|
18
32
|
readonly effectiveUsdPer1kConformant: number;
|
|
33
|
+
/** Q1 quality gate (spec §4.2): Wilson 95% CI on the quality PASS rate
|
|
34
|
+
* (samples the gate accepted / all samples). null when no gate ran on this
|
|
35
|
+
* variant — computeProposal's third clause only fires when both arms have one. */
|
|
36
|
+
readonly qualityPassCi95: readonly [number, number] | null;
|
|
37
|
+
/** Q1: (base + repair + search costs) / quality-PASSING outputs × 1000 —
|
|
38
|
+
* reported BESIDE effectiveUsdPer1kConformant (never redefining it; flag F6).
|
|
39
|
+
* ∞ when a gate ran but nothing passed; null when no gate ran. */
|
|
40
|
+
readonly effectiveUsdPer1kPassing: number | null;
|
|
19
41
|
/** Addendum A1: any sample served by a model with no pricing entry —
|
|
20
42
|
* cost columns under-count; this variant is never ranked or proposed. */
|
|
21
43
|
readonly pricingMissing: boolean;
|
|
44
|
+
/** RB-1: mean per-sample quality (value accuracy where present, else 1/0
|
|
45
|
+
* conformance). The number regret is computed from. Null when no samples. */
|
|
46
|
+
readonly qualityMean: number | null;
|
|
47
|
+
/** RB-1: which metric qualityMean is on, named on the report. */
|
|
48
|
+
readonly qualityMetric: QualityMetric;
|
|
49
|
+
/** RB-1: this arm is the reference (ceiling) arm. */
|
|
50
|
+
readonly isReference?: boolean;
|
|
51
|
+
/** RB-1 (architect B-3): the objective gate proposed THIS arm — stamped on the
|
|
52
|
+
* winning variant so the mirror/console read the artifact's decision rather
|
|
53
|
+
* than recomputing a (mode-blind) winner client-side. */
|
|
54
|
+
readonly proposed?: boolean;
|
|
55
|
+
/** RB-1: this arm's regret vs the reference; null on the reference arm and
|
|
56
|
+
* when there is no reference. */
|
|
57
|
+
readonly regret?: RegretStats | null;
|
|
58
|
+
/** RB-1: the reference arm hit its envelope — regret is on the paired subset. */
|
|
59
|
+
readonly referencePartial?: boolean;
|
|
60
|
+
}
|
|
61
|
+
/** RB-1: the resolved objective recorded on the artifact (evidence + report). */
|
|
62
|
+
export interface BakeoffObjective {
|
|
63
|
+
readonly mode: ObjectiveMode;
|
|
64
|
+
readonly maxRegret: number | null;
|
|
65
|
+
readonly priceCeiling: {
|
|
66
|
+
readonly usdPer1k?: number;
|
|
67
|
+
readonly usdPerRun?: number;
|
|
68
|
+
} | null;
|
|
69
|
+
readonly lambda: number | null;
|
|
70
|
+
}
|
|
71
|
+
/** RB-1: the resolved reference arm recorded on the artifact. */
|
|
72
|
+
export interface BakeoffReference {
|
|
73
|
+
/** The arm name carrying the reference metrics. */
|
|
74
|
+
readonly variant: string;
|
|
75
|
+
/** "provider/model". */
|
|
76
|
+
readonly model: string;
|
|
77
|
+
/** Resolved via `reference: auto` (vs an explicit declaration). */
|
|
78
|
+
readonly auto: boolean;
|
|
79
|
+
/** The `auto` rule + source, when auto. */
|
|
80
|
+
readonly rule: string | null;
|
|
81
|
+
/** Out-of-candidate reference (reference_allow_external). */
|
|
82
|
+
readonly external: boolean;
|
|
83
|
+
/** The reference replay hit its envelope (partial paired set). */
|
|
84
|
+
readonly partial: boolean;
|
|
22
85
|
}
|
|
23
86
|
export interface BakeoffArtifact {
|
|
24
87
|
readonly id: string;
|
|
@@ -27,11 +90,19 @@ export interface BakeoffArtifact {
|
|
|
27
90
|
readonly ranAt: string;
|
|
28
91
|
readonly replayN: number;
|
|
29
92
|
readonly variants: readonly VariantMetrics[];
|
|
30
|
-
/** Winner
|
|
31
|
-
*
|
|
93
|
+
/** Winner among variants meeting the objective's gates; null when nobody beats
|
|
94
|
+
* the incumbent. `vsIncumbentSavingsPct` is the effective-cost saving (kept for
|
|
95
|
+
* every mode); `regret` is the proposed arm's regret vs the reference. */
|
|
32
96
|
readonly proposal: {
|
|
33
97
|
readonly variant: string;
|
|
34
98
|
readonly vsIncumbentSavingsPct: number;
|
|
99
|
+
readonly regret?: RegretStats | null;
|
|
35
100
|
} | null;
|
|
101
|
+
/** RB-1: the objective the gate ran under (default quality-first). */
|
|
102
|
+
readonly objective?: BakeoffObjective;
|
|
103
|
+
/** RB-1: the reference arm, or null when no reference was declared/resolved. */
|
|
104
|
+
readonly reference?: BakeoffReference | null;
|
|
105
|
+
/** RB-1: the metric the bake-off's quality numbers are on. */
|
|
106
|
+
readonly qualityMetric?: QualityMetric;
|
|
36
107
|
}
|
|
37
108
|
//# sourceMappingURL=types.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/bakeoff/types.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/bakeoff/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAEpD;sEACsE;AACtE,MAAM,MAAM,aAAa,GAAG,gBAAgB,GAAG,aAAa,CAAC;AAE7D;;8CAE8C;AAC9C,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/C,qEAAqE;IACrE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,wFAAwF;IACxF,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1C,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,+EAA+E;IAC/E,QAAQ,CAAC,2BAA2B,EAAE,MAAM,CAAC;IAC7C;;sFAEkF;IAClF,QAAQ,CAAC,eAAe,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;IAC3D;;sEAEkE;IAClE,QAAQ,CAAC,wBAAwB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjD;6EACyE;IACzE,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC;iFAC6E;IAC7E,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,iEAAiE;IACjE,QAAQ,CAAC,aAAa,EAAE,aAAa,CAAC;IACtC,qDAAqD;IACrD,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;IAC/B;;6DAEyD;IACzD,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;IAC5B;qCACiC;IACjC,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;IACrC,iFAAiF;IACjF,QAAQ,CAAC,gBAAgB,CAAC,EAAE,OAAO,CAAC;CACrC;AAED,iFAAiF;AACjF,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,YAAY,EAAE;QAAE,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IAC1F,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CAChC;AAED,iEAAiE;AACjE,MAAM,WAAW,gBAAgB;IAC/B,mDAAmD;IACnD,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,wBAAwB;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,mEAAmE;IACnE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,2CAA2C;IAC3C,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,6DAA6D;IAC7D,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B,kEAAkE;IAClE,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAC;IAC7C;;8EAE0E;IAC1E,QAAQ,CAAC,QAAQ,EACb;QACE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;QACvC,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;KACtC,GACD,IAAI,CAAC;IACT,sEAAsE;IACtE,QAAQ,CAAC,SAAS,CAAC,EAAE,gBAAgB,CAAC;IACtC,gFAAgF;IAChF,QAAQ,CAAC,SAAS,CAAC,EAAE,gBAAgB,GAAG,IAAI,CAAC;IAC7C,8DAA8D;IAC9D,QAAQ,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;CACxC"}
|
package/dist/bakeoff/types.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/**
|
|
3
|
-
* Conformance bake-off contracts — Phase 3 spec WS5 (closed core)
|
|
4
|
-
*
|
|
5
|
-
*
|
|
3
|
+
* Conformance bake-off contracts — Phase 3 spec WS5 (closed core), extended for
|
|
4
|
+
* RB-1 regret-bounded objectives (near-optimal spec §4). The headline metric is
|
|
5
|
+
* effective $ per 1K conformant outputs (F1-corrected); RB-1 adds per-sample
|
|
6
|
+
* value accuracy, paired regret vs a reference arm, and an objective-aware gate.
|
|
6
7
|
*/
|
|
7
8
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8
9
|
// Implemented: engine.ts assembles this from replay results; stats.ts owns
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `modelrig query` — WI-1b. Runs a telemetry-grammar query against the analytics
|
|
3
|
+
* surface of the configured server (`POST /v1/analytics/query`), authed with a
|
|
4
|
+
* `rig_sk_` key. Goes through the API boundary (T12: the CLI never holds a DB
|
|
5
|
+
* client); the server scopes to the key's org and folds the grammar.
|
|
6
|
+
*
|
|
7
|
+
* modelrig query "SELECT subject, sum(cost) FROM inferences GROUP BY subject"
|
|
8
|
+
* modelrig query --saved cost-per-subject
|
|
9
|
+
*
|
|
10
|
+
* Kept in its own file so cli.ts stays under the LOC cap (like validate-cli.ts).
|
|
11
|
+
*/
|
|
12
|
+
import type { Flags } from "./cli";
|
|
13
|
+
export declare function cmdQuery(flags: Flags): Promise<number>;
|
|
14
|
+
//# sourceMappingURL=cli-query.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli-query.d.ts","sourceRoot":"","sources":["../src/cli-query.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAIH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAMnC,wBAAsB,QAAQ,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAyC5D"}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* `modelrig query` — WI-1b. Runs a telemetry-grammar query against the analytics
|
|
4
|
+
* surface of the configured server (`POST /v1/analytics/query`), authed with a
|
|
5
|
+
* `rig_sk_` key. Goes through the API boundary (T12: the CLI never holds a DB
|
|
6
|
+
* client); the server scopes to the key's org and folds the grammar.
|
|
7
|
+
*
|
|
8
|
+
* modelrig query "SELECT subject, sum(cost) FROM inferences GROUP BY subject"
|
|
9
|
+
* modelrig query --saved cost-per-subject
|
|
10
|
+
*
|
|
11
|
+
* Kept in its own file so cli.ts stays under the LOC cap (like validate-cli.ts).
|
|
12
|
+
*/
|
|
13
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
|
+
exports.cmdQuery = cmdQuery;
|
|
15
|
+
const config_1 = require("./config");
|
|
16
|
+
const grammar_1 = require("./query/grammar");
|
|
17
|
+
function savedCatalogue() {
|
|
18
|
+
return grammar_1.SAVED_QUERIES.map((q) => ` ${q.id.padEnd(20)} ${q.title}`).join("\n");
|
|
19
|
+
}
|
|
20
|
+
async function cmdQuery(flags) {
|
|
21
|
+
const saved = flags.named.get("saved");
|
|
22
|
+
const grammar = flags.positional[0];
|
|
23
|
+
// Config is the only env reader (package-invariants). The analytics endpoint
|
|
24
|
+
// shares the api-mode ingest server, so base = that URL's origin, key = the api key.
|
|
25
|
+
const exp = (0, config_1.loadConfigFromEnv)().export;
|
|
26
|
+
const api = exp !== undefined && exp.mode === "api" ? exp : undefined;
|
|
27
|
+
const base = api !== undefined ? new URL(api.ingestUrl).origin : "";
|
|
28
|
+
const key = api?.apiKey ?? "";
|
|
29
|
+
if ((saved === undefined && (grammar === undefined || grammar === "")) || base === "" || key === "") {
|
|
30
|
+
console.error([
|
|
31
|
+
'usage: modelrig query "<grammar>" | modelrig query --saved <id>',
|
|
32
|
+
"",
|
|
33
|
+
"Runs a telemetry query against your org's analytics surface (metadata only —",
|
|
34
|
+
"enums, numbers, hashes, identifiers; never content). Needs api-mode telemetry:",
|
|
35
|
+
" MODELRIG_API_KEY a rig_sk_ key with the `query` scope",
|
|
36
|
+
" MODELRIG_INGEST_URL optional — the server base; defaults to the hosted API",
|
|
37
|
+
"",
|
|
38
|
+
"Saved questions:",
|
|
39
|
+
savedCatalogue(),
|
|
40
|
+
].join("\n"));
|
|
41
|
+
return 1;
|
|
42
|
+
}
|
|
43
|
+
const body = saved !== undefined ? { saved } : { query: grammar };
|
|
44
|
+
const res = await fetch(`${base}/v1/analytics/query`, {
|
|
45
|
+
method: "POST",
|
|
46
|
+
headers: { authorization: `Bearer ${key}`, "content-type": "application/json" },
|
|
47
|
+
body: JSON.stringify(body),
|
|
48
|
+
});
|
|
49
|
+
const payload = (await res.json().catch(() => ({})));
|
|
50
|
+
if (!res.ok) {
|
|
51
|
+
console.error(`query ${res.status}: ${typeof payload.error === "string" ? payload.error : "failed"}`);
|
|
52
|
+
return 1;
|
|
53
|
+
}
|
|
54
|
+
console.log(JSON.stringify(payload, null, 2));
|
|
55
|
+
if (payload.truncated === true) {
|
|
56
|
+
console.error("note: result truncated (LIMIT or the 10k-row scan cap) — narrow the window or raise LIMIT");
|
|
57
|
+
}
|
|
58
|
+
return 0;
|
|
59
|
+
}
|
package/dist/cli.d.ts
CHANGED
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
/**
|
|
3
3
|
* modelrig CLI — Phase 3 plan §3 + Phase 4 plan §3:
|
|
4
4
|
* modelrig bakeoff --route X --variants a,b,c --replay-last N
|
|
5
|
-
* [--budget-usd usd] [--
|
|
5
|
+
* [--budget-usd usd] [--objective quality-first|balanced|cost-first]
|
|
6
|
+
* [--json] [--no-mirror] (--envelope is a kept alias)
|
|
6
7
|
* modelrig bakeoff --from-proposal <id> [--replay-last N] [--json]
|
|
7
8
|
* modelrig watch run [--events fixtures.json] [--json]
|
|
8
9
|
*
|
package/dist/cli.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA
|
|
1
|
+
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;;GAgBG;AAgDH,MAAM,WAAW,KAAK;IACpB,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5C,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CACpC;AAeD,wBAAgB,SAAS,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,KAAK,CAuBxD;AAqhBD,wBAAsB,IAAI,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAuCnE"}
|
package/dist/cli.js
CHANGED
|
@@ -3,7 +3,8 @@
|
|
|
3
3
|
/**
|
|
4
4
|
* modelrig CLI — Phase 3 plan §3 + Phase 4 plan §3:
|
|
5
5
|
* modelrig bakeoff --route X --variants a,b,c --replay-last N
|
|
6
|
-
* [--budget-usd usd] [--
|
|
6
|
+
* [--budget-usd usd] [--objective quality-first|balanced|cost-first]
|
|
7
|
+
* [--json] [--no-mirror] (--envelope is a kept alias)
|
|
7
8
|
* modelrig bakeoff --from-proposal <id> [--replay-last N] [--json]
|
|
8
9
|
* modelrig watch run [--events fixtures.json] [--json]
|
|
9
10
|
*
|
|
@@ -33,6 +34,17 @@ const status_1 = require("./status");
|
|
|
33
34
|
const engine_1 = require("./bakeoff/engine");
|
|
34
35
|
const samples_1 = require("./bakeoff/samples");
|
|
35
36
|
const report_1 = require("./bakeoff/report");
|
|
37
|
+
const OBJECTIVE_MODES = new Set(["quality-first", "balanced", "cost-first"]);
|
|
38
|
+
/** Parse & validate `--objective <mode>`; undefined when absent. */
|
|
39
|
+
function objectiveOf(flags) {
|
|
40
|
+
const raw = flags.named.get("objective");
|
|
41
|
+
if (raw === undefined)
|
|
42
|
+
return undefined;
|
|
43
|
+
if (!OBJECTIVE_MODES.has(raw)) {
|
|
44
|
+
throw new Error(`--objective must be one of [${[...OBJECTIVE_MODES].join(", ")}]`);
|
|
45
|
+
}
|
|
46
|
+
return raw;
|
|
47
|
+
}
|
|
36
48
|
const cli_run_1 = require("./verify/cli-run");
|
|
37
49
|
const loader_1 = require("./routes/loader");
|
|
38
50
|
const rawonly_1 = require("./run/rawonly");
|
|
@@ -45,6 +57,7 @@ const state_1 = require("./watch/state");
|
|
|
45
57
|
const state_store_1 = require("./watch/state-store");
|
|
46
58
|
const swaps_client_1 = require("./savings/swaps-client");
|
|
47
59
|
const swap_execute_1 = require("./savings/swap-execute");
|
|
60
|
+
const cli_query_1 = require("./cli-query");
|
|
48
61
|
const BARE_FLAGS = new Set([
|
|
49
62
|
"json",
|
|
50
63
|
"no-mirror",
|
|
@@ -99,7 +112,7 @@ function keepOutputsMode(flags) {
|
|
|
99
112
|
return "winner";
|
|
100
113
|
return undefined;
|
|
101
114
|
}
|
|
102
|
-
async function runAndReportBakeoff(config, flags, route, variants, replayLastN, envelope) {
|
|
115
|
+
async function runAndReportBakeoff(config, flags, route, variants, replayLastN, envelope, objectiveMode) {
|
|
103
116
|
const keep = keepOutputsMode(flags);
|
|
104
117
|
if (keep !== undefined)
|
|
105
118
|
(0, config_1.directExportOf)(config, "--keep-outputs (bakeoff_samples lives there)");
|
|
@@ -111,6 +124,7 @@ async function runAndReportBakeoff(config, flags, route, variants, replayLastN,
|
|
|
111
124
|
replayLastN,
|
|
112
125
|
...(envelope !== undefined ? { envelopeUsdPerVariant: envelope } : {}),
|
|
113
126
|
...(keep !== undefined ? { keepOutputs: keep } : {}),
|
|
127
|
+
...(objectiveMode !== undefined ? { objectiveMode } : {}),
|
|
114
128
|
});
|
|
115
129
|
const outDir = (0, node_path_1.join)((0, node_path_1.dirname)(config.telemetryDbPath), "bakeoffs");
|
|
116
130
|
(0, node_fs_1.mkdirSync)(outDir, { recursive: true });
|
|
@@ -194,26 +208,12 @@ async function cmdBakeoff(flags) {
|
|
|
194
208
|
const perVariant = reservation.remainingUsd / variants.length;
|
|
195
209
|
console.log(`watch budget ${reservation.key}: $${reservation.remainingUsd.toFixed(2)} remaining → ` +
|
|
196
210
|
`$${perVariant.toFixed(2)}/variant cap`);
|
|
197
|
-
const artifact = await runAndReportBakeoff(config, flags, proposal.route, variants, replayLastN, perVariant);
|
|
211
|
+
const artifact = await runAndReportBakeoff(config, flags, proposal.route, variants, replayLastN, perVariant, objectiveOf(flags));
|
|
198
212
|
const spentUsd = artifact.variants.reduce((a, m) => a + m.samples * m.meanCostUsd, 0);
|
|
199
213
|
budgetStore.accrue(reservation.key, spentUsd);
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
ran_at: artifact.ranAt,
|
|
204
|
-
spent_usd: spentUsd,
|
|
205
|
-
proposal: artifact.proposal,
|
|
206
|
-
matrix: artifact.variants.map((m) => ({
|
|
207
|
-
variant: m.variant,
|
|
208
|
-
samples: m.samples,
|
|
209
|
-
conform_rate: m.conformRate,
|
|
210
|
-
conform_ci95: m.conformCi95,
|
|
211
|
-
effective_usd_per_1k_conformant: Number.isFinite(m.effectiveUsdPer1kConformant)
|
|
212
|
-
? m.effectiveUsdPer1kConformant
|
|
213
|
-
: null,
|
|
214
|
-
pricing_missing: m.pricingMissing,
|
|
215
|
-
})),
|
|
216
|
-
});
|
|
214
|
+
// Evidence jsonb (winner + RB-1 objective/reference/regret/quality) is
|
|
215
|
+
// built by the engine so the CLI stays thin; the PR body reads it back.
|
|
216
|
+
await client.attachEvidence(proposal.id, (0, engine_1.bakeoffEvidence)(artifact, spentUsd));
|
|
217
217
|
console.log(`evidence attached → proposal ${proposal.id} is now evidenced ` +
|
|
218
218
|
`($${spentUsd.toFixed(2)} accrued to ${reservation.key}). ` +
|
|
219
219
|
"Approval stays human: modelrig swap execute <proposalId> --approve");
|
|
@@ -234,7 +234,7 @@ async function cmdBakeoff(flags) {
|
|
|
234
234
|
const env = envRaw !== undefined ? Number(envRaw) : undefined;
|
|
235
235
|
if (env !== undefined && (!Number.isFinite(env) || env <= 0))
|
|
236
236
|
throw new Error("--budget-usd must be positive");
|
|
237
|
-
return (0, report_1.runEvalCaseBakeoffCli)(config, evalTask, vs, env, keepOutputsMode(flags), flags.bare.has("json"));
|
|
237
|
+
return (0, report_1.runEvalCaseBakeoffCli)(config, evalTask, vs, env, keepOutputsMode(flags), flags.bare.has("json"), objectiveOf(flags));
|
|
238
238
|
}
|
|
239
239
|
const route = flags.named.get("route");
|
|
240
240
|
const variantsRaw = flags.named.get("variants");
|
|
@@ -253,7 +253,7 @@ async function cmdBakeoff(flags) {
|
|
|
253
253
|
if (envelope !== undefined && (!Number.isFinite(envelope) || envelope <= 0)) {
|
|
254
254
|
throw new Error("--budget-usd must be a positive number of dollars (per replay variant)");
|
|
255
255
|
}
|
|
256
|
-
await runAndReportBakeoff(config, flags, route, variants, replayLastN, envelope);
|
|
256
|
+
await runAndReportBakeoff(config, flags, route, variants, replayLastN, envelope, objectiveOf(flags));
|
|
257
257
|
return 0;
|
|
258
258
|
}
|
|
259
259
|
const DEFAULT_WATCH_THRESHOLD_PCT = 10;
|
|
@@ -542,13 +542,14 @@ const USAGE = [
|
|
|
542
542
|
" modelrig validate (structurally check rig.yaml + every route bundle — no provider key needed)",
|
|
543
543
|
" modelrig status (print export/artifact/otlp posture)",
|
|
544
544
|
" modelrig serve (needs the modelrig-server package)",
|
|
545
|
-
" modelrig bakeoff --route <route> --variants a,b,c --replay-last N [--budget-usd usd] [--keep-outputs[=all]] [--json] [--no-mirror]",
|
|
545
|
+
" modelrig bakeoff --route <route> --variants a,b,c --replay-last N [--budget-usd usd] [--objective quality-first|balanced|cost-first] [--keep-outputs[=all]] [--json] [--no-mirror]",
|
|
546
546
|
" modelrig bakeoff --from-proposal <id> [--replay-last N] [--keep-outputs[=all]] [--json]",
|
|
547
547
|
" modelrig watch run [--events fixtures.json] [--json]",
|
|
548
548
|
" modelrig verify-swap --route <route> --captures <fixture.json> [--budget-usd usd] [--out report.md] [--json]",
|
|
549
549
|
" modelrig swap execute <proposalId> --approve [--approved-by name]",
|
|
550
550
|
" modelrig swap revert <swapId>",
|
|
551
551
|
" modelrig judge <runId> (E5: score a run with the bounded model-judge; explicit + metered)",
|
|
552
|
+
' modelrig query "<grammar>" | query --saved <id> (WI-1b: telemetry analytics — metadata only, never content)',
|
|
552
553
|
" modelrig samples purge [--bakeoff <id>]",
|
|
553
554
|
].join("\n");
|
|
554
555
|
async function main(argv) {
|
|
@@ -582,6 +583,8 @@ async function main(argv) {
|
|
|
582
583
|
return cmdVerifySwap(flags);
|
|
583
584
|
case "judge":
|
|
584
585
|
return cmdJudge(flags);
|
|
586
|
+
case "query":
|
|
587
|
+
return (0, cli_query_1.cmdQuery)(flags);
|
|
585
588
|
case "serve":
|
|
586
589
|
return cmdServe();
|
|
587
590
|
default:
|
package/dist/config.d.ts
CHANGED
|
@@ -4,6 +4,12 @@
|
|
|
4
4
|
* __tests__/package-invariants.test.ts).
|
|
5
5
|
*/
|
|
6
6
|
import type { CapabilityFlag, ProviderId } from "./types/provider";
|
|
7
|
+
/** The pipeline git sha from a deployment env var, if any (WI-1 RunMeta, spec
|
|
8
|
+
* §3), in priority order: an explicit override, then the CI/platform commit vars
|
|
9
|
+
* ModelRig deploys on (Vercel/Render). Env reads live ONLY here (the process.env
|
|
10
|
+
* invariant); `telemetry/attempt-meta.ts:resolvePipelineSha` composes this with
|
|
11
|
+
* a `.git/HEAD` fallback and never throws. Returns undefined when none is set. */
|
|
12
|
+
export declare function pipelineShaFromEnv(): string | undefined;
|
|
7
13
|
export interface RigConfig {
|
|
8
14
|
/** Directory of route bundles, OR `null` for a first-class runRaw-only rig
|
|
9
15
|
* (CE-8, `ce8-runraw-only-rig-spec-plan.md` R1). With `null`, createRig skips
|
package/dist/config.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAEnE,MAAM,WAAW,SAAS;IACxB;;;;;4EAKwE;IACxE,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC;;;;qFAIiF;IACjF,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,IAAI,EAAE;QACb,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAC5B,qDAAqD;QACrD,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QACvB,+DAA+D;QAC/D,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAC5B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;KAC7B,CAAC;IACF,kFAAkF;IAClF,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IACpC;8EAC0E;IAC1E,QAAQ,CAAC,sBAAsB,CAAC,EAAE,MAAM,CAAC;IACzC;;;6CAGyC;IACzC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,SAAS,cAAc,EAAE,CAAC,CAAC,CAAC;IACtF;8DAC0D;IAC1D,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B;;eAEW;IACX,QAAQ,CAAC,MAAM,CAAC,EAAE;QAChB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;QAC9B,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;QACnC,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,GAAG,UAAU,CAAC;KAC7C,CAAC;IACF;;wCAEoC;IACpC,QAAQ,CAAC,KAAK,CAAC,EAAE;QACf,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;QAC7B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;KAChC,CAAC;IACF;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,eAAe,GAAG,kBAAkB,CAAC;IACvD;;;;;;;;OAQG;IACH,QAAQ,CAAC,IAAI,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC;IAClC;;;;;;;;OAQG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,eAAe,CAAC;CACtC;AAED;iCACiC;AACjC,MAAM,MAAM,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE9D;;;yBAGyB;AACzB,MAAM,WAAW,qBAAqB;IACpC,qEAAqE;IACrE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,qFAAqF;IACrF,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;IAC1B,2EAA2E;IAC3E,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED;6EAC6E;AAC7E,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,OAAO,CAAC,EAAE,qBAAqB,CAAC;CAC1C;AAWD;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,sBAAsB,CACpC,GAAG,EAAE;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;CAC5B;AACD;;0DAE0D;AAC1D,sBAAsB,UAAQ,GAC7B,eAAe,CAuBjB;AAED;uEACuE;AACvE,MAAM,WAAW,UAAU;IACzB,kFAAkF;IAClF,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,mFAAmF;IACnF,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC1C;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAYhF;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAC/B,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,UAAU,EAAE,MAAM,GAAG,SAAS,GAC7B,UAAU,GAAG,IAAI,CAGnB;AAED,UAAU,YAAY;IACpB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,eAAgB,SAAQ,YAAY;IACnD,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC;IACrB;+EAC2E;IAC3E,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,kBAAmB,SAAQ,YAAY;IACtD,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC;;uCAEmC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC;AAED;;kEAEkE;AAClE,eAAO,MAAM,gBAAgB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAQ7D,CAAC;AAKF,wBAAgB,iBAAiB,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,SAAS,CAgI3E;AAED;;;;;;;;GAQG;AACH,wBAAgB,cAAc,CAC5B,MAAM,EAAE,SAAS,EACjB,IAAI,EAAE,MAAM,GACX;IAAE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAgB/F"}
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAEnE;;;;kFAIkF;AAClF,wBAAgB,kBAAkB,IAAI,MAAM,GAAG,SAAS,CAMvD;AAED,MAAM,WAAW,SAAS;IACxB;;;;;4EAKwE;IACxE,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC;;;;qFAIiF;IACjF,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,IAAI,EAAE;QACb,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAC5B,qDAAqD;QACrD,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QACvB,+DAA+D;QAC/D,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAC5B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;KAC7B,CAAC;IACF,kFAAkF;IAClF,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IACpC;8EAC0E;IAC1E,QAAQ,CAAC,sBAAsB,CAAC,EAAE,MAAM,CAAC;IACzC;;;6CAGyC;IACzC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,SAAS,cAAc,EAAE,CAAC,CAAC,CAAC;IACtF;8DAC0D;IAC1D,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B;;eAEW;IACX,QAAQ,CAAC,MAAM,CAAC,EAAE;QAChB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;QAC9B,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;QACnC,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,GAAG,UAAU,CAAC;KAC7C,CAAC;IACF;;wCAEoC;IACpC,QAAQ,CAAC,KAAK,CAAC,EAAE;QACf,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;QAC7B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;KAChC,CAAC;IACF;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,eAAe,GAAG,kBAAkB,CAAC;IACvD;;;;;;;;OAQG;IACH,QAAQ,CAAC,IAAI,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC;IAClC;;;;;;;;OAQG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,eAAe,CAAC;CACtC;AAED;iCACiC;AACjC,MAAM,MAAM,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE9D;;;yBAGyB;AACzB,MAAM,WAAW,qBAAqB;IACpC,qEAAqE;IACrE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,qFAAqF;IACrF,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;IAC1B,2EAA2E;IAC3E,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED;6EAC6E;AAC7E,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,OAAO,CAAC,EAAE,qBAAqB,CAAC;CAC1C;AAWD;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,sBAAsB,CACpC,GAAG,EAAE;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;CAC5B;AACD;;0DAE0D;AAC1D,sBAAsB,UAAQ,GAC7B,eAAe,CAuBjB;AAED;uEACuE;AACvE,MAAM,WAAW,UAAU;IACzB,kFAAkF;IAClF,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,mFAAmF;IACnF,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC1C;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAYhF;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAC/B,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,UAAU,EAAE,MAAM,GAAG,SAAS,GAC7B,UAAU,GAAG,IAAI,CAGnB;AAED,UAAU,YAAY;IACpB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,eAAgB,SAAQ,YAAY;IACnD,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC;IACrB;+EAC2E;IAC3E,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,kBAAmB,SAAQ,YAAY;IACtD,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC;;uCAEmC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC;AAED;;kEAEkE;AAClE,eAAO,MAAM,gBAAgB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAQ7D,CAAC;AAKF,wBAAgB,iBAAiB,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,SAAS,CAgI3E;AAED;;;;;;;;GAQG;AACH,wBAAgB,cAAc,CAC5B,MAAM,EAAE,SAAS,EACjB,IAAI,EAAE,MAAM,GACX;IAAE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAgB/F"}
|